[svn] r6761: vs-plugin/trunk:
Nemerle.Compiler.Utils/Nemerle.Completion2/CodeModel/Project.n
Nemerle.Compi...
IT
svnadmin at nemerle.org
Mon Oct 16 04:33:21 CEST 2006
Log:
ScanLexer. Finished strings.
Author: IT
Date: Mon Oct 16 04:33:15 2006
New Revision: 6761
Modified:
vs-plugin/trunk/Nemerle.Compiler.Utils/Nemerle.Completion2/CodeModel/Project.n
vs-plugin/trunk/Nemerle.Compiler.Utils/Nemerle.Completion2/CodeModel/ScanLexer.n
vs-plugin/trunk/Nemerle.Compiler.Utils/Nemerle.Completion2/CodeModel/ScanState.n
vs-plugin/trunk/Nemerle.Compiler.Utils/Nemerle.Completion2/CodeModel/ScanTokenColor.n
vs-plugin/trunk/Nemerle.Compiler.Utils/Nemerle.Completion2/CodeModel/ScanTokenInfo.n
vs-plugin/trunk/Nemerle.Compiler.Utils/Nemerle.Completion2/CompiledUnitAstBrowser.n
vs-plugin/trunk/Nemerle.VsIntegration/LanguageService/NemerleLanguageService.cs
vs-plugin/trunk/Nemerle.VsIntegration/LanguageService/NemerleScanner.cs
Modified: vs-plugin/trunk/Nemerle.Compiler.Utils/Nemerle.Completion2/CodeModel/Project.n
==============================================================================
--- vs-plugin/trunk/Nemerle.Compiler.Utils/Nemerle.Completion2/CodeModel/Project.n (original)
+++ vs-plugin/trunk/Nemerle.Compiler.Utils/Nemerle.Completion2/CodeModel/Project.n Mon Oct 16 04:33:15 2006
@@ -168,11 +168,11 @@
)
: void
{
- foreach (cm in _engine.CompilerMessages)
- addError(cm);
-
def fileIndex = _compileUnits.GetFileIndex(fileName);
+ foreach (cm when cm.Location.FileIndex == fileIndex in _engine.CompilerMessages)
+ addError(cm);
+
def isNext(line, col, ch)
{
def str = source.GetLine(line);
Modified: vs-plugin/trunk/Nemerle.Compiler.Utils/Nemerle.Completion2/CodeModel/ScanLexer.n
==============================================================================
--- vs-plugin/trunk/Nemerle.Compiler.Utils/Nemerle.Completion2/CodeModel/ScanLexer.n (original)
+++ vs-plugin/trunk/Nemerle.Compiler.Utils/Nemerle.Completion2/CodeModel/ScanLexer.n Mon Oct 16 04:33:15 2006
@@ -32,8 +32,17 @@
_onPreprocessor = false;
_tokenInfo.Token = null;
+ _bracketCount = -1;
+
+ _tokenInfo.IsEndOfLine = eol();
}
+ mutable _last_line : int;
+ mutable _last_col : int;
+ mutable _onPreprocessor : bool;
+ mutable _keepDollar : bool;
+ mutable _bracketCount : int;
+
_tokenInfo : ScanTokenInfo = ScanTokenInfo();
static _types : Dictionary[string,string] = Dictionary();
@@ -48,33 +57,62 @@
}
}
- private GetWhiteSpaceToken() : Token
+ skip() : void
{
- def wsloop()
+ ignore(read());
+ }
+
+ eol() : bool
{
- when (pos < reader.Length)
+ pos >= reader.Length
+ }
+
+ step_back() : void
{
- match (peek())
+ pos--;
+ col--;
+ }
+
+ back_to_start() : void
{
- | ' ' | '\t' | '\r' | '\n' => _ = read(); wsloop();
- | _ => ()
+ col = _last_col;
+ pos = col - 1;
}
+
+ peek_next() : char
+ {
+ if (pos+1 < reader.Length) reader[pos+1] else '\0'
}
+
+ skip_to_end() : void
+ {
+ pos = reader.Length;
+ col = pos + 1;
}
- wsloop();
+ skip_whitespace() : void
+ {
+ def loop()
+ {
+ match (peek())
+ {
+ | ' ' | '\t' | '\r' | '\n' => skip(); loop();
+ | '\0' | _ => ()
+ }
+ }
- Token.WhiteSpace(reader.Substring(_last_col - 1, col - _last_col));
+ loop();
}
- mutable _last_line : int;
- mutable _last_col : int;
- mutable _onPreprocessor : bool;
+ CurrentValue : string
+ {
+ get { reader.Substring(_last_col - 1, col - _last_col) }
+ }
- private SkipToEnd() : void
+ private GetWhiteSpaceToken() : Token
{
- pos = reader.Length;
- col = pos + 1;
+ skip_whitespace();
+ Token.WhiteSpace(CurrentValue);
}
private FindEndOfComment() : Token.Comment
@@ -83,36 +121,164 @@
if (idx >= 0)
{
- _tokenInfo.ScanState &= ~ScanState.Comment;
+ _tokenInfo.State &= ~ScanState.Comment;
pos = idx + 2;
col = pos + 1;
}
else
{
- _tokenInfo.ScanState |= ScanState.Comment;
- SkipToEnd();
+ _tokenInfo.State |= ScanState.Comment;
+ skip_to_end();
}
- Token.Comment(reader.Substring(_last_col - 1, col - _last_col));
+ Token.Comment(CurrentValue);
}
private GetCommentToken() : Token
{
- _ = read();
+ skip();
match (peek())
{
| '/' =>
- SkipToEnd();
- Token.Comment(reader.Substring(_last_col - 1));
+ skip_to_end();
+ Token.Comment(CurrentValue);
| '*' =>
- _ = read();
+ skip();
FindEndOfComment()
- | _ => pos--; col--; null
+ | _ => step_back(); null
+ }
+ }
+
+ private GetStringToken() : Token * ScanTokenType * ScanTokenColor * ScanTokenTriggers
+ {
+ when (_tokenInfo.IsDollar && _bracketCount < 0)
+ _bracketCount = _tokenInfo.BracketCount;
+
+ def getDollarContent()
+ {
+ def loop()
+ {
+ def ch = read();
+
+ match (ch)
+ {
+ | '(' =>
+
+ _bracketCount++;
+ _tokenInfo.BracketCount = _bracketCount;
+ loop();
+
+ | ')' =>
+
+ _bracketCount--;
+ _tokenInfo.BracketCount = _bracketCount;
+
+ unless (_bracketCount == 0)
+ loop();
+
+ | '\\' when !_tokenInfo.IsMultiLineString && peek() == '"'
+ | '"' when _tokenInfo.IsMultiLineString && peek() == '"' =>
+
+ skip();
+ loop();
+
+ | '"' =>
+
+ _Debug("");
+ _tokenInfo.State &= ~(ScanState.String | ScanState.MultiLineString);
+
+ | '\0' => ()
+ | _ => loop()
+ }
+ }
+
+ loop();
+ (Token.StringLiteral(CurrentValue), TP.String, C.StringEx, TR.None)
+ }
+
+ if (_bracketCount > 0)
+ {
+ getDollarContent()
+ }
+ else match (peek())
+ {
+ | '"' when _tokenInfo.IsMultiLineString && peek_next() == '"' =>
+
+ repeat (2) skip();
+ (Token.StringLiteral(CurrentValue), TP.String, C.StringEx, TR.None)
+
+ | '"' =>
+
+ _Debug('"');
+ skip();
+ _tokenInfo.State &= ~(ScanState.String | ScanState.MultiLineString);
+ (Token.StringLiteral(CurrentValue), TP.String, C.StringEx, TR.None)
+
+ | '\\' when !_tokenInfo.IsMultiLineString =>
+
+ skip();
+
+ def opt = Manager.Options.ThrowOnError;
+
+ Manager.Options.ThrowOnError = true;
+
+ def tok = try
+ {
+ _ = escape_value(read());
+ (Token.StringLiteral(CurrentValue), TP.String, C.StringEx, TR.None)
+ }
+ catch
+ {
+ _ => (Token.StringLiteral(CurrentValue), TP.String, C.String, TR.None)
+ }
+
+ Manager.Options.ThrowOnError = opt;
+
+ tok;
+
+ | '$' when _tokenInfo.IsDollar && peek_next() == '(' =>
+
+ skip();
+ getDollarContent();
+
+ | '$' when _tokenInfo.IsDollar && IsIdBeginning(peek_next()) =>
+
+ repeat (2) skip();
+
+ def loop()
+ {
+ def ch = peek();
+
+ when (IsIdBeginning(ch) || char.IsDigit(ch))
+ {
+ skip();
+ loop();
+ }
+ }
+
+ loop();
+ (Token.StringLiteral(CurrentValue), TP.String, C.StringEx, TR.None)
+
+ | _ =>
+
+ def loop()
+ {
+ skip();
+ match (peek())
+ {
+ | '"' | '\\' | '$' when _tokenInfo.IsDollar | '\0' => ()
+ | _ => loop()
+ }
+ }
+
+ loop();
+ (Token.StringLiteral(CurrentValue), TP.String, C.String, TR.None)
+
}
}
@@ -123,12 +289,12 @@
def readWord() : string
{
while (Char.IsWhiteSpace(peek()))
- _ = read();
+ skip();
last_col = col;
for (mutable c = peek(); IsIdBeginning(c) || Char.IsDigit(c); c = peek())
- _ = read();
+ skip();
reader.Substring(last_col - 1, col - last_col);
}
@@ -170,11 +336,18 @@
col = last_col;
pos = col - 1;
- Token.Keyword(reader.Substring(_last_col - 1, col - _last_col));
+ Token.Keyword(CurrentValue);
+ }
+
+ _Debug[T](o : T) : void
+ {
+ _ = o.ToString();
}
private GetBaseToken() : Token * ScanTokenType * ScanTokenColor * ScanTokenTriggers
{
+ _keepDollar = false;
+
def tok = base.GetToken();
def (tp, color, trigger) = match (tok)
@@ -187,7 +360,7 @@
| Operator(n) when n == "." => (TP.Delimiter, C.Operator, TR.MemberSelect)
| Semicolon
| Operator => (TP.Operator, C.Operator, TR.None)
- | StringLiteral => (TP.String, C.String, TR.None)
+ //| StringLiteral => (TP.String, C.String, TR.None)
| CharLiteral => (TP.Literal, C.String, TR.None)
| IntegerLiteral
| FloatLiteral
@@ -202,8 +375,21 @@
| EndBrace /* } */
| BeginSquare /* [ */
| EndSquare /* ] */ => (TP.Operator, C.Operator, TR.MatchBraces)
- | BeginQuote /* <[ */
- | EndQuote /* ]> */
+ | BeginQuote /* <[ */ =>
+
+ _tokenInfo.State |= ScanState.Quotation;
+ (TP.Operator, C.Quotation, TR.None)
+
+ | EndQuote /* ]> */ =>
+
+ _tokenInfo.State &= ~ScanState.Quotation;
+ (TP.Operator, C.Quotation, TR.None)
+
+ //| RoundGroup(LooseGroup(Operator(name))) when name == "$" =>
+
+ // _Debug(tok);
+ // (TP.String, C.String, TR.None)
+
| _ => (TP.Unknown, C.Text, TR.None)
}
@@ -212,28 +398,58 @@
public GetToken(prevState : ScanState) : ScanTokenInfo
{
- _tokenInfo.ScanState = prevState;
+ _tokenInfo.State = prevState;
_last_line = line;
_last_col = col;
+ _keepDollar = true;
(_tokenInfo.Token, _tokenInfo.Type, _tokenInfo.Color, _tokenInfo.Triggers)
=
- if (pos >= reader.Length)
+ if (eol())
{
+ when (_tokenInfo.IsString && !_tokenInfo.IsMultiLineString)
+ _tokenInfo.State &= ~ScanState.String;
+
+ _tokenInfo.IsEndOfLine = eol();
+
(Token.EndOfFile(), TP.Unknown, C.Text, TR.None)
}
else if (_onPreprocessor)
{
- SkipToEnd();
+ skip_to_end();
(Token.Identifier(""), TP.Unknown, C.Text, TR.None)
}
- else if ((prevState & ScanState.Comment) != 0)
+ else if (_tokenInfo.IsComment)
{
(FindEndOfComment(), TP.Comment, C.Comment, TR.None)
}
+ else if (_tokenInfo.IsString)
+ {
+ GetStringToken()
+ }
else
{
+ def getString()
+ {
+ match (peek())
+ {
+ | '"' =>
+
+ _tokenInfo.State |= ScanState.String;
+ skip();
+
+ | '@' =>
+
+ _tokenInfo.State |= (ScanState.String | ScanState.MultiLineString);
+ repeat (2) skip();
+
+ | _ => throw InvalidOperationException()
+ }
+
+ (Token.StringLiteral(CurrentValue), TP.String, C.StringEx, TR.None)
+ }
+
match (peek())
{
| ' ' | '\t' | '\n' | '\r' =>
@@ -260,7 +476,7 @@
_onPreprocessor = true;
white_beginning = false;
- _ = read();
+ skip();
if (pos >= reader.Length)
(Token.Keyword("#"), TP.Keyword, C.Keyword, TR.MemberSelect)
@@ -276,10 +492,58 @@
else
GetBaseToken();
+ | '"' | '@' when peek_next() == '"' =>
+
+ getString();
+
+ | '$' =>
+
+ skip();
+ skip_whitespace();
+
+ match (peek())
+ {
+ | '\0' =>
+
+ _tokenInfo.State |= ScanState.Dollar;
+ (Token.Operator(CurrentValue), TP.Operator, C.Operator, TR.None)
+
+ | '"' | '@' when peek_next() == '"' =>
+
+ _tokenInfo.State &= ~ScanState.BracketCounter;
+ _tokenInfo.State |= ScanState.Dollar;
+ getString();
+
+ | _ => back_to_start(); GetBaseToken();
+ }
+
| _ => GetBaseToken();
}
}
+ when (!_keepDollar && _tokenInfo.IsDollar)
+ {
+ _tokenInfo.State &= ~ScanState.Dollar;
+ _tokenInfo.BracketCount = 0;
+ _bracketCount = 0;
+ }
+
+ when (_tokenInfo.IsQuotation && !(_tokenInfo.Token is Token.BeginQuote))
+ {
+ _tokenInfo.Color = match (_tokenInfo.Color)
+ {
+ | Text => ScanTokenColor.QuotationText
+ | Keyword => ScanTokenColor.QuotationKeyword
+ | Comment => ScanTokenColor.QuotationComment
+ | Identifier => ScanTokenColor.QuotationIdentifier
+ | String => ScanTokenColor.QuotationString
+ | Number => ScanTokenColor.QuotationNumber
+ | Operator => ScanTokenColor.QuotationOperator
+ | StringEx => ScanTokenColor.QuotationStringEx
+ | _ => _tokenInfo.Color
+ }
+ }
+
_tokenInfo.Token.Location = Location(file_idx, _last_line, _last_col, line, col);
_tokenInfo
}
Modified: vs-plugin/trunk/Nemerle.Compiler.Utils/Nemerle.Completion2/CodeModel/ScanState.n
==============================================================================
--- vs-plugin/trunk/Nemerle.Compiler.Utils/Nemerle.Completion2/CodeModel/ScanState.n (original)
+++ vs-plugin/trunk/Nemerle.Compiler.Utils/Nemerle.Completion2/CodeModel/ScanState.n Mon Oct 16 04:33:15 2006
@@ -5,7 +5,12 @@
[Flags]
public enum ScanState
{
- | None = 0x0000
- | Comment = 0x0001
+ | None = 0x00000000
+ | Comment = 0x00000001
+ | Quotation = 0x00000002
+ | Dollar = 0x00000004
+ | String = 0x00000010
+ | MultiLineString = 0x00000020
+ | BracketCounter = 0x0F000000
}
}
Modified: vs-plugin/trunk/Nemerle.Compiler.Utils/Nemerle.Completion2/CodeModel/ScanTokenColor.n
==============================================================================
--- vs-plugin/trunk/Nemerle.Compiler.Utils/Nemerle.Completion2/CodeModel/ScanTokenColor.n (original)
+++ vs-plugin/trunk/Nemerle.Compiler.Utils/Nemerle.Completion2/CodeModel/ScanTokenColor.n Mon Oct 16 04:33:15 2006
@@ -13,5 +13,16 @@
| Number = 5
| Operator = 7
+ | StringEx
+
+ | Quotation
+ | QuotationText
+ | QuotationKeyword
+ | QuotationComment
+ | QuotationIdentifier
+ | QuotationString
+ | QuotationNumber
+ | QuotationOperator
+ | QuotationStringEx
}
}
Modified: vs-plugin/trunk/Nemerle.Compiler.Utils/Nemerle.Completion2/CodeModel/ScanTokenInfo.n
==============================================================================
--- vs-plugin/trunk/Nemerle.Compiler.Utils/Nemerle.Completion2/CodeModel/ScanTokenInfo.n (original)
+++ vs-plugin/trunk/Nemerle.Compiler.Utils/Nemerle.Completion2/CodeModel/ScanTokenInfo.n Mon Oct 16 04:33:15 2006
@@ -8,14 +8,28 @@
public class ScanTokenInfo
{
public mutable Token : Token;
- public mutable ScanState : ScanState;
+ public mutable State : ScanState;
public mutable Color : ScanTokenColor;
public mutable Triggers : ScanTokenTriggers;
public mutable Type : ScanTokenType;
+ public mutable IsEndOfLine : bool;
- public IsEndOfLine : bool
+ public BracketCount : int
{
- get { match (Token) { | EndOfFile => true | _ => false } }
+ get { (State & ScanState.BracketCounter) :> int >> 24 }
+ set
+ {
+ def val = if (value > 0xF) 0xF else value;
+
+ State &= ~ScanState.BracketCounter;
+ State |= (val << 24) :> ScanState;
}
}
+
+ public IsQuotation : bool { get { State %&& ScanState.Quotation } }
+ public IsComment : bool { get { State %&& ScanState.Comment } }
+ public IsString : bool { get { State %&& ScanState.String } }
+ public IsMultiLineString : bool { get { State %&& ScanState.MultiLineString } }
+ public IsDollar : bool { get { State %&& ScanState.Dollar } }
+ }
}
Modified: vs-plugin/trunk/Nemerle.Compiler.Utils/Nemerle.Completion2/CompiledUnitAstBrowser.n
==============================================================================
--- vs-plugin/trunk/Nemerle.Compiler.Utils/Nemerle.Completion2/CompiledUnitAstBrowser.n (original)
+++ vs-plugin/trunk/Nemerle.Compiler.Utils/Nemerle.Completion2/CompiledUnitAstBrowser.n Mon Oct 16 04:33:15 2006
@@ -7,7 +7,7 @@
namespace Nemerle.Completion2
{
- public delegate ShowLocation(_ : Location) : void;
+ public delegate ShowLocation(loc : Location) : void;
variant PropInfo
{
Modified: vs-plugin/trunk/Nemerle.VsIntegration/LanguageService/NemerleLanguageService.cs
==============================================================================
--- vs-plugin/trunk/Nemerle.VsIntegration/LanguageService/NemerleLanguageService.cs (original)
+++ vs-plugin/trunk/Nemerle.VsIntegration/LanguageService/NemerleLanguageService.cs Mon Oct 16 04:33:15 2006
@@ -84,10 +84,21 @@
new NemerleColorableItem("Keyword", COLORINDEX.CI_BLUE, COLORINDEX.CI_USERTEXT_BK),
new NemerleColorableItem("Comment", COLORINDEX.CI_DARKGREEN, COLORINDEX.CI_USERTEXT_BK),
new NemerleColorableItem("Identifier", COLORINDEX.CI_SYSPLAINTEXT_FG, COLORINDEX.CI_USERTEXT_BK),
- new NemerleColorableItem("String", COLORINDEX.CI_DARKBLUE, COLORINDEX.CI_USERTEXT_BK),
+ new NemerleColorableItem("String", COLORINDEX.CI_AQUAMARINE, COLORINDEX.CI_USERTEXT_BK),
new NemerleColorableItem("Number", COLORINDEX.CI_DARKBLUE, COLORINDEX.CI_USERTEXT_BK),
new NemerleColorableItem("Text", COLORINDEX.CI_SYSPLAINTEXT_FG, COLORINDEX.CI_USERTEXT_BK),
- new NemerleColorableItem("Operator", COLORINDEX.CI_DARKBLUE, COLORINDEX.CI_USERTEXT_BK)
+ new NemerleColorableItem("Operator", COLORINDEX.CI_DARKBLUE, COLORINDEX.CI_USERTEXT_BK),
+ new NemerleColorableItem("StringEx", COLORINDEX.CI_CYAN, COLORINDEX.CI_USERTEXT_BK),
+
+ new NemerleColorableItem("Quotation", COLORINDEX.CI_BROWN, COLORINDEX.CI_USERTEXT_BK),
+ new NemerleColorableItem("Quotation - Keyword", COLORINDEX.CI_BLUE, COLORINDEX.CI_USERTEXT_BK),
+ new NemerleColorableItem("Quotation - Comment", COLORINDEX.CI_DARKGREEN, COLORINDEX.CI_USERTEXT_BK),
+ new NemerleColorableItem("Quotation - Identifier", COLORINDEX.CI_SYSPLAINTEXT_FG, COLORINDEX.CI_USERTEXT_BK),
+ new NemerleColorableItem("Quotation - String", COLORINDEX.CI_AQUAMARINE, COLORINDEX.CI_USERTEXT_BK),
+ new NemerleColorableItem("Quotation - Number", COLORINDEX.CI_DARKBLUE, COLORINDEX.CI_USERTEXT_BK),
+ new NemerleColorableItem("Quotation - Text", COLORINDEX.CI_SYSPLAINTEXT_FG, COLORINDEX.CI_USERTEXT_BK),
+ new NemerleColorableItem("Quotation - Operator", COLORINDEX.CI_DARKBLUE, COLORINDEX.CI_USERTEXT_BK),
+ new NemerleColorableItem("Quotation - StringEx", COLORINDEX.CI_DARKGREEN, COLORINDEX.CI_USERTEXT_BK),
};
public override void Dispose()
Modified: vs-plugin/trunk/Nemerle.VsIntegration/LanguageService/NemerleScanner.cs
==============================================================================
--- vs-plugin/trunk/Nemerle.VsIntegration/LanguageService/NemerleScanner.cs (original)
+++ vs-plugin/trunk/Nemerle.VsIntegration/LanguageService/NemerleScanner.cs Mon Oct 16 04:33:15 2006
@@ -41,19 +41,18 @@
return _lexer;
}
+ //System.Diagnostics.Debug.WriteLine(source);
// Its calling depends on IVsColorizer.GetStateMaintenanceFlag, which defaults to true.
//
public bool ScanTokenAndProvideInfoAboutIt(TokenInfo tokenInfo, ref int state)
{
- ScanLexer lexer = GetLexer();
-
- if (lexer == null)
+ if (_lexer == null)
return false;
- ScanTokenInfo info = lexer.GetToken((ScanState)state);
+ ScanTokenInfo info = _lexer.GetToken((ScanState)state);
- state = (int)info.ScanState;
+ state = (int)info.State;
tokenInfo.Type = (TokenType) info.Type;
tokenInfo.Color = (TokenColor) info.Color;
More information about the svn
mailing list