[svn] r5852: nemerle/trunk/ncc: parsing/PreParser.n
testsuite/positive/indentation-syntax.n
malekith
svnadmin at nemerle.org
Wed Oct 26 23:29:58 CEST 2005
Log:
Make the indentation syntax work, when there is comment at the end of the file. Simplify it a bit.
Author: malekith
Date: Wed Oct 26 23:29:57 2005
New Revision: 5852
Modified:
nemerle/trunk/ncc/parsing/PreParser.n
nemerle/trunk/ncc/testsuite/positive/indentation-syntax.n
Modified: nemerle/trunk/ncc/parsing/PreParser.n
==============================================================================
--- nemerle/trunk/ncc/parsing/PreParser.n (original)
+++ nemerle/trunk/ncc/parsing/PreParser.n Wed Oct 26 23:29:57 2005
@@ -452,93 +452,54 @@
public class PreParserIndent : PreParser {
// For indentation syntax
- mutable insert_unindents : int;
mutable indent_level : int;
- mutable new_indent : string;
- mutable current_indent : string;
+ mutable current_indent : string = "";
// The number of unmatched { ( [ tokens found in the user code at this point
mutable explicit_groups : int;
- static mutable indentation_syntax_active : bool;
+ mutable indentation_syntax_active : bool = true;
mutable insertLocation : Location;
- mutable token_pending : Token;
- mutable last_real_tok : Token;
- mutable indent_strings : System.Collections.Generic.List [string];
+
+ mutable had_some_real_input : bool;
+
+ mutable force_brace_after_newline : bool;
+ indent_strings : System.Collections.Generic.List [string];
+ tokens_pending : Queue [Token] = Queue ();
+
+ // 'set' directives handling
mutable set_namespace : bool;
mutable set_class : bool;
- mutable force_brace_after_newline : bool;
public this (lex : LexerBase) {
base (lex);
- indentation_syntax_active = true;
- new_indent = "";
- current_indent = "";
indent_strings = System.Collections.Generic.List (20);
- _ = indent_strings.Add("");
+ _ = indent_strings.Add ("");
}
- /** Fetch next token (from one token buffer or lexer if it's empty */
- protected override get_token () : Token {
- mutable lexer_tok = null;
- mutable source = 0;
-
- mutable tok =
- if (insert_unindents > 0) {
- insert_unindents--;
- indent_level--;
- //Message.Debug (insertLocation, "Generate '}'");
- Token.EndBrace (true);
- }
- // last_token has priority over token_pending
- else if (last_token != null) {
- source = 2;
- def result = last_token;
- last_token = null;
- result;
- }
- else if (token_pending != null) {
- source = 1;
- def result = token_pending;
- token_pending = null;
- result;
+ push_end_brace () : void
+ {
+ tokens_pending.Push (Token.EndBrace (insertLocation, true));
}
- else {
- source = 3;
- try {
- lexer_tok = lexer.GetToken ();
+
+ handle_real_token (lexer_tok : Token) : void
+ {
+ unless (lexer_tok is Token.EndOfFile)
+ had_some_real_input = true;
+
match (lexer_tok) {
- | Token.Indent as t =>
- if (indentation_syntax_active) {
- if (force_brace_after_newline) {
- force_brace_after_newline = false;
- Token.BeginBrace (true);
- }
- else {
- new_indent = t.value;
+ | Token.EndOfFile =>
insertLocation = lexer_tok.Location;
- //Message.Debug (insertLocation, "Calling get_token_after_indent ()");
- source = 4;
- get_token_after_indent ();
- }
- }
- else
- get_token();
- | Token.EndOfFile =>
- insert_unindents = indent_level;
+ while (indent_level > 0) {
+ push_end_brace ();
+ indent_level--;
+ }
when (set_class)
- insert_unindents++;
+ push_end_brace ();
when (set_namespace)
- insert_unindents++;
- if (insert_unindents > 0) {
- insertLocation = lexer_tok.Location;
- token_pending = lexer_tok;
- Token.EndBrace (true);
- }
- else {
- last_real_tok = lexer_tok;
- lexer_tok
- }
+ push_end_brace ();
+
+ tokens_pending.Push (lexer_tok);
| BeginBrace
| BeginRound
@@ -546,8 +507,7 @@
| BeginQuote =>
explicit_groups++;
indentation_syntax_active = false;
- last_real_tok = lexer_tok;
- lexer_tok
+ tokens_pending.Push (lexer_tok)
| EndBrace
| EndRound
@@ -559,141 +519,159 @@
explicit_groups--;
when (explicit_groups == 0)
indentation_syntax_active = true;
- last_real_tok = lexer_tok;
- lexer_tok
+ tokens_pending.Push (lexer_tok)
- | _ =>
- last_real_tok = lexer_tok;
- lexer_tok
- }
- }
- catch {
- | e is LexerBase.Error =>
- Message.Error (lexer.Location, e.name);
- get_token ()
- }
- }
-
- //Message.Debug (tok.ToString() + ", " + tok.GetType().ToString() + ", " + new_indent.Length.ToString());
// If this is the 'set' keyword in the first column of a new line:
- //when (new_indent.Length == 0 && tok is Token.Identifier && (tok :> Token.Identifier).name == "set") {
- when (current_indent.Length == 0 && tok.ToString() == "set") {
- def tokNext = lexer.GetToken ();
- match (tokNext) {
+ | Identifier ("set") when current_indent == ""
+ | Keyword ("set") when current_indent == "" =>
+ def next = lexer.GetToken ();
+ match (next) {
| Token.Keyword ("namespace") =>
when (set_namespace)
- throw PreParserException (tokNext.Location, "the 'set namespace' directive can only be used once per file");
+ throw PreParserException (next.Location,
+ "the 'set namespace' directive can only be used once per file");
set_namespace = true;
- //tokNext = get_token_after_indent (tokNext);
force_brace_after_newline = true;
| Token.Keyword ("class") =>
when (set_class)
- throw PreParserException (tokNext.Location, "the 'set class' directive can only be used once per file");
+ throw PreParserException (next.Location,
+ "the 'set class' directive can only be used once per file");
set_class = true;
- //tokNext = get_token_after_indent (tokNext);
force_brace_after_newline = true;
| _ =>
- throw PreParserException (tokNext.Location, $"unrecognized 'set' directive: '$tokNext'");
- }
- tok = tokNext;
+ throw PreParserException (next.Location, $"unrecognized 'set' directive: '$next'");
}
- //Message.Debug (tok.Location, tok.ToString() + $" ($source)");
- tok;
- }
+ tokens_pending.Push (next);
- get_token_after_indent () : Token {
- //Message.Debug ("1");
- mutable tok = lexer.GetToken ();
+ | _ =>
+ tokens_pending.Push (lexer_tok)
+ }
+ }
- //Message.Debug ("2");
- while (tok is Token.Indent || tok is Token.Comment) {
- // TODO: what's the best way to do this?
- match (tok) {
- | Token.Indent (value) => new_indent = value;
- | _ => ()
+ /** Fetch next token (from one token buffer or lexer if it's empty */
+ protected override get_token () : Token
+ {
+ // last_token has priority over tokens_pending
+ if (last_token != null) {
+ def result = last_token;
+ last_token = null;
+ result;
+ } else if (! tokens_pending.IsEmpty) {
+ tokens_pending.Take ()
}
- tok = lexer.GetToken ();
+ else {
+ try {
+ match (lexer.GetToken ()) {
+ | Token.Indent as t =>
+ if (indentation_syntax_active) {
+ /*
+ if (force_brace_after_newline) {
+ force_brace_after_newline = false;
+ Token.BeginBrace (true);
+ }
+ else */ {
+ //Message.Debug (insertLocation, "Calling get_token_after_indent ()");
+ get_token_after_indent (t)
}
+ }
+ else
+ get_token ()
- when (!force_brace_after_newline)
- tok = get_token_after_indent (tok);
+ | tok =>
+ handle_real_token (tok);
+ tokens_pending.Take ()
+ }
+ }
+ catch {
+ | e is LexerBase.Error =>
+ Message.Error (lexer.Location, e.name);
+ get_token ()
+ }
+ }
+ }
- tok;
+ #if false
+ display_space (s : string) : string
+ {
+ s.Replace (" ", "S").Replace("\t", "T")
}
+ #endif
+
+ get_token_after_indent (tok : Token, new_indent : string = "") : Token
+ {
+ match (tok) {
+ | Token.Indent (value) =>
+ insertLocation = tok.Location;
+ get_token_after_indent (lexer.GetToken (), value)
+
+ | Token.Comment =>
+ get_token_after_indent (lexer.GetToken (), new_indent)
+
+ | _ when force_brace_after_newline =>
+ force_brace_after_newline = false;
+ tokens_pending.Push (Token.BeginBrace (insertLocation, true));
+ get_token_after_indent (tok, new_indent)
+
+ | _ =>
+ def previous_indent = current_indent;
+ current_indent = new_indent;
- get_token_after_indent (tok : Token) : Token {
// If we have not unindented
- if (new_indent.Length >= current_indent.Length) {
+ if (new_indent.Length >= previous_indent.Length) {
// Make sure that the beginning of the new indent string
// starts with the contents of the current indent string.
- when (!new_indent.StartsWith(current_indent)) {
+ when (!new_indent.StartsWith (previous_indent)) {
// TODO: make a better error message -- checkout python error message
Message.Error (tok.Location, "inconsistent indentation");
- current_indent = new_indent;
}
// If we have remained at the same indentation level
- if (new_indent == current_indent) {
- if (last_real_tok != null) {
- token_pending = tok;
+ if (new_indent == previous_indent) {
+ when (had_some_real_input)
//Message.Debug (tok.Location, $"Generate ';'");
- mutable tokInsert =
- /*if (force_brace_after_newline) {
- force_brace_after_newline = false;
- Token.BeginBrace (true);
- }
- else*/
- Token.Semicolon (true);
- tokInsert.Location = insertLocation;
- tokInsert;
- }
- else
- tok;
+ tokens_pending.Push (Token.Semicolon (insertLocation, true));
+
+ handle_real_token (tok);
}
// If we've indented further than the previous line
else {
match (tok) {
| Token.Operator =>
- tok;
+ current_indent = previous_indent;
+ handle_real_token (tok)
| _ =>
indent_level++;
- _ = indent_strings.Add(new_indent);
- current_indent = new_indent;
- token_pending = tok;
+ _ = indent_strings.Add (new_indent);
+ tokens_pending.Push (Token.BeginBrace (insertLocation, true));
+ handle_real_token (tok);
//Message.Debug (tok.Location, "Generate '{'");
- mutable tokInsert = Token.BeginBrace (true);
- tokInsert.Location = insertLocation;
- tokInsert;
}
}
}
// Otherwise, we've unintented:
else {
- def i = indent_strings.IndexOf(new_indent);
+ def i = indent_strings.IndexOf (new_indent);
when (i == -1) {
// TODO: make a better error message -- checkout python error message
throw PreParserException (tok.Location, "inconsistent indentation");
}
- insert_unindents = indent_level - i;
+ repeat (indent_level - i)
+ push_end_brace ();
+
indent_level = i;
- indent_strings.RemoveRange(i + 1, indent_strings.Count - (i + 1));
- //while (indent_strings.Length > i + 1)
- // indent_strings.Pop ();
+ indent_strings.RemoveRange (i + 1, indent_strings.Count - (i + 1));
- current_indent = new_indent;
- token_pending = tok;
+ handle_real_token (tok);
//Message.Debug (tok.Location, $"Generate $insert_unindents '}'");
- insert_unindents--;
- mutable tokInsert = Token.EndBrace (true);
- tokInsert.Location = insertLocation;
- tokInsert;
- }
}
+ get_token ()
+ }
+ }
}
}
Modified: nemerle/trunk/ncc/testsuite/positive/indentation-syntax.n
==============================================================================
--- nemerle/trunk/ncc/testsuite/positive/indentation-syntax.n (original)
+++ nemerle/trunk/ncc/testsuite/positive/indentation-syntax.n Wed Oct 26 23:29:57 2005
@@ -3,20 +3,11 @@
// 4. Add a condition with a single statement beneath it
// 5. Add nested conditions to test whether multi-unindenting works
// 6. Add 'set' keyword for namespace and class
-// 7. Add two kinds of line-continuation + missing newline at end of file
+// 7. Add two kinds of line-continuation
using System.Console
-/*
-BEGIN-OUTPUT
-Hello, World!
-n > 0
-c = 3, d = 4
-END-OUTPUT
-*/
-
set namespace Test
-
set class App
static Main() : void
@@ -47,3 +38,11 @@
c
WriteLine($"c = $c, d = $d")
+/*
+BEGIN-OUTPUT
+Hello, World!
+n > 0
+c = 3, d = 4
+END-OUTPUT
+*/
+
More information about the svn
mailing list