[svn] r5856: nemerle/trunk/ncc: Makefile parsing/PreParser.n
parsing/PreParserIndent.n
malekith
svnadmin at nemerle.org
Thu Oct 27 09:55:50 CEST 2005
Log:
Move PreParserIndent to a separate file.
Author: malekith
Date: Thu Oct 27 09:55:49 2005
New Revision: 5856
Added:
nemerle/trunk/ncc/parsing/PreParserIndent.n
- copied, changed from rev 5855, nemerle/trunk/ncc/parsing/PreParser.n
Modified:
nemerle/trunk/ncc/Makefile
nemerle/trunk/ncc/parsing/PreParser.n
Modified: nemerle/trunk/ncc/Makefile
==============================================================================
--- nemerle/trunk/ncc/Makefile (original)
+++ nemerle/trunk/ncc/Makefile Thu Oct 27 09:55:49 2005
@@ -106,6 +106,7 @@
parsing/MainParser.n \
parsing/ParseTree.n \
parsing/PreParser.n \
+ parsing/PreParserIndent.n \
parsing/Utility.n \
passes.n \
typing/ConstantFolder.n \
Modified: nemerle/trunk/ncc/parsing/PreParser.n
==============================================================================
--- nemerle/trunk/ncc/parsing/PreParser.n (original)
+++ nemerle/trunk/ncc/parsing/PreParser.n Thu Oct 27 09:55:49 2005
@@ -449,222 +449,4 @@
top
}
}
-
- public class PreParserIndent : PreParser {
- // For indentation syntax
- mutable current_indent : string = "";
- // The number of unmatched { ( [ tokens found in the user code at this point
- mutable explicit_groups : int;
- mutable indentation_syntax_active : bool = true;
- mutable insertLocation : Location;
-
- mutable had_some_real_input : bool;
-
- mutable force_brace_after_newline : bool;
- indent_strings : Stack [string] = Stack (20);
- tokens_pending : Queue [Token] = Queue ();
-
- // 'set' directives handling
- mutable set_namespace : bool;
- mutable set_class : bool;
-
- public this (lex : LexerBase) {
- base (lex);
-
- indent_strings.Push ("");
- }
-
- push_end_brace () : void
- {
- tokens_pending.Push (Token.EndBrace (insertLocation, true));
- }
-
- handle_real_token (lexer_tok : Token) : void
- {
- unless (lexer_tok is Token.EndOfFile)
- had_some_real_input = true;
-
- match (lexer_tok) {
- | Token.EndOfFile =>
- insertLocation = lexer_tok.Location;
-
- while (indent_strings.Count > 1) {
- _ = indent_strings.Pop ();
- push_end_brace ();
- }
- when (set_class)
- push_end_brace ();
- when (set_namespace)
- push_end_brace ();
-
- tokens_pending.Push (lexer_tok);
-
- | BeginBrace
- | BeginRound
- | BeginSquare
- | BeginQuote =>
- explicit_groups++;
- indentation_syntax_active = false;
- tokens_pending.Push (lexer_tok)
-
- | EndBrace
- | EndRound
- | EndSquare
- | EndQuote =>
- when (explicit_groups == 0)
- // TODO: better error message
- throw PreParserException (lexer_tok.Location, "unmatched group end");
- explicit_groups--;
- when (explicit_groups == 0)
- indentation_syntax_active = true;
- tokens_pending.Push (lexer_tok)
-
- // If this is the 'set' keyword in the first column of a new line:
- | Identifier ("set") when current_indent == ""
- | Keyword ("set") when current_indent == "" =>
- def next = lexer.GetToken ();
- match (next) {
- | Token.Keyword ("namespace") =>
- when (set_namespace)
- throw PreParserException (next.Location,
- "the 'set namespace' directive can only be used once per file");
- set_namespace = true;
- force_brace_after_newline = true;
-
- | Token.Keyword ("class") =>
- when (set_class)
- throw PreParserException (next.Location,
- "the 'set class' directive can only be used once per file");
- set_class = true;
- force_brace_after_newline = true;
-
- | _ =>
- throw PreParserException (next.Location, $"unrecognized 'set' directive: '$next'");
- }
-
- tokens_pending.Push (next);
-
- | _ =>
- tokens_pending.Push (lexer_tok)
- }
- }
-
- /** Fetch next token (from one token buffer or lexer if it's empty */
- protected override get_token () : Token
- {
- // last_token has priority over tokens_pending
- if (last_token != null) {
- def result = last_token;
- last_token = null;
- result;
- } else if (! tokens_pending.IsEmpty) {
- tokens_pending.Take ()
- }
- else {
- try {
- match (lexer.GetToken ()) {
- | Token.Indent as t =>
- if (indentation_syntax_active) {
- /*
- if (force_brace_after_newline) {
- force_brace_after_newline = false;
- Token.BeginBrace (true);
- }
- else */ {
- //Message.Debug (insertLocation, "Calling get_token_after_indent ()");
- get_token_after_indent (t)
- }
- }
- else
- get_token ()
-
- | tok =>
- handle_real_token (tok);
- tokens_pending.Take ()
- }
- }
- catch {
- | e is LexerBase.Error =>
- Message.Error (lexer.Location, e.name);
- get_token ()
- }
- }
- }
-
- #if false
- display_space (s : string) : string
- {
- s.Replace (" ", "S").Replace("\t", "T")
- }
- #endif
-
- get_token_after_indent (tok : Token, new_indent : string = "") : Token
- {
- match (tok) {
- | Token.Indent (value) =>
- insertLocation = tok.Location;
- get_token_after_indent (lexer.GetToken (), value)
-
- | Token.Comment =>
- get_token_after_indent (lexer.GetToken (), new_indent)
-
- | _ when force_brace_after_newline =>
- force_brace_after_newline = false;
- tokens_pending.Push (Token.BeginBrace (insertLocation, true));
- get_token_after_indent (tok, new_indent)
-
- | _ =>
- def previous_indent = current_indent;
- current_indent = new_indent;
-
- // If we have not unindented
- if (new_indent.Length >= previous_indent.Length) {
- // Make sure that the beginning of the new indent string
- // starts with the contents of the current indent string.
- when (!new_indent.StartsWith (previous_indent)) {
- // TODO: make a better error message -- checkout python error message
- Message.Error (tok.Location, "inconsistent indentation");
- }
-
- // If we have remained at the same indentation level
- if (new_indent == previous_indent) {
- when (had_some_real_input)
- //Message.Debug (tok.Location, $"Generate ';'");
- tokens_pending.Push (Token.Semicolon (insertLocation, true));
-
- handle_real_token (tok);
- }
- // If we've indented further than the previous line
- else {
- match (tok) {
- | Token.Operator =>
- current_indent = previous_indent;
- handle_real_token (tok)
-
- | _ =>
- indent_strings.Push (new_indent);
- tokens_pending.Push (Token.BeginBrace (insertLocation, true));
- handle_real_token (tok);
- //Message.Debug (tok.Location, "Generate '{'");
- }
- }
- }
- // Otherwise, we've unintented:
- else {
- if (indent_strings.Contains (new_indent)) {
- while (indent_strings.Top != new_indent) {
- push_end_brace ();
- _ = indent_strings.Pop ();
- }
- handle_real_token (tok)
- } else {
- // TODO: make a better error message -- checkout python error message
- throw PreParserException (tok.Location, "inconsistent indentation");
- }
- }
-
- get_token ()
- }
- }
- }
}
Copied: nemerle/trunk/ncc/parsing/PreParserIndent.n (from rev 5855, nemerle/trunk/ncc/parsing/PreParser.n)
==============================================================================
--- nemerle/trunk/ncc/parsing/PreParser.n (original)
+++ nemerle/trunk/ncc/parsing/PreParserIndent.n Thu Oct 27 09:55:49 2005
@@ -33,423 +33,6 @@
namespace Nemerle.Compiler
{
- class PreParserException : System.Exception {
- public Location : Location;
-
- public this (loc : Location, msg : string) {
- base (msg);
- this.Location = loc;
- }
- }
-
- /** Transforms stream of tokens from given LexerBase to token tree
- with matched brackets.
- */
- public class PreParser
- {
- protected lexer : LexerBase;
- protected mutable last_token : Token = null;
- mutable Env : GlobalEnv;
-
- mutable finished : bool = false;
-
- /** Parent stream is the stack of processed token nodes,
- which are already assigned to be in currently build sequence.
- For example:
- a; b; c (); d e _we_are_here_
- 'a, b, c()', are alredy known to be in parent sequence,
- while 'd e' are in current temporary sequence, which might
- get added to parent_stream if separator (e.g. ';') occurs
- */
- parent_stream : SCG.List [Token] = SCG.List (100);
-
- /** Currently builded stream of token nodes is an array of
- loose tokens, which have occured after last separator.
- It will probably form LooseGroup as an element of parent
- sequence or all elements will constitue parent
- */
- current_stream : SCG.List [Token] = SCG.List (50);
-
- internal static mutable doc_comments : Map [Location, string];
-
- public static Init () : void {
- if (Options.XmlDoc)
- doc_comments = NemerleMap ();
- else
- doc_comments = null;
- }
-
- public this (lex : LexerBase) {
- Env = GlobalEnv.Core;
- lex.Keywords = Env.Keywords;
- lexer = lex;
- }
-
- static reset_comment (tok : Token) : void {
- when (Options.XmlDoc) doc_comments = doc_comments.Replace (tok.Location, "");
- }
- static reset_comment (loc : Location) : void {
- when (Options.XmlDoc) doc_comments = doc_comments.Replace (loc, "");
- }
-
- /** Fetch next token (from one token buffer or lexer if it's empty */
- protected virtual get_token () : Token {
- if (last_token != null) {
- def result = last_token;
- last_token = null;
- result;
- }
- else {
- try {
- lexer.GetToken ()
- }
- catch {
- | e is LexerBase.Error =>
- Message.Error (lexer.Location, e.name + " " + e.StackTrace);
- get_token ()
- }
- }
- }
-
- /** Store token in our mini one token buffer */
- push_back (tok : Token) : void {
- assert (last_token == null);
- last_token = tok;
- }
-
- /** links Tokens from specified subarray to form a list and return its head */
- static make_list (tokens : SCG.List [Token], start : int) : Token
- {
- for (mutable i = tokens.Count - 2; i >= start; --i)
- tokens [i].Next = tokens [i + 1];
- tokens [start]
- }
-
-
- public static Dump (tok : Token, ident : string) : string {
- def (open, close, sepstr, elements) =
- match (tok) {
- | Token.RoundGroup => ("(", ")", ", ", tok)
- | Token.BracesGroup => ("{\n" + ident, "}", ";\n" + ident, tok)
- | Token.SquareGroup => ("[", "]", ", ", tok)
- | Token.QuoteGroup => ("<[\n", "]>", "; ", tok)
- | Token.LooseGroup => ("", "", " ", tok)
-
- | _ => ("", tok.ToString (false), "", null)
- }
- def builder = System.Text.StringBuilder (open);
- when (elements != null)
- foreach (e is Token in elements)
- _ = builder.Append (Dump (e, ident + " ")).Append (sepstr);
- builder.Append (close).ToString ()
- }
-
- /** Closes both currently created LooseGroup and parent group.
- Returns list of tokens composing parent group */
- finish_parent (parent_begin : int, current_begin : int) : Token {
- finish_current (current_begin);
- def parent_group =
- if (parent_begin == parent_stream.Count)
- null // case of `(` `)`
- else
- make_list (parent_stream, parent_begin);
- parent_stream.RemoveRange (parent_begin, parent_stream.Count - parent_begin);
- parent_group
- }
-
- /** Closes currently created LooseGroup and adds it at the end of the
- parent group. After that we are ready to make another LooseGroup.
-
- It is called mainly when separator token occurs.
- */
- finish_current (current_begin : int) : void {
- unless (current_begin == current_stream.Count) {
- def loose_group = make_list (current_stream, current_begin);
- def loose = Token.LooseGroup (loose_group.Location, loose_group);
- parent_stream.Add (loose);
- current_stream.RemoveRange (current_begin, current_stream.Count - current_begin);
- }
- }
-
- /** Handle standard situations when new bracket group is beginning
- or there is erronous situation. Any non bracket token is
- appended to current LooseGroup.
-
- Throws PreParserException when there is unmatched end bracket.
- */
- handle_default_token (current_begin : int, tok : Token) : void {
- match (tok) {
- | Token.BeginBrace =>
- def brace_group = parse_brace_group (tok.Location);
- current_stream.Add (brace_group);
- finish_current (current_begin);
-
- | Token.BeginRound =>
- def round_group = parse_round_group (tok.Location);
- current_stream.Add (round_group);
-
- | Token.BeginSquare =>
- def square_group = parse_square_group (tok.Location);
- current_stream.Add (square_group);
-
- | Token.BeginQuote =>
- def quote_group = parse_quote_group (tok.Location);
- current_stream.Add (quote_group);
-
- | Token.EndRound | Token.EndSquare | Token.EndQuote | Token.EndBrace =>
- push_back (tok);
- throw PreParserException (tok.Location, $"unexpected closing bracket `$(tok)'");
-
- | Token.EndOfFile =>
- throw PreParserException (tok.Location, "unexpected end of file");
-
- | Token.Comment (comment) when Options.XmlDoc =>
- doc_comments = doc_comments.Replace (tok.Location, comment);
-
- | Token.Comment => ()
-
- | _ => current_stream.Add (tok);
- }
- }
-
- parse_brace_group (loc : Location) : Token.BracesGroup
- {
- def parent_begin = parent_stream.Count;
- def current_begin = current_stream.Count;
-
- reset_comment (loc);
-
- def loop () {
- def tok = get_token ();
- match (tok) {
- // finish entire brace group
- | Token.EndBrace =>
- reset_comment (tok);
- def brace_group = finish_parent (parent_begin, current_begin);
- Token.BracesGroup (loc + tok.Location, brace_group);
-
- // finish current loose group
- | Token.Semicolon =>
- reset_comment (tok);
- finish_current (current_begin); loop ()
-
- | Token.EndOfFile when parent_begin == 0 =>
- def brace_group = finish_parent (parent_begin, current_begin);
- finished = true;
- Token.BracesGroup (loc + tok.Location, brace_group);
-
- | _ => handle_default_token (current_begin, tok); loop ()
- }
- }
- try { loop () }
- catch { e is PreParserException =>
- Message.Error (loc, "when parsing this `{' brace group");
- Message.Error (e.Location, e.Message);
- def group = finish_parent (parent_begin, current_begin);
- Token.BracesGroup (loc, group);
- }
- }
-
- parse_round_group (loc : Location) : Token.RoundGroup
- {
- def parent_begin = parent_stream.Count;
- def current_begin = current_stream.Count;
-
- def loop () {
- def tok = get_token ();
- match (tok) {
- // finish entire round group
- | Token.EndRound =>
- def round_group = finish_parent (parent_begin, current_begin);
- Token.RoundGroup (loc + tok.Location, round_group);
-
- // finish current loose group
- | Token.Comma =>
- finish_current (current_begin);
- loop ()
-
- | _ => handle_default_token (current_begin, tok); loop ()
- }
- }
- try { loop () }
- catch { e is PreParserException =>
- Message.Error (loc, "when parsing this `(' brace group");
- Message.Error (e.Location, e.Message);
- def group = finish_parent (parent_begin, current_begin);
- Token.RoundGroup (loc, group);
- }
- }
-
- parse_square_group (loc : Location) : Token.SquareGroup
- {
- def parent_begin = parent_stream.Count;
- def current_begin = current_stream.Count;
-
- def loop () {
- def tok = get_token ();
- match (tok) {
- // finish entire brace group
- | Token.EndSquare =>
- def group = finish_parent (parent_begin, current_begin);
- Token.SquareGroup (loc + tok.Location, group);
-
- // finish current loose group
- | Token.Comma => finish_current (current_begin); loop ()
-
- | _ => handle_default_token (current_begin, tok); loop ()
- }
- }
- try { loop () }
- catch { e is PreParserException =>
- Message.Error (loc, "when parsing this `[' brace group");
- Message.Error (e.Location, e.Message);
- def group = finish_parent (parent_begin, current_begin);
- Token.SquareGroup (loc, group);
- }
- }
-
- parse_quote_group (loc : Location) : Token.QuoteGroup
- {
- def parent_begin = parent_stream.Count;
- def current_begin = current_stream.Count;
-
- def loop () {
- def tok = get_token ();
- match (tok) {
- // finish entire brace group
- | Token.EndQuote =>
- def group = finish_parent (parent_begin, current_begin);
- Token.QuoteGroup (loc + tok.Location, group);
-
- // finish current loose group
- | Token.Semicolon => finish_current (current_begin); loop ()
-
- | _ => handle_default_token (current_begin, tok); loop ()
- }
- }
- try { loop () }
- catch { e is PreParserException =>
- Message.Error (loc, "when parsing this `<[' brace group");
- Message.Error (e.Location, e.Message);
- def group = finish_parent (parent_begin, current_begin);
- Token.QuoteGroup (loc, group);
- }
- }
-
- ParseTopLevelImpl () : Token
- {
- def parent_begin = parent_stream.Count;
- def current_begin = current_stream.Count;
-
- def get_qid () {
- match (get_token ()) {
- | Token.Identifier (x) =>
- match (get_token ()) {
- | Token.Operator (".") => x :: get_qid ()
- | t => push_back (t); [x]
- }
- | t => Message.Error (t.Location, "expected qualified identifier"); []
- }
- }
-
- def loop () {
- def tok = get_token ();
- match (tok) {
- | Token.Keyword ("using") =>
- finish_current (current_begin);
-
- def id = get_qid ();
- match (get_token ()) {
- | Token.Semicolon as st =>
- def loc = tok.Location + st.Location;
- Env = Env.AddOpenNamespace (id, loc);
- lexer.Keywords = Env.Keywords;
-
- def using_tok = Token.Using (loc, Env);
- current_stream.Add (using_tok);
-
- | Token.Operator ("=") =>
- def id' = get_qid ();
-
- def st = get_token ();
- match (st) {
- | Token.Semicolon => ()
- | _ => Message.Error (st.Location, "expecting `;' after using alias")
- }
-
- match (id) {
- | [name] => Env = Env.AddNamespaceAlias (name, id', tok.Location);
- | _ => Message.Error (tok.Location, "using alias must be simple name without dots")
- }
-
- def using_tok = Token.Using (tok.Location + st.Location, Env);
- current_stream.Add (using_tok);
-
- | x => Message.Error (x.Location, "expecting `;' or `='")
- }
- finish_current (current_begin);
- loop ()
-
- | Token.Keyword ("namespace") =>
- finish_current (current_begin);
-
- def id = get_qid ();
- match (get_token ()) {
- | Token.BeginBrace as br =>
- def loc = tok.Location + br.Location;
- def oldenv = Env;
- Env = Env.EnterIntoNamespace (id);
- lexer.Keywords = Env.Keywords;
-
- def decls = ParseTopLevelImpl ();
- def namespace_tok = Token.Namespace (loc, Env, decls);
-
- Env = oldenv;
- lexer.Keywords = Env.Keywords;
-
- current_stream.Add (namespace_tok);
-
- | x => Message.Error (x.Location, "expecting `{' opening namespace scope")
- }
- finish_current (current_begin);
- loop ()
-
- // finish entire brace group
- | Token.EndBrace =>
- reset_comment (tok);
- finish_parent (parent_begin, current_begin);
-
- // finish current loose group
- | Token.Semicolon => finish_current (current_begin); loop ()
-
- | Token.EndOfFile when parent_begin == 0 =>
- def brace_group = finish_parent (parent_begin, current_begin);
- finished = true;
- brace_group;
-
- | _ => handle_default_token (current_begin, tok); loop ()
- }
- }
- try { loop () }
- catch { e is PreParserException =>
- Message.Error (e.Location, e.Message);
- finish_parent (parent_begin, current_begin);
- }
- }
-
- public ParseTopLevel () : Token.BracesGroup {
- def stream = ParseTopLevelImpl ();
- lexer.Dispose (); // make lexer release file handle
- Token.BracesGroup (stream)
- }
-
- public PreParse () : Token.BracesGroup {
- def top = parse_brace_group (Location.Default);
- unless (finished) Message.Error (lexer.Location, "expected end of file, encountered closing brace");
- top
- }
- }
-
public class PreParserIndent : PreParser {
// For indentation syntax
mutable current_indent : string = "";
More information about the svn
mailing list