[svn] r6440: nemerle/trunk/ncc/parsing/Lexer.n

[nazgul]

Log:
Convert CR chars into newline when necessary

Author: nazgul
Date: Sun Jul  9 12:34:44 2006
New Revision: 6440

Modified:
   nemerle/trunk/ncc/parsing/Lexer.n

Modified: nemerle/trunk/ncc/parsing/Lexer.n
==============================================================================
--- nemerle/trunk/ncc/parsing/Lexer.n	(original)
+++ nemerle/trunk/ncc/parsing/Lexer.n	Sun Jul  9 12:34:44 2006
@@ -272,12 +272,19 @@
   
   do_read () : char
   {
-    def ch = read_from_input ();
+    mutable ch = read_from_input ();
     // Message.Debug (Location, $"ch = '$ch'");
 
+    when (ch == '\r')
+      if (peek_from_input_or_white () != '\n')
+        ch = '\n'; // \r is alone here, so we change it to newline
+      else
+        when (is_check_wrong_chars)
+          Message.Warning (10002, this.Location, "CR character found in input stream");
+   
     when (in_indentation_mode) {
       // Start counting indentation anew after a newline
-      if (ch == '\n' || ch == '\r') {
+      if (ch == '\n') {
         // Message.Debug (Location, $"start");
         indent_string = "";
         counting_indentation = true;
@@ -350,6 +357,7 @@
   }
 
   abstract protected read_from_input () : char;
+  abstract protected peek_from_input_or_white () : char;
 
   public abstract Dispose () : void;
   
@@ -364,14 +372,6 @@
   
     match (ch) {
       | '\n' => ++line; col = 1;
-      | '\r' =>
-        if (peek_or_white () == '\n')
-          () // will be handled later
-        else {
-          ++line; col = 1; // it is a new line character
-        }
-        when (is_check_wrong_chars)
-          Message.Warning (10002, this.Location, "CR character found in input stream")
           
       | '\t' =>
         ++col;
@@ -383,6 +383,14 @@
     ch
   }
 
+  protected peek_or_white () : char
+  {
+    if (putback)
+      putbackVal
+    else
+      peek_from_input_or_white ()
+  }
+  
   protected peek () : char
   {
     unless (putback) 
@@ -391,12 +399,6 @@
     putbackVal
   }
 
-  protected peek_or_white () : char
-  {
-    try { peek ();  }
-    catch { | _ is LexerBase.Error => ' ' }
-  }
-  
   public static IsIdBeginning (ch : char) : bool
   {
     Char.IsLetter (ch) || ch == '_'
@@ -872,7 +874,7 @@
       }
       else
         match (ch) {
-          | ' ' | '\t' | '\r' => loop (true);
+          | ' ' | '\t'  => loop (true);
  
           | '\n' =>
             white_beginning = true;
@@ -1435,6 +1437,15 @@
       
   }
 
+  override protected peek_from_input_or_white () : char
+  {
+    def inp = reader.Peek ();
+    if (inp >= 0)
+      inp :> char
+    else
+      ' '
+  }
+  
   override protected read_from_input () : char
   {
     def inp = reader.Read ();
@@ -1551,6 +1562,14 @@
 
   public override Dispose () : void { }
   
+  override protected peek_from_input_or_white () : char
+  {
+    if (pos < reader.Length)
+      reader[pos]
+    else
+      ' '
+  }
+  
   override protected read_from_input () : char
   {
     if (pos < reader.Length) {