[svn] r5948: nemerle/trunk/ncc: parsing/Lexer.n
testsuite/positive/lexer1.n
nazgul
svnadmin at nemerle.org
Sat Nov 19 22:40:31 CET 2005
Log:
Use safer peek when needed, avoiding crash when end of file encountered. Fixes #565
Author: nazgul
Date: Sat Nov 19 22:40:28 2005
New Revision: 5948
Modified:
nemerle/trunk/ncc/parsing/Lexer.n
nemerle/trunk/ncc/testsuite/positive/lexer1.n
Modified: nemerle/trunk/ncc/parsing/Lexer.n
==============================================================================
--- nemerle/trunk/ncc/parsing/Lexer.n (original)
+++ nemerle/trunk/ncc/parsing/Lexer.n Sat Nov 19 22:40:28 2005
@@ -380,6 +380,12 @@
putbackVal
}
+ protected peek_or_white () : char
+ {
+ try { peek (); }
+ catch { | _ is LexerBase.Error => ' ' }
+ }
+
public static IsIdBeginning (ch : char) : bool
{
Char.IsLetter (ch) || ch == '_'
@@ -410,7 +416,7 @@
mutable go = true;
while (go) {
- if (IsOperatorChar (peek ())) {
+ if (IsOperatorChar (peek_or_white ())) {
def c = read ();
if (c == '/')
if (comment_beginning () == '/')
@@ -435,12 +441,12 @@
match (first_ch) {
| '.' => NumberMode.Float
| '0' =>
- match (peek ()) {
+ match (peek_or_white ()) {
| 'x' | 'X' => ignore (read ()); NumberMode.Hex
| 'o' | 'O' => ignore (read ()); NumberMode.Octal
| 'b' | 'B' =>
ignore (read ());
- unless (Char.IsDigit (peek ())) already_seen_type = true;
+ unless (Char.IsDigit (peek_or_white ())) already_seen_type = true;
NumberMode.Binary
| x when Char.IsDigit (x) =>
@@ -458,12 +464,12 @@
// read digits and . between them if it is present
def loop () {
- match (peek ()) {
+ match (peek_or_white ()) {
| '.' =>
when (mode == NumberMode.Decimal) {
mode = NumberMode.Float;
ignore (read ());
- if (Char.IsDigit (peek ())) {
+ if (Char.IsDigit (peek_or_white ())) {
ignore (id_buffer.Append ('.'));
loop ()
}
@@ -484,7 +490,7 @@
| '_' =>
_ = read ();
- if (char.IsDigit (peek ()))
+ if (char.IsDigit (peek_or_white ()))
loop ()
else {
isPendingChar = true;
@@ -501,24 +507,24 @@
def exponent_part (only_realsuf) {
when (!only_realsuf) {
- match (peek ()) {
+ match (peek_or_white ()) {
| 'E' | 'e' =>
ignore (id_buffer.Append (read ()));
- match (peek ()) {
+ match (peek_or_white ()) {
| '+' | '-' =>
ignore (id_buffer.Append (read ()));
| _ => ()
};
- if (Char.IsDigit (peek ()))
+ if (Char.IsDigit (peek_or_white ()))
do {
ignore (id_buffer.Append (read ()));
- } while (Char.IsDigit (peek ()))
+ } while (Char.IsDigit (peek_or_white ()))
else
throw Error ("no digits after exponent sign in float literal")
| _ => ()
}
};
- match (Char.ToLower (peek (), CultureInfo.InvariantCulture)) {
+ match (Char.ToLower (peek_or_white (), CultureInfo.InvariantCulture)) {
| 'f' =>
ignore (read ());
Token.FloatLiteral (Single.Parse (id_buffer.ToString (),
@@ -553,13 +559,13 @@
if (already_seen_type)
'b'
else
- Char.ToLower (peek (), CultureInfo.InvariantCulture);
+ Char.ToLower (peek_or_white (), CultureInfo.InvariantCulture);
if (special (ch)) {
unless (already_seen_type) ignore (read ());
mutable unsigned = ch == 'u';
// we can have two letter suffixes
- def ch' = Char.ToLower (peek (), CultureInfo.InvariantCulture);
+ def ch' = Char.ToLower (peek_or_white (), CultureInfo.InvariantCulture);
if (special (ch')) {
ignore (read ());
if (ch' == 'u') {
@@ -614,13 +620,13 @@
try {
match (mode) {
| NumberMode.Float =>
- match (peek ()) {
+ match (peek_or_white ()) {
| 'E' | 'e' => exponent_part (false)
| _ => exponent_part (true)
}
| NumberMode.Decimal =>
if (last_was_digit)
- match (Char.ToLower (peek (), CultureInfo.InvariantCulture))
+ match (Char.ToLower (peek_or_white (), CultureInfo.InvariantCulture))
{
| 'e' => exponent_part (false)
| 'f' | 'd' | 'm' => exponent_part (true)
@@ -670,17 +676,17 @@
protected get_id (first_ch : char) : Token
{
- if (first_ch == '\'' && !IsIdBeginning (peek ())) {
+ if (first_ch == '\'' && !IsIdBeginning (peek_or_white ())) {
get_char ()
}
else {
clear_id_buffer ();
_ = id_buffer.Append (first_ch);
- mutable next = peek ();
+ mutable next = peek_or_white ();
while (IsIdBeginning (next) || Char.IsDigit (next) || next == '\'') {
_ = id_buffer.Append (read ());
- next = peek ();
+ next = peek_or_white ();
};
def str = System.String.Intern (id_buffer.ToString ());
@@ -776,7 +782,7 @@
loop ();
| '$' when !is_dollarized && end_ch == '"' =>
- def next = peek ();
+ def next = peek_or_white ();
when (char.IsLetter (next) || next == '_' || next == '(')
Message.Warning (10007, this.Location,
"`$' occurs inside string literal, which is not prefixed itself with `$'");
@@ -789,7 +795,7 @@
loop ();
| _ =>
- when (eat_whitespace () && peek () == end_ch) {
+ when (eat_whitespace () && peek_or_white () == end_ch) {
ignore_comments ();
_ = read ();
loop ();
@@ -808,7 +814,7 @@
def loop () {
match (read ()) {
| '"' =>
- match (peek ()) {
+ match (peek_or_white ()) {
| '"' =>
_ = buf.Append ('"');
_ = read ();
@@ -912,7 +918,7 @@
| '\'' => get_id ('\'')
| '.' =>
- if (Char.IsDigit (peek ()))
+ if (Char.IsDigit (peek_or_white ()))
get_number (ch)
else
Token.Operator (".")
@@ -921,7 +927,7 @@
| '}' => Token.EndBrace (generated = false)
| '[' => Token.BeginSquare ()
| ']' =>
- if (peek () == '>') {
+ if (peek_or_white () == '>') {
ignore (read ());
Token.EndQuote ()
}
@@ -932,7 +938,7 @@
| ',' => Token.Comma ()
| ';' => Token.Semicolon (generated = false)
- | '<' when peek () == '[' => ignore (read ()); Token.BeginQuote ()
+ | '<' when peek_or_white () == '[' => ignore (read ()); Token.BeginQuote ()
| '@' =>
def next = read ();
@@ -953,7 +959,7 @@
throw Error ("expecting operator, identifier or string literal after '@'")
| '*' =>
- if (peek () == '*')
+ if (peek_or_white () == '*')
get_op (ch)
else
get_op (ch)
@@ -965,16 +971,16 @@
=> get_op (ch)
| '-' =>
- if (peek () != '.') get_op (ch)
+ if (peek_or_white () != '.') get_op (ch)
else Token.Operator ("-")
| '+'
=>
- if (peek () != '.') get_op (ch)
+ if (peek_or_white () != '.') get_op (ch)
else Token.Operator ("+")
| '$' =>
- if (eat_whitespace () && (peek () == '"' || peek () == '@')) {
+ if (eat_whitespace () && (peek_or_white () == '"' || peek_or_white () == '@')) {
// we will not warn about $ in string literal in this mode
def c = read ();
@@ -1073,7 +1079,7 @@
def read_word () : string {
def word = StringBuilder (eat_spaces ().ToString ());
try {
- while (IsIdBeginning (peek ()) || Char.IsDigit (peek ()))
+ while (IsIdBeginning (peek_or_white ()) || Char.IsDigit (peek_or_white ()))
_ = word.Append (read ())
} catch { _ is LexerBase.Error => () };
word.ToString ()
@@ -1140,10 +1146,8 @@
}
else {
def num = StringBuilder (c.ToString ());
- try {
- while (Char.IsDigit (peek ()))
- ignore (num.Append (read ()))
- } catch { _ is LexerBase.Error => c = ' ' };
+ while (Char.IsDigit (peek_or_white ()))
+ ignore (num.Append (read ()));
(Int32.Parse (num.ToString ()), read_to_the_end_of_line ().Trim ())
};
@@ -1431,7 +1435,7 @@
override protected comment_beginning () : char
{
- match (peek ()) {
+ match (peek_or_white ()) {
| '/' =>
// we are for sure in one line comment
_ = read ();
@@ -1470,7 +1474,7 @@
| _ => loop1 (false, store)
}
};
- if (store_comments && peek () == '*') {
+ if (store_comments && peek_or_white () == '*') {
comment_loc = Location (file_idx, line, col - 2);
_ = read ();
loop1 (true, true);
@@ -1548,7 +1552,7 @@
override protected comment_beginning () : char
{
- match (peek ()) {
+ match (peek_or_white ()) {
| '/' =>
// we are for sure in one line comment
try {
Modified: nemerle/trunk/ncc/testsuite/positive/lexer1.n
==============================================================================
--- nemerle/trunk/ncc/testsuite/positive/lexer1.n (original)
+++ nemerle/trunk/ncc/testsuite/positive/lexer1.n Sat Nov 19 22:40:28 2005
@@ -1,4 +1,5 @@
// W: no new line
// OPTIONS: -pedantic-lexer
- def _ = "no newline";
\ No newline at end of file
+ // check for operator ending the program without newline
+ [assembly: System.Runtime.InteropServices.ComVisible (false)]
\ No newline at end of file
More information about the svn
mailing list