/* * Copyright (c) 2003-2005 The University of Wroclaw. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met : * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the University may not be used to endorse or promote * products derived from this software without specific prior * written permission. * * THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN * NO EVENT SHALL THE UNIVERSITY BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ // NO-TEST using System; using System.Console; using System.Diagnostics; using System.ComponentModel; using System.IO; using System.Text.RegularExpressions; using System.String; using Nemerle.Collections; using Nemerle.Collections.List; using Nemerle.Utility; using Nemerle.Text; using Nemerle.IO; using Nemerle.Compiler; public class End_test : System.Exception { public this () {} } public class Tester { public static mutable peverify : string = ""; mutable Manager : ManagerClass; private dnet_runtime : string; private runtime_parms : list [string]; private parameters : list [string]; private dlls : list [string]; private nem_runtime : string; private verbose : int; private get_short_filename : Regex; private external_compiler : bool; private mutable no_test : bool; private mutable filename : string; private mutable filename_short : string; private mutable checked_msg : Hashtable [string, string]; private mutable warn_lines : Hashtable [string, Regex]; private mutable err_lines : Hashtable [string, Regex]; private mutable global_warn_lines : Hashtable [string, Regex]; private mutable global_err_lines : Hashtable [string, Regex]; private mutable err_overall_count : int; private mutable ok_lines : Hashtable [string, string]; private mutable is_reading_output : bool; private mutable expected_output : list [string]; private mutable ref_dlls : list [string]; private mutable options : list [string]; private mutable ref_pkgs : list [string]; private mutable was_output : bool; private mutable is_reading_input : bool; private mutable input : list [string]; private mutable error_log : StreamWriter; private mutable was_error_header : bool; private mutable log_file_created : bool; private mutable no_verify : bool; mutable final_message : string; mutable nem_output : list [string] = []; public this (dnet_runtime : string, runtime_parms : list [string], parms : list [string], dlls : list [string], nem_comp : string, verbose : int, external_compiler : bool) { this.dnet_runtime = dnet_runtime; this.runtime_parms = runtime_parms; this.parameters = parms; this.dlls = dlls; this.nem_runtime = nem_comp; this.verbose = verbose; this.external_compiler = external_compiler; err_overall_count = 0; get_short_filename = Regex (@".*/(?<1>[\w\d-_]+)\.n$"); error_log = null; log_file_created = false; Init (""); } private Init (filename : string) : void { checked_msg = Hashtable (); this.filename = filename; def group_match = get_short_filename.Match (filename).Groups[1]; if (group_match.Success) filename_short = group_match.ToString (); else filename_short = filename; warn_lines = Hashtable (); err_lines = Hashtable (); global_warn_lines = Hashtable (); global_err_lines = Hashtable (); ok_lines = Hashtable (); is_reading_output = false; expected_output = []; ref_dlls = []; ref_pkgs = []; options = []; was_output = false; is_reading_input = false; input = []; no_test = false; no_verify = false; was_error_header = false; when (error_log!= null) { error_log.Close (); error_log = null } } private CheckLine (str : string, line_index : int) : void { regexp match (str) { | "^END-OUTPUT" => is_reading_output = false | "^END-INPUT" => is_reading_input = false | _ => if (is_reading_output) expected_output = (str :: (expected_output)) else if (is_reading_input) input = (str :: (input)) else regexp match (str) { | @".*//\s*E:\s*(?.*)\s*$" => err_lines.Add (line_index.ToString (), Regex (text)) | @".*//\s*W:\s*(?.*)\s*" => warn_lines.Add (line_index.ToString (), Regex (text)) | @".*//\s*GE:\s*(?.*)\s*$" => global_err_lines.Add (line_index.ToString (), Regex (text)) | @".*//\s*GW:\s*(?.*)\s*" => global_warn_lines.Add (line_index.ToString (), Regex (text)) | "^BEGIN-OUTPUT" => is_reading_output = true; was_output = true | "^BEGIN-INPUT" => is_reading_input = true | @".*//\s*OK\s*$" => ok_lines.Add (line_index.ToString (), "") | @".*REFERENCE\s{0,1}:\s*(?.*)$" => ref_dlls = text :: ref_dlls | @".*OPTIONS\s{0,1}:\s*(?.*)$" => options = text :: options | @".*PKG-CONFIG\s{0,1}:\s*(?.*)$" => ref_pkgs = text :: ref_pkgs | @".*NO-TEST.*" => throw End_test () | @".*NO-VERIFY.*" => no_verify = true | _ => () } } } private GetLineAndMsgType (out_str : string) : int*string { def out_line_match = (Regex (":(?<1>\\d*):\\d*:(?<2>\\d*:\\d*:){0,1}\\s+(?<3>.*)$")).Match (out_str); if (out_line_match.Success) { def line_number = ((out_line_match.Groups[1]).ToString ()); def msg_type_str = ((out_line_match.Groups[3]).ToString ()); if (msg_type_str.IndexOf ("error:") != -1) (0, line_number) else if (msg_type_str.IndexOf ("warning:") != -1) (1, line_number) else (-1, line_number) } else { if (out_str.IndexOf ("error:") != -1) (2, "") else if (out_str.IndexOf ("warning:") != -1) (3, "") else (-1, "") } } private HandleMsg (out_str : string) : bool { match (GetLineAndMsgType (out_str)) { | (0, line) => match (err_lines.Get (line)) { | Some (err) => when (err.Match (out_str).Success && (!checked_msg.Contains (line))) { checked_msg.Add (line, ""); } true | None => if (err_lines.Count == 0) false else { match (ok_lines.Get (line)) { | Some (_) => false | None => true } } } | (1, line) => match (warn_lines.Get (line)) { | Some (warn) => when (warn.Match (out_str).Success && (!checked_msg.Contains (line))) checked_msg.Add (line, ""); true | None => false } | (2, _) => def matched = global_err_lines.Filter ((_line, r) => r.Match (out_str).Success); matched.Iter ((line, _r) => when (!checked_msg.Contains (line)) checked_msg.Add (line, "")); matched.Count > 0 | (3, _) => def matched = global_warn_lines.Filter ((_line, r) => r.Match (out_str).Success); matched.Iter ((line, _r) => when (!checked_msg.Contains (line)) checked_msg.Add (line, "")); matched.Count > 0 | _ => true } } private NemerleTest () : bool { mutable args = []; foreach (dll in dlls) args += ["-r:" + dll]; foreach (dll in ref_dlls) args += ["-r:" + dll]; args += options; foreach (pkg in ref_pkgs) args += ["-pkg-config:" + pkg]; args += parameters; args += ["-no-color"]; if (was_output) args += ["-out:" + filename_short + ".exe", "-t:exe"] else args += ["-out:" + filename_short + ".dll", "-t:library"]; mutable nem_compile = null; when (external_compiler) { args += [filename]; nem_compile = Process (); if (dnet_runtime.Length > 0) { nem_compile.StartInfo.FileName = dnet_runtime; args = nem_runtime :: (runtime_parms + args); } else { nem_compile.StartInfo.FileName = nem_runtime; } nem_compile.StartInfo.Arguments = (args.ToString (" ")); nem_compile.StartInfo.RedirectStandardOutput = true; nem_compile.StartInfo.UseShellExecute = false; } try { when (verbose > 0) print ("RUN: $args on $filename\n"); mutable exit_code = 0; nem_output = []; if (external_compiler) { ignore (nem_compile.Start ()); def read_output (acc) { def line = nem_compile.StandardOutput.ReadLine (); if (line == null) List.Rev (acc) else read_output (line :: acc) }; // gotta read output *first*, and then wait for process to exit // otherwise pipe buffer overflows nem_output = read_output ([]); unless (nem_compile.WaitForExit (20000) || nem_compile.HasExited) nem_compile.Kill (); exit_code = nem_compile.ExitCode; } else { def Options = CompilationOptions (); Getopt.Parse (Getopt.Error, Options.GetCommonOptions (), args); Options.IgnoreConfusion = true; Options.ProgressBar = false; Options.Sources = [filename]; Manager = ManagerClass (Options); Manager.InitOutput (System.IO.TextWriter.Null); Manager.MessageOccured += fun (_, s) { // FIXME, change + to :: and see nem_output = NString.Split (s, array ['\n']) + nem_output; }; try { Manager.Run () } catch { | _ is System.IO.FileNotFoundException => exit_code = 1 | _ is AssemblyFindException => exit_code = 3 | e => when (verbose > 0) print ("EXN: $e\n$(e.StackTrace)\n"); exit_code = 2 } when (Message.SeenError && exit_code != 3) exit_code = 1; nem_output = List.Rev (nem_output); } mutable is_ok = true; def Check_err (line : string, msg : Regex) { match (checked_msg.Get (line)) { | Some (_) => () | None => HandleErrorMsg ("Expected error:\n`" + msg.ToString () + "'\n hasn't occured in line:" + line); is_ok = false } } def Check_warn (line : string, msg : Regex) { match (checked_msg.Get (line)) { | Some (_) => () | None => HandleErrorMsg ("Expected warning:\n`" + msg.ToString () + "'\n hasn't occured in line:" + line); is_ok = false } }; def Check_global_err (line : string, msg : Regex) { match (checked_msg.Get (line)) { | Some (_) => () | None => HandleErrorMsg ("Expected error:\n`" + msg.ToString () + "'\n hasn't occured"); is_ok = false } } def Check_global_warn (line : string, msg : Regex) { match (checked_msg.Get (line)) { | Some (_) => () | None => HandleErrorMsg ("Expected warning:\n`" + msg.ToString () + "'\n hasn't occured"); is_ok = false } }; when (exit_code != 3) { foreach (output in nem_output) { when (this.verbose > 0) printf ("VERB-NEM : %s\n", output); unless (HandleMsg (output)) { HandleErrorMsg ("Unexpected Nemerle compiler's message :\n" + output); is_ok = false }; }; err_lines.Iter (Check_err); warn_lines.Iter (Check_warn); global_err_lines.Iter (Check_global_err); global_warn_lines.Iter (Check_global_warn); } if (exit_code == 2) { HandleErrorMsg ("Nemerle compiler reported internal error"); false } else { if (exit_code == 3) { was_output = false; no_verify = true; final_message = "unable to load library"; } else when ((exit_code == 0) != (err_lines.Count == 0)) is_ok = false; is_ok } } catch { | _ is Win32Exception => HandleErrorMsg ("Error occured while running Nemerle compiler" " (could not run the compiler file)"); false | e => HandleErrorMsg ($"Error occured while running Nemerle compiler, $e"); false } } private VerifyOutputAssembly () : bool { if (peverify == "") true else if (no_verify) { printf ("skipped verification..."); true } else { def verifier = Process (); verifier.StartInfo.FileName = peverify; verifier.StartInfo.Arguments = filename_short + if (was_output) ".exe" else ".dll"; verifier.StartInfo.UseShellExecute = false; verifier.StartInfo.RedirectStandardOutput = true; verifier.StartInfo.RedirectStandardError = true; printf ("verify..."); try { def _ = verifier.Start (); def _stdout = verifier.StandardOutput.ReadToEnd (); def _stderr = verifier.StandardError.ReadToEnd (); unless (verifier.WaitForExit (20000) || verifier.HasExited) verifier.Kill (); when (verifier.ExitCode != 0) { print (_stdout); print (_stderr); } verifier.ExitCode == 0 } catch { | e => print (e.Message); false } } } private TestOutput () : bool { try { def runtime = Process (); if (dnet_runtime.Length> 0) { runtime.StartInfo.FileName = dnet_runtime; runtime.StartInfo.Arguments = "./" + filename_short + ".exe"; } else { runtime.StartInfo.FileName = filename_short + ".exe"; runtime.StartInfo.Arguments = ""; }; //System.Console.WriteLine (runtime.StartInfo.FileName); runtime.StartInfo.UseShellExecute = false; runtime.StartInfo.RedirectStandardOutput = true; runtime.StartInfo.RedirectStandardInput = true; runtime.StartInfo.RedirectStandardError = true; printf ("run..."); def _ = runtime.Start (); def WriteInput (in_list : list [string], input : StreamWriter) : void { match (in_list) { | [] => () | h :: tl => input.WriteLine (h); WriteInput (tl, input) } }; WriteInput (input, runtime.StandardInput); def read_output (acc) { def line = runtime.StandardOutput.ReadLine (); if (line == null) List.Rev (acc) else read_output (line :: acc) }; // gotta read output *first*, and then wait for process to exit // otherwise pipe buffer overflows def out_list = read_output ([]); unless (runtime.WaitForExit (20000) || runtime.HasExited) runtime.Kill (); def CheckOutput (in_list : list [string], out_list : list [string], is_ok : bool) : bool { match ((in_list, out_list)) { | ([], []) => is_ok | ([], _) => HandleErrorMsg ("Runtime output is longer than it should be"); false | (_, []) => HandleErrorMsg ("Unexpected end of runtime output"); false | (h :: tl, str :: strs) => when (verbose > 1) printf ("VERB-RUN : %s\n", str); /// get rid of whitespaces in program's output and test, then compare if (h.Trim () == str.Trim ()) CheckOutput (tl, strs, is_ok) else { HandleErrorMsg ("Runtime output : \n\"" + str + "\"\ndoes not match expected : \n\"" + h +"\""); CheckOutput (tl, strs, false) } } }; def stderr = runtime.StandardError.ReadToEnd (); if (runtime.ExitCode != 0) { HandleErrorMsg ($"Test finished with exit code $(runtime.ExitCode)"); false } else if (stderr != null && ! stderr.Equals("")) { HandleErrorMsg ("Following error message has been written :"+stderr); false } else CheckOutput (expected_output, out_list, true) } catch { e is Win32Exception => HandleErrorMsg ("Error occured while running the program "); when (verbose > 1) { print(e); } false } } private HandleErrorMsg (str : string) : void { if (error_log == null) { Console.ForegroundColor = ConsoleColor.Red; printf ("failed\n%s\n", str); Console.ResetColor(); if (log_file_created) error_log = File.AppendText ("test_error.log") else { error_log = File.CreateText ("test_error.log"); log_file_created = true } } else printf ("%s\n", str); when (!was_error_header) { error_log.WriteLine ("-------------------------"); error_log.WriteLine ("Test of " + filename); error_log.WriteLine ("-------------------------"); was_error_header = true }; error_log.WriteLine (str); error_log.WriteLine ("-------------------------") } public AttachFile (filename : string) : bool { try { Init (filename); def reader = File.OpenText (filename); mutable temp_str = reader.ReadLine (); mutable index = 1; while (temp_str != null) { CheckLine (temp_str, index); index = index+1; temp_str = reader.ReadLine (); }; reader.Close (); expected_output = List.Rev (expected_output); input = List.Rev (input); true } catch { | _ is End_test => no_test = true; true | err => printf ("Testing %s...", filename); err_overall_count = err_overall_count+1; match (err) { | _ is FileNotFoundException => HandleErrorMsg ("File `"+filename+"' does not exist") | _ is DirectoryNotFoundException => HandleErrorMsg ("This path `"+filename+"' is invalid") | _ is PathTooLongException => HandleErrorMsg ("This path `"+filename+"' is invalid") | e => HandleErrorMsg ("Error occured while reading the file: " + e.Message) }; false } } public Test () : void { printf ("Testing %s...", filename); if (no_test) printf ("skipped\n") else { final_message = "passed"; def prindFinalMessage() { when (final_message != "passed") Console.ForegroundColor = ConsoleColor.Yellow; Console.WriteLine (final_message); Console.ResetColor(); } when (verbose > 0) printf ("\n"); try { if (NemerleTest ()) { if (err_lines.Count == 0) { if (VerifyOutputAssembly ()) try { if (was_output) { if (TestOutput ()) prindFinalMessage(); else { HandleErrorMsg ("Runtime test failed"); ++err_overall_count } } else prindFinalMessage(); } catch { _ is Win32Exception => HandleErrorMsg ("Error occured while running testcase") } else { HandleErrorMsg ("Verification failed"); ++err_overall_count; } } else prindFinalMessage(); } else { HandleErrorMsg ("Nemerle compile test failed"); ++err_overall_count; } } catch { _ is Win32Exception => HandleErrorMsg ("Error occured while running Nemerle compiler") } } } public GetReport () : int { when (error_log!= null) { error_log.Close (); error_log = null }; if (err_overall_count == 0) { Console.ForegroundColor = ConsoleColor.Cyan; Write ("All tests passed"); Console.ResetColor(); 0 } else { Console.ForegroundColor = ConsoleColor.Red; WriteLine ($"$err_overall_count of tests failed"); Console.ResetColor(); WriteLine ("See test_error.log file for details"); 1 } } private static mutable global_exit_code : int; public static Main () : int { def needs_bigger_stack () { typeof (object).Assembly.GetType ("System.RuntimeType") != null } if (needs_bigger_stack ()) { def thread = System.Threading.Thread (main_with_exit_code, 12 * 1024 * 1024); thread.Start (); thread.Join (); } else main_with_exit_code (); global_exit_code } private static main_with_exit_code () : void { global_exit_code = main () } private static main () : int { mutable dnet_env = ""; mutable runtime_parms = []; mutable nem_comp = "ncc.exe"; mutable dlls = []; mutable verbose = 0; mutable test_files = []; mutable test_dirs = []; mutable parms = []; mutable sort = false; mutable external_compiler = false; def split_opt (s) { if (s == null) [] else { def split = NString.Split (s, array [' ', '\t', '\n', '\r']); def split = List.Map (split, fun (x : string) { x.Trim () }); List.Filter (split, fun (x) { x != "" }) } } def opts = [ Getopt.CliOption.String (name = "-n", aliases = ["-ncc"], help = "use this Nemerle compiler (default : internal) ", handler = fun (s) { nem_comp = s; external_compiler = true; }), Getopt.CliOption.String (name = "-reference", aliases = ["-ref"], help = "dll to be referenced during file(s)" " compilation (default : none) ", handler = fun (s) { dlls = s :: dlls }), Getopt.CliOption.String (name = "-parameters", aliases = ["-parms", "-p"], help = "parameters passed to the compiler (default : none) ", handler = fun (s) { parms = parms + split_opt (s) }), Getopt.CliOption.String (name = "-verifier", aliases = ["-verify"], help = "command for executing PEVerify program (default : none) ", handler = fun (s : string) { Tester.peverify = s.Trim (array [' ']) }), Getopt.CliOption.String (name = "-r", aliases = ["-runtime"], help = "use this .Net runtime engine (default : none) ", handler = fun (s) { dnet_env = (s : String).Trim (); }), Getopt.CliOption.String (name = "-rp", aliases = ["-runtime-params"], help = "parameters passed to the .Net runtime (default : none)", handler = fun (s) { runtime_parms = runtime_parms + split_opt (s) }), Getopt.CliOption.Flag (name = "-v", aliases = ["-verbose"], help = "prints all Nemerle output (default : off) ", handler = fun () { verbose = verbose + 1; }), Getopt.CliOption.Flag (name = "-vv", help = "prints all Nemerle and runtime output (default : off) ", handler = fun () { verbose = verbose + 2; }), Getopt.CliOption.Flag (name = "-s", aliases = ["-sort"], help = "sort testcases (default : off) ", handler = fun () { sort = true; }), Getopt.CliOption.String (name = "-d", aliases = ["-directory"], help = "directory with test files", handler = fun (s) { test_dirs = s :: test_dirs }), Getopt.CliOption.Flag (name = "-debugger", aliases = [], help = "Display assert dialog for user can start debug session", handler = () => System.Diagnostics.Debug.Assert(false, "Press Retry to start debug session of test. " "If you wants to see this dialog at next time you should " "remove '-debugger' option from command line")), Getopt.CliOption.NonOption (name = "", help = "specify file (s) to be tested (default :" " All *.n files from current directory) ", handler = fun (s) { test_files = s :: test_files; }) ]; Getopt.Parse (opts); def tester = Tester (dnet_env, runtime_parms, parms, dlls, nem_comp, verbose, external_compiler); def load_files_from_dir (directory) { def str = Directory.GetFiles (directory, "*.n"); Array.Sort (str); mutable fcounter = str.Length - 1; while (fcounter >= 0) { // For Windows compatibility : test_files = (str[fcounter].Replace ("\\", "/")) :: test_files; fcounter = fcounter - 1 } } test_dirs.Iter (load_files_from_dir (_)); if (List.IsEmpty (test_files)) load_files_from_dir (Directory.GetCurrentDirectory ()) else test_files = List.Rev (test_files); when (sort) test_files = List.Sort (test_files, String.CompareOrdinal); foreach (file in test_files) when (tester.AttachFile (file)) tester.Test (); tester.GetReport () } }