From 42ff9c9e790ab2527b382337693b24beb563154a Mon Sep 17 00:00:00 2001 From: Gleb Koval Date: Fri, 21 Feb 2025 18:19:23 +0000 Subject: [PATCH] test: backend tests --- src/test/wacc/examples.scala | 122 +++++++++++++++++++---------------- 1 file changed, 67 insertions(+), 55 deletions(-) diff --git a/src/test/wacc/examples.scala b/src/test/wacc/examples.scala index f62d537..abff693 100644 --- a/src/test/wacc/examples.scala +++ b/src/test/wacc/examples.scala @@ -3,6 +3,9 @@ package wacc import org.scalatest.{ParallelTestExecution, BeforeAndAfterAll} import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.Inspectors.forEvery +import java.io.File +import sys.process._ +import java.io.PrintStream class ParallelExamplesSpec extends AnyFlatSpec with BeforeAndAfterAll with ParallelTestExecution { val files = @@ -20,12 +23,51 @@ class ParallelExamplesSpec extends AnyFlatSpec with BeforeAndAfterAll with Paral } // tests go here - forEvery(files.filter { (filename, _) => - !fileIsDissallowed(filename) - }) { (filename, expectedResult) => - s"$filename" should "be parsed with correct result" in { - val contents = os.read(os.Path(filename)) - assert(expectedResult.contains(compile(contents))) + forEvery(files) { (filename, expectedResult) => + val baseFilename = filename.stripSuffix(".wacc") + given stdout: PrintStream = PrintStream(File(baseFilename + ".out")) + val result = compile(filename) + + s"$filename" should "be compiled with correct result" in { + assert(expectedResult.contains(result)) + } + + if (result == 0) it should "run with correct result" in { + if (fileIsDisallowedBackend(filename)) pending + + // Retrieve contents to get input and expected output + exit code + val contents = scala.io.Source.fromFile(File(filename)).getLines.toList + val inputLine = + contents.find(_.matches("^# ?[Ii]nput:.*$")).map(_.split(":").last.strip).getOrElse("") + val outputLineIdx = contents.indexWhere(_.matches("^# ?[Oo]utput:.*$")) + val expectedOutput = + if (outputLineIdx == -1) "" + else + contents + .drop(outputLineIdx + 1) + .takeWhile(_.startsWith("#")) + .map(_.stripPrefix("#").stripLeading) + .mkString("\n") + + val exitLineIdx = contents.indexWhere(_.matches("^# ?[Ee]xit:.*$")) + val expectedExit = + if (exitLineIdx == -1) 0 + else contents(exitLineIdx + 1).stripPrefix("#").strip.toInt + + // Assembly and link using gcc + val asmFilename = baseFilename + ".s" + val execFilename = baseFilename + val gccResult = s"gcc -o $execFilename -z noexecstack $asmFilename".! + assert(gccResult == 0) + + // Run the executable with the provided input + val stdout = new StringBuilder + // val execResult = s"$execFilename".!(ProcessLogger(stdout.append(_))) + val execResult = + s"echo $inputLine" #| s"timeout 5s $execFilename" ! ProcessLogger(stdout.append(_)) + + assert(execResult == expectedExit) + assert(stdout.toString == expectedOutput) } } @@ -33,57 +75,27 @@ class ParallelExamplesSpec extends AnyFlatSpec with BeforeAndAfterAll with Paral val d = java.io.File(dir) os.walk(os.Path(d.getAbsolutePath)).filter { _.ext == "wacc" } - def fileIsDissallowed(filename: String): Boolean = + def fileIsDisallowedBackend(filename: String): Boolean = Seq( // format: off // disable formatting to avoid binPack - // "wacc-examples/valid/advanced", - // "wacc-examples/valid/array", - // "wacc-examples/valid/basic/exit", - // "wacc-examples/valid/basic/skip", - // "wacc-examples/valid/expressions", - // "wacc-examples/valid/function/nested_functions", - // "wacc-examples/valid/function/simple_functions", - // "wacc-examples/valid/if", - // "wacc-examples/valid/IO/print", - // "wacc-examples/valid/IO/read", - // "wacc-examples/valid/IO/IOLoop.wacc", - // "wacc-examples/valid/IO/IOSequence.wacc", - // "wacc-examples/valid/pairs", - // "wacc-examples/valid/runtimeErr", - // "wacc-examples/valid/scope", - // "wacc-examples/valid/sequence", - // "wacc-examples/valid/variables", - // "wacc-examples/valid/while", - // invalid (syntax) - // "wacc-examples/invalid/syntaxErr/array", - // "wacc-examples/invalid/syntaxErr/basic", - // "wacc-examples/invalid/syntaxErr/expressions", - // "wacc-examples/invalid/syntaxErr/function", - // "wacc-examples/invalid/syntaxErr/if", - // "wacc-examples/invalid/syntaxErr/literals", - // "wacc-examples/invalid/syntaxErr/pairs", - // "wacc-examples/invalid/syntaxErr/print", - // "wacc-examples/invalid/syntaxErr/sequence", - // "wacc-examples/invalid/syntaxErr/variables", - // "wacc-examples/invalid/syntaxErr/while", - // invalid (semantic) - // "wacc-examples/invalid/semanticErr/array", - // "wacc-examples/invalid/semanticErr/exit", - // "wacc-examples/invalid/semanticErr/expressions", - // "wacc-examples/invalid/semanticErr/function", - // "wacc-examples/invalid/semanticErr/if", - // "wacc-examples/invalid/semanticErr/IO", - // "wacc-examples/invalid/semanticErr/multiple", - // "wacc-examples/invalid/semanticErr/pairs", - // "wacc-examples/invalid/semanticErr/print", - // "wacc-examples/invalid/semanticErr/read", - // "wacc-examples/invalid/semanticErr/scope", - // "wacc-examples/invalid/semanticErr/variables", - // "wacc-examples/invalid/semanticErr/while", - // invalid (whack) - // "wacc-examples/invalid/whack" + "^.*wacc-examples/valid/advanced.*$", + "^.*wacc-examples/valid/array.*$", + "^.*wacc-examples/valid/basic/skip.*$", + "^.*wacc-examples/valid/expressions.*$", + "^.*wacc-examples/valid/function/nested_functions.*$", + "^.*wacc-examples/valid/function/simple_functions.*$", + "^.*wacc-examples/valid/if.*$", + "^.*wacc-examples/valid/IO/print.*$", + "^.*wacc-examples/valid/IO/read(?!echoInt\\.wacc).*$", + "^.*wacc-examples/valid/IO/IOLoop.wacc.*$", + "^.*wacc-examples/valid/IO/IOSequence.wacc.*$", + "^.*wacc-examples/valid/pairs.*$", + "^.*wacc-examples/valid/runtimeErr.*$", + "^.*wacc-examples/valid/scope.*$", + "^.*wacc-examples/valid/sequence.*$", + "^.*wacc-examples/valid/variables.*$", + "^.*wacc-examples/valid/while.*$", // format: on - // format: on - ).find(filename.contains).isDefined + ).find(filename.matches).isDefined }