feat: parallelised compilation #39

Merged
gk1623 merged 18 commits from gleb/parallelisation into master 2025-03-13 01:08:58 +00:00
5 changed files with 358 additions and 227 deletions
Showing only changes of commit 94ee489faf - Show all commits

View File

@@ -23,11 +23,10 @@ import cats.data.ValidatedNel
/* /*
TODO: TODO:
1) IO correctness 1) IO correctness
2) --greedy, 2) Errors can be handled more gracefully - currently, parallelised compilation is not fail fast as far as I am aware
3) parallelised compilation 3) splitting the file up and nicer refactoring
4) splitting the file up and nicer refactoring 4) logging could be removed
5) logging could be removed 5) general cleanup and comments (things like replacing home/<user> with ~ , and names of parameters and args, descriptions etc)
6) errors can be handled more gracefully probably
*/ */
given logger: Logger[IO] = Slf4jLogger.getLogger[IO] given logger: Logger[IO] = Slf4jLogger.getLogger[IO]
@@ -49,12 +48,14 @@ val filesOpt: Opts[NonEmptyList[Path]] =
_.traverse(validateFile) _.traverse(validateFile)
} }
// TODO: Is intermediate String necessary
val outputOpt: Opts[Option[Path]] = val outputOpt: Opts[Option[Path]] =
Opts Opts
.option[Path]("output", metavar = "path", help = "Output directory for compiled files.") .option[Path]("output", metavar = "path", help = "Output directory for compiled files.")
.orNone .orNone
val greedyOpt: Opts[Boolean] =
Opts.flag("greedy", "Compile WACC files sequentially instead of parallelly", short = "g").orFalse
def frontend( def frontend(
contents: String contents: String
): IO[Either[Int, microWacc.Program]] = { ): IO[Either[Int, microWacc.Program]] = {
@@ -138,27 +139,43 @@ def compile(filePath: Path, outputDir: Option[Path], log: Boolean): IO[Int] = {
} yield exitCode } yield exitCode
} }
// TODO: this is sequential, thus should be what occurs when --greedy is passed in // TODO: Remove duplicate code between compileCommandSequential and compileCommandParallel
val compileCommand: Opts[IO[ExitCode]] = def compileCommandSequential(
(filesOpt, logOpt, outputOpt).mapN { (files, log, outDir) => files: NonEmptyList[Path],
log: Boolean,
outDir: Option[Path]
): IO[ExitCode] =
files files
.traverse { file => .traverse { file =>
compile(file.toAbsolutePath, outDir, log).handleErrorWith { err => compile(file.toAbsolutePath, outDir, log).handleErrorWith { err =>
// TODO: probably a more elegant way of doing this // TODO: probably a more elegant way of doing this
// also, -1 arbitrary // also, -1 arbitrary
// also - this outputs two messages for some reason // also - this outputs two messages for some reason
logger.error(err.getMessage) // *> IO.pure(ExitCode(-1)) logger.error(err.getMessage) // *> IO.raiseError(err)
} }
} }
.map { exitCodes => .map { exitCodes =>
if (exitCodes.exists(_ != 0)) if (exitCodes.exists(_ != 0)) ExitCode.Error else ExitCode.Success
ExitCode.Error // TODO- it should be the first one to exit when parallelised :)
else ExitCode.Success
} }
def compileCommandParallel(
files: NonEmptyList[Path],
log: Boolean,
outDir: Option[Path]
): IO[ExitCode] =
files
.parTraverse { file =>
compile(file.toAbsolutePath, outDir, log).handleErrorWith { err =>
// TODO: probably a more elegant way of doing this
// also, -1 arbitrary
// also - this outputs two messages for some reason
logger.error(err.getMessage) // *> IO.raiseError(err)
}
}
.map { exitCodes =>
if (exitCodes.exists(_ != 0)) ExitCode.Error else ExitCode.Success
} }
// TODO: add parallelisable option
object Main object Main
extends CommandIOApp( extends CommandIOApp(
name = "wacc", name = "wacc",
@@ -166,6 +183,9 @@ object Main
version = "1.0" version = "1.0"
) { ) {
def main: Opts[IO[ExitCode]] = def main: Opts[IO[ExitCode]] =
compileCommand (greedyOpt, filesOpt, logOpt, outputOpt).mapN { (greedy, files, log, outDir) =>
if (greedy) compileCommandSequential(files, log, outDir)
else compileCommandParallel(files, log, outDir)
}
} }