Commit 67e822c6 authored by Jens Korinth's avatar Jens Korinth

Implement --parallel switch

* Jobs are executed sequentially by default, but there are cases in
  which it is useful to execute them in parallel instead (especially in
  combination with SLURM)
* implemented global --parallel switch to activate this mode
* added to all representations (parser, Json, prettyPrint)
parent 25ef5cf5
......@@ -25,6 +25,8 @@ import itapasco.controller._
import parser._
import slurm._
import java.nio.file.Path
import scala.concurrent._
import scala.concurrent.ExecutionContext.Implicits.global
object Tapasco {
import org.slf4j.LoggerFactory
......@@ -74,7 +76,11 @@ object Tapasco {
FileAssetManager.start()
cfg.logFile map { logfile: Path => setupLogFileAppender(logfile.toString) }
logger.info(cfg.toString)
runGui(args) || (cfg.jobs map { execute(_) } fold true) (_ && _)
def get(f: Future[Boolean]): Boolean = { Await.ready(f, duration.Duration.Inf); f.value map (_ getOrElse false) getOrElse false }
if (cfg.parallel)
runGui(args) || (cfg.jobs map { j => Future { jobs.executors.execute(j) } } map (get _) fold true) (_ && _)
else
runGui(args) || (cfg.jobs map { jobs.executors.execute(_) } fold true) (_ && _)
} else {
logger.error("invalid arguments: {}", c.left.get.toString)
logger.error(Usage())
......
......@@ -46,6 +46,8 @@ trait Configuration {
def logFile(p: Option[Path]): Configuration
def slurm: Boolean
def slurm(enabled: Boolean): Configuration
def parallel: Boolean
def parallel(enabled: Boolean): Configuration
/** Returns the default output directory for the given kernel and target. */
def outputDir(kernel: Kernel, target: Target): Path =
......
......@@ -42,6 +42,7 @@ private case class ConfigurationImpl (
private val _compositionDir: Path = TapascoCommon.homeDir.resolve("bd"),
private val _logFile: Option[Path] = None,
slurm: Boolean = false,
parallel: Boolean = false,
jobs: Seq[Job] = Seq()
) extends Description(descPath: Path) with Configuration {
def descPath(p: Path): Configuration = this.copy(descPath = p)
......@@ -58,6 +59,7 @@ private case class ConfigurationImpl (
val logFile: Option[Path] = _logFile map (resolve _)
def logFile(op: Option[Path]): Configuration = this.copy(_logFile = op)
def slurm(enabled: Boolean): Configuration = this.copy(slurm = enabled)
def parallel(enabled: Boolean): Configuration = this.copy(parallel = enabled)
def jobs(js: Seq[Job]): Configuration = this.copy(jobs = js)
// these directories must exist
......
......@@ -80,6 +80,7 @@ private object PrettyPrinter {
"ArchDir = " + c.archDir,
"PlatformDir = " + c.platformDir,
"Slurm = " + c.slurm,
"Parallel = " + c.parallel,
"Jobs = " + c.jobs
) mkString NL
......
......@@ -345,6 +345,7 @@ package object json {
(JsPath \ "CompositionDir").readNullable[Path].map (_ getOrElse Paths.get("bd")) ~
(JsPath \ "LogFile").readNullable[Path] ~
(JsPath \ "Slurm").readNullable[Boolean].map (_ getOrElse false) ~
(JsPath \ "Parallel").readNullable[Boolean].map (_ getOrElse false) ~
(JsPath \ "Jobs").read[Seq[Job]]
) (ConfigurationImpl.apply _)
implicit private val configurationWrites: Writes[ConfigurationImpl] = (
......@@ -356,6 +357,7 @@ package object json {
(JsPath \ "CompositionDir").write[Path] ~
(JsPath \ "LogFile").writeNullable[Path] ~
(JsPath \ "Slurm").write[Boolean] ~
(JsPath \ "Parallel").write[Boolean] ~
(JsPath \ "Jobs").write[Seq[Job]]
) (unlift(ConfigurationImpl.unapply _))
implicit object ConfigurationWrites extends Writes[Configuration] {
......
......@@ -43,6 +43,10 @@ object CommandLineParser extends JavaTokenParsers {
p => ("Slurm", p._2 getOrElse true)
}
private[parser] def parallel: Parser[(String, Boolean)] = param("parallel", false) ~ opt(boolLiteral) ^^ {
p => ("Parallel", p._2 getOrElse true)
}
private[parser] def configFile: Parser[String] = param("configfile", false) ~> path
private[parser] def jobsFile: Parser[String] = param("jobsfile", false) ~> path
......
......@@ -24,6 +24,7 @@ private object ConfigurationParser {
configFile ^^ { p => ("ConfigFile", p) } |
entityDir ^^ { p => (p._1.toString, p._2) } |
slurm ^^ { p => (p._1, p._2.toString) } |
parallel ^^ { p => (p._1, p._2.toString) } |
jobsFile ^^ { p => ("JobsFile", p) }
)
......@@ -48,6 +49,7 @@ private object ConfigurationParser {
m.get("Kernels") foreach { d => c = c.kernelDir(Paths.get(d)) }
m.get("Platforms") foreach { d => c = c.platformDir(Paths.get(d)) }
m.get("Slurm") foreach { d => c = c.slurm(d.toBoolean) }
m.get("Parallel") foreach { d => c = c.parallel(d.toBoolean) }
m.get("LogFile") foreach { d => c = c.logFile(Some(Paths.get(d))) }
if (jobs.nonEmpty || m.get("JobsFile").nonEmpty) {
c.jobs(m.get("JobsFile") map (p => readJobsFile(p)) getOrElse jobs)
......
......@@ -16,6 +16,7 @@ object Usage {
--configFile [FILE] Path to Json file with Configuration
--jobsFile [FILE] Path to Json file with Jobs array
--slurm Activate SLURM cluster execution (requires sbatch)
--parallel Execute all jobs in parallel (careful!)
Bulk Import Job: bulkimport [options*]
--csv [FILE] [FILE] should be in comma-separated values (CSV) format
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment