Commit 0bfbe567 authored by Lukas Weber's avatar Lukas Weber Committed by Lukas Sommer

Added a Flag to compose and explore tasks that allows automatic deletion of...

Added a Flag to compose and explore tasks that allows automatic deletion of project folders of failed runs
parent 845b4d2a
......@@ -123,7 +123,7 @@ object Tapasco {
if (! ok) {
logger.error("TaPaSCo finished with errors")
sys.exit(1)
//sys.exit(1)
} else {
logger.info("TaPaSCo finished successfully")
}
......
......@@ -111,6 +111,8 @@ class VivadoComposer()(implicit cfg: Configuration) extends Composer {
/** @inheritdoc */
def clean(bd: Composition, target: Target, f: Double = 0)(implicit cfg: Configuration): Unit = {
cfg.outputDir(bd, target, f).resolve("microarch").toFile.deleteOnExit
cfg.outputDir(bd, target, f).resolve("user_ip").toFile.deleteOnExit
Common.getFiles(cfg.outputDir(bd, target, f).resolve("microarch").toFile).filter(_.isFile).map(_.delete)
Common.getFiles(cfg.outputDir(bd, target, f).resolve("microarch").toFile).filter(_.isDirectory).map(_.deleteOnExit)
Common.getFiles(cfg.outputDir(bd, target, f).resolve("user_ip").toFile).filter(_.isFile).map(_.delete)
......
......@@ -57,7 +57,7 @@ private class ConcreteRun(val no: Int, val element: DesignSpace.Element, val tar
logFile = Some("%s/%s/%s.log".format(exploration.basePath, id, id)),
debugMode = debugMode,
onComplete = res => stop(signal),
deleteOnFinish = true)
deleteOnFail = true)
_task = Some(t)
exploration.publish(Exploration.Events.RunStarted(element, t))
exploration.tasks(t) // start task
......
......@@ -67,7 +67,8 @@ final case class ComposeJob(
private val _architectures: Option[Seq[String]] = None,
private val _platforms: Option[Seq[String]] = None,
features: Option[Seq[Feature]] = None,
debugMode: Option[String] = None) extends Job("compose") {
debugMode: Option[String] = None,
deleteProjects: Option[Boolean] = None) extends Job("compose") {
/** Returns the selected composer tool implementation. */
lazy val implementation: Composer.Implementation = Composer.Implementation(_implementation)
......@@ -143,7 +144,8 @@ final case class DesignSpaceExplorationJob(
private val _architectures: Option[Seq[String]] = None,
private val _platforms: Option[Seq[String]] = None,
features: Option[Seq[Feature]] = None,
debugMode: Option[String] = None) extends Job("dse") {
debugMode: Option[String] = None,
deleteProjects: Option[Boolean] = None) extends Job("dse") {
private final val logger = de.tu_darmstadt.cs.esa.tapasco.Logging.logger(getClass)
// warn if dimensions are completely empty
dimensions match {
......
......@@ -80,7 +80,9 @@ private object Compose extends Executor[ComposeJob] {
target = t,
features = job.features,
debugMode = job.debugMode,
onComplete = _ => signal.release())
onComplete = _ => signal.release(),
deleteOnFail = job.deleteProjects getOrElse(false)
)
composeTasks foreach { tsk.apply _ }
......
......@@ -109,6 +109,7 @@ private object DesignSpaceExploration extends Executor[DesignSpaceExplorationJob
job.features,
None, // logfile
job.debugMode,
onComplete
onComplete,
deleteOnFail = job.deleteProjects getOrElse(false)
)
}
......@@ -116,7 +116,8 @@ package object json {
(JsPath \ "Architectures").readNullable[Seq[String]] ~
(JsPath \ "Platforms").readNullable[Seq[String]] ~
(JsPath \ "Features").readNullable[Seq[Feature]] ~
(JsPath \ "DebugMode").readNullable[String]
(JsPath \ "DebugMode").readNullable[String] ~
(JsPath \ "DeleteProjects").readNullable[Boolean]
) (ComposeJob.apply _)
implicit val composeJobWrites: Writes[ComposeJob] = (
......@@ -127,7 +128,8 @@ package object json {
(JsPath \ "Architectures").writeNullable[Seq[String]] ~
(JsPath \ "Platforms").writeNullable[Seq[String]] ~
(JsPath \ "Features").writeNullable[Seq[Feature]] ~
(JsPath \ "DebugMode").writeNullable[String]
(JsPath \ "DebugMode").writeNullable[String] ~
(JsPath \ "DeleteProjects").writeNullable[Boolean]
) (unlift(ComposeJob.unapply _ andThen (_ map ("Compose" +: _))))
/* ComposeJob @} */
......@@ -143,7 +145,8 @@ package object json {
(JsPath \ "Architectures").readNullable[Seq[String]] ~
(JsPath \ "Platforms").readNullable[Seq[String]] ~
(JsPath \ "Features").readNullable[Seq[Feature]] ~
(JsPath \ "DebugMode").readNullable[String]
(JsPath \ "DebugMode").readNullable[String] ~
(JsPath \ "DeleteÜrpkects").readNullable[Boolean]
) (DesignSpaceExplorationJob.apply _)
implicit val dseJobWrites: Writes[DesignSpaceExplorationJob] = (
......@@ -157,7 +160,8 @@ package object json {
(JsPath \ "Architectures").writeNullable[Seq[String]] ~
(JsPath \ "Platforms").writeNullable[Seq[String]] ~
(JsPath \ "Features").writeNullable[Seq[Feature]] ~
(JsPath \ "DebugMode").writeNullable[String]
(JsPath \ "DebugMode").writeNullable[String] ~
(JsPath \ "DeleteProjects").writeNullable[Boolean]
) (unlift(DesignSpaceExplorationJob.unapply _ andThen (_ map ("DesignSpaceExploration" +: _))))
/* DesignSpaceExplorationJob @} */
......
......@@ -67,6 +67,9 @@ private object CommonArgParsers {
longOption("debugMode", "DebugMode") ~ ws ~/
qstring.opaque("debug mode name, any string") ~ ws
def delProj: Parser[(String, Boolean)] =
(longOption("deleteProjects", "DeleteProjects") ~ ws) map {case s => (s, true)}
def implementation: Parser[(String, String)] =
longOption("implementation", "Implementation") ~
ws ~/
......
......@@ -39,7 +39,7 @@ private object ComposeParser {
private val jobid = identity[ComposeJob] _
private def options: Parser[ComposeJob => ComposeJob] =
(implementation | architectures | platforms | features | debugMode).rep
(implementation | architectures | platforms | features | debugMode | delProj).rep
.map (opts => (opts map (applyOption _) fold jobid) (_ andThen _))
private def applyOption(opt: (String, _)): ComposeJob => ComposeJob =
......@@ -49,6 +49,7 @@ private object ComposeParser {
case ("Platforms", ps: Seq[String @unchecked]) => _.copy(_platforms = Some(ps))
case ("Features", fs: Seq[Feature @unchecked]) => _.copy(features = Some(fs))
case ("DebugMode", m: String) => _.copy(debugMode = Some(m))
case ("DeleteProjects", e: Boolean) => _.copy(deleteProjects = Some(e))
case o => throw new Exception(s"parsed illegal option: $o")
}
}
......@@ -41,7 +41,7 @@ private object DesignSpaceExplorationParser {
))}
private def optionsMap: Parser[Seq[(String, _)]] =
(heuristic | batchSize | basePath | architectures | platforms | features | debugMode).rep
(heuristic | batchSize | basePath | architectures | platforms | features | debugMode | delProj).rep
private val jobid = identity[DesignSpaceExplorationJob] _
......@@ -66,6 +66,7 @@ private object DesignSpaceExplorationParser {
case ("BasePath", p: Path) => _.copy(basePath = Some(p))
case ("Features", fs: Seq[Feature @unchecked]) => _.copy(features = Some(fs))
case ("DebugMode", m: String) => _.copy(debugMode = Some(m))
case ("DeleteProjects", e: Boolean) => _.copy(deleteProjects = Some(e))
case o => throw new Exception(s"parsed illegal option: $o")
}
......
......@@ -42,7 +42,7 @@ class ComposeTask(composition: Composition,
logFile: Option[String] = None,
debugMode: Option[String] = None,
val onComplete: Boolean => Unit,
val deleteOnFinish: Boolean = false)
val deleteOnFail: Boolean = false)
(implicit cfg: Configuration) extends Task with LogTracking {
private[this] implicit val _logger = de.tu_darmstadt.cs.esa.tapasco.Logging.logger(getClass)
private[this] val _slurm = Slurm.enabled
......@@ -59,6 +59,7 @@ class ComposeTask(composition: Composition,
def job: Boolean = if (! _slurm) nodeExecution else slurmExecution
private def nodeExecution: Boolean = {
val appender = LogFileTracker.setupLogFileAppender(_logFile.toString)
val composer = Composer(implementation)(cfg)
_logger.debug("launching compose run for {}@{} [current thread: {}], logfile {}",
......@@ -89,7 +90,7 @@ class ComposeTask(composition: Composition,
LogFileTracker.stopLogFileAppender(appender)
val result = (_composerResult map (_.result) getOrElse false) == ComposeResult.Success
if (result || deleteOnFinish) { composer.clean(composition, target, designFrequency) } // TODO This has to be run, if this compose task is part of a DSE
if (result || deleteOnFail) { composer.clean(composition, target, designFrequency) } // TODO This has to be run, if this compose task is part of a DSE
result
}
......
......@@ -50,7 +50,8 @@ private class DesignSpaceExplorationTask(
features: Option[Seq[Feature]],
logFile: Option[String],
debugMode: Option[String],
val onComplete: Boolean => Unit)
val onComplete: Boolean => Unit,
val deleteOnFail: Boolean = false)
(implicit cfg: Configuration, tsk: Tasks) extends Task with LogTracking with ExplorationTask {
private[this] val _logger = de.tu_darmstadt.cs.esa.tapasco.Logging.logger(getClass)
/** Internal representation of result. **/
......@@ -167,7 +168,8 @@ object DesignSpaceExplorationTask {
features: Option[Seq[Feature]],
logFile: Option[String],
debugMode: Option[String],
onComplete: Boolean => Unit)
onComplete: Boolean => Unit,
deleteOnFail: Boolean = false)
(implicit cfg: Configuration, tsk: Tasks): ExplorationTask = {
new DesignSpaceExplorationTask(
composition,
......@@ -180,7 +182,8 @@ object DesignSpaceExplorationTask {
features,
logFile,
debugMode,
onComplete)
onComplete,
deleteOnFail)
}
// scalastyle:on parameter.number
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment