Added a Flag to compose and explore tasks that allows automatic deletion of...

Added a Flag to compose and explore tasks that allows automatic deletion of project folders of failed runs

src/main/scala/tapasco/Tapasco.scala

@@ -123,7 +123,7 @@ object Tapasco {
 
     if (! ok) {
       logger.error("TaPaSCo finished with errors")
-      sys.exit(1)
+      //sys.exit(1)
     } else {
       logger.info("TaPaSCo finished successfully")
     }

src/main/scala/tapasco/activity/composers/VivadoComposer.scala

@@ -111,6 +111,8 @@ class VivadoComposer()(implicit cfg: Configuration) extends Composer {
 
   /** @inheritdoc */
   def clean(bd: Composition, target: Target, f: Double = 0)(implicit cfg: Configuration): Unit = {
+    cfg.outputDir(bd, target, f).resolve("microarch").toFile.deleteOnExit
+    cfg.outputDir(bd, target, f).resolve("user_ip").toFile.deleteOnExit
     Common.getFiles(cfg.outputDir(bd, target, f).resolve("microarch").toFile).filter(_.isFile).map(_.delete)
     Common.getFiles(cfg.outputDir(bd, target, f).resolve("microarch").toFile).filter(_.isDirectory).map(_.deleteOnExit)
     Common.getFiles(cfg.outputDir(bd, target, f).resolve("user_ip").toFile).filter(_.isFile).map(_.delete)

src/main/scala/tapasco/dse/Run.scala

@@ -57,7 +57,7 @@ private class ConcreteRun(val no: Int, val element: DesignSpace.Element, val tar
       logFile         = Some("%s/%s/%s.log".format(exploration.basePath, id, id)),
       debugMode       = debugMode,
       onComplete      = res => stop(signal),
-      deleteOnFinish  = true)
+      deleteOnFail  = true)
     _task = Some(t)
     exploration.publish(Exploration.Events.RunStarted(element, t))
     exploration.tasks(t) // start task

src/main/scala/tapasco/jobs/Jobs.scala

@@ -67,7 +67,8 @@ final case class ComposeJob(
     private val _architectures: Option[Seq[String]] = None,
     private val _platforms: Option[Seq[String]] = None,
     features: Option[Seq[Feature]] = None,
-    debugMode: Option[String] = None) extends Job("compose") {
+    debugMode: Option[String] = None,
+    deleteProjects: Option[Boolean] = None) extends Job("compose") {
   /** Returns the selected composer tool implementation. */
   lazy val implementation: Composer.Implementation = Composer.Implementation(_implementation)
 
@@ -143,7 +144,8 @@ final case class DesignSpaceExplorationJob(
     private val _architectures: Option[Seq[String]] = None,
     private val _platforms: Option[Seq[String]] = None,
     features: Option[Seq[Feature]] = None,
-    debugMode: Option[String] = None) extends Job("dse") {
+    debugMode: Option[String] = None,
+    deleteProjects: Option[Boolean] = None) extends Job("dse") {
   private final val logger = de.tu_darmstadt.cs.esa.tapasco.Logging.logger(getClass)
   // warn if dimensions are completely empty
   dimensions match {

src/main/scala/tapasco/jobs/executors/Compose.scala

@@ -80,7 +80,9 @@ private object Compose extends Executor[ComposeJob] {
           target = t,
           features = job.features,
           debugMode = job.debugMode,
-          onComplete = _ => signal.release())
+          onComplete = _ => signal.release(),
+          deleteOnFail = job.deleteProjects getOrElse(false)
+      )
 
       composeTasks foreach { tsk.apply _ }
 

src/main/scala/tapasco/jobs/executors/DesignSpaceExploration.scala

@@ -109,6 +109,7 @@ private object DesignSpaceExploration extends Executor[DesignSpaceExplorationJob
       job.features,
       None, // logfile
       job.debugMode,
-      onComplete
+      onComplete,
+      deleteOnFail = job.deleteProjects getOrElse(false)
     )
 }

src/main/scala/tapasco/jobs/json/package.scala

@@ -116,7 +116,8 @@ package object json {
     (JsPath \ "Architectures").readNullable[Seq[String]] ~
     (JsPath \ "Platforms").readNullable[Seq[String]] ~
     (JsPath \ "Features").readNullable[Seq[Feature]] ~
-    (JsPath \ "DebugMode").readNullable[String]
+    (JsPath \ "DebugMode").readNullable[String] ~
+    (JsPath \ "DeleteProjects").readNullable[Boolean]
   ) (ComposeJob.apply _)
 
   implicit val composeJobWrites: Writes[ComposeJob] = (
@@ -127,7 +128,8 @@ package object json {
     (JsPath \ "Architectures").writeNullable[Seq[String]] ~
     (JsPath \ "Platforms").writeNullable[Seq[String]] ~
     (JsPath \ "Features").writeNullable[Seq[Feature]] ~
-    (JsPath \ "DebugMode").writeNullable[String]
+    (JsPath \ "DebugMode").writeNullable[String] ~
+    (JsPath \ "DeleteProjects").writeNullable[Boolean]
   ) (unlift(ComposeJob.unapply _ andThen (_ map ("Compose" +: _))))
   /* ComposeJob @} */
 
@@ -143,7 +145,8 @@ package object json {
     (JsPath \ "Architectures").readNullable[Seq[String]] ~
     (JsPath \ "Platforms").readNullable[Seq[String]] ~
     (JsPath \ "Features").readNullable[Seq[Feature]] ~
-    (JsPath \ "DebugMode").readNullable[String]
+    (JsPath \ "DebugMode").readNullable[String] ~
+    (JsPath \ "DeleteÜrpkects").readNullable[Boolean]
   ) (DesignSpaceExplorationJob.apply _)
 
   implicit val dseJobWrites: Writes[DesignSpaceExplorationJob] = (
@@ -157,7 +160,8 @@ package object json {
     (JsPath \ "Architectures").writeNullable[Seq[String]] ~
     (JsPath \ "Platforms").writeNullable[Seq[String]] ~
     (JsPath \ "Features").writeNullable[Seq[Feature]] ~
-    (JsPath \ "DebugMode").writeNullable[String]
+    (JsPath \ "DebugMode").writeNullable[String] ~
+    (JsPath \ "DeleteProjects").writeNullable[Boolean]
   ) (unlift(DesignSpaceExplorationJob.unapply _ andThen (_ map ("DesignSpaceExploration" +: _))))
   /* DesignSpaceExplorationJob @} */
 

src/main/scala/tapasco/parser/CommonArgParsers.scala

@@ -67,6 +67,9 @@ private object CommonArgParsers {
     longOption("debugMode", "DebugMode") ~ ws ~/
     qstring.opaque("debug mode name, any string") ~ ws
 
+  def delProj: Parser[(String, Boolean)] =
+    (longOption("deleteProjects", "DeleteProjects") ~ ws) map {case s => (s, true)}
+
   def implementation: Parser[(String, String)] =
     longOption("implementation", "Implementation") ~
     ws ~/

src/main/scala/tapasco/parser/ComposeParser.scala

@@ -39,7 +39,7 @@ private object ComposeParser {
   private val jobid = identity[ComposeJob] _
 
   private def options: Parser[ComposeJob => ComposeJob] =
-    (implementation | architectures | platforms | features | debugMode).rep
+    (implementation | architectures | platforms | features | debugMode | delProj).rep
       .map (opts => (opts map (applyOption _) fold jobid) (_ andThen _))
 
   private def applyOption(opt: (String, _)): ComposeJob => ComposeJob =
@@ -49,6 +49,7 @@ private object ComposeParser {
       case ("Platforms", ps: Seq[String @unchecked]) => _.copy(_platforms = Some(ps))
       case ("Features", fs: Seq[Feature @unchecked]) => _.copy(features = Some(fs))
       case ("DebugMode", m: String) => _.copy(debugMode = Some(m))
+      case ("DeleteProjects", e: Boolean) => _.copy(deleteProjects = Some(e))
       case o => throw new Exception(s"parsed illegal option: $o")
     }
 }

src/main/scala/tapasco/parser/DesignSpaceExploration.scala

@@ -41,7 +41,7 @@ private object DesignSpaceExplorationParser {
     ))}
 
   private def optionsMap: Parser[Seq[(String, _)]] =
-    (heuristic | batchSize | basePath | architectures | platforms | features | debugMode).rep
+    (heuristic | batchSize | basePath | architectures | platforms | features | debugMode | delProj).rep
 
   private val jobid = identity[DesignSpaceExplorationJob] _
 
@@ -66,6 +66,7 @@ private object DesignSpaceExplorationParser {
       case ("BasePath", p: Path) => _.copy(basePath = Some(p))
       case ("Features", fs: Seq[Feature @unchecked]) => _.copy(features = Some(fs))
       case ("DebugMode", m: String) => _.copy(debugMode = Some(m))
+      case ("DeleteProjects", e: Boolean) => _.copy(deleteProjects = Some(e))
       case o => throw new Exception(s"parsed illegal option: $o")
     }
 

src/main/scala/tapasco/task/ComposeTask.scala

@@ -42,7 +42,7 @@ class ComposeTask(composition: Composition,
                   logFile: Option[String] = None,
                   debugMode: Option[String] = None,
                   val onComplete: Boolean => Unit,
-                  val deleteOnFinish: Boolean = false)
+                  val deleteOnFail: Boolean = false)
                  (implicit cfg: Configuration) extends Task with LogTracking {
   private[this] implicit val _logger = de.tu_darmstadt.cs.esa.tapasco.Logging.logger(getClass)
   private[this] val _slurm = Slurm.enabled
@@ -59,6 +59,7 @@ class ComposeTask(composition: Composition,
   def job: Boolean = if (! _slurm) nodeExecution else slurmExecution
 
   private def nodeExecution: Boolean = {
+
     val appender = LogFileTracker.setupLogFileAppender(_logFile.toString)
     val composer = Composer(implementation)(cfg)
     _logger.debug("launching compose run for {}@{} [current thread: {}], logfile {}",
@@ -89,7 +90,7 @@ class ComposeTask(composition: Composition,
 
     LogFileTracker.stopLogFileAppender(appender)
     val result = (_composerResult map (_.result) getOrElse false) == ComposeResult.Success
-    if (result || deleteOnFinish) { composer.clean(composition, target, designFrequency) } // TODO This has to be run, if this compose task is part of a DSE
+    if (result || deleteOnFail) { composer.clean(composition, target, designFrequency) } // TODO This has to be run, if this compose task is part of a DSE
     result
   }
 

src/main/scala/tapasco/task/DesignSpaceExplorationTask.scala

@@ -50,7 +50,8 @@ private class DesignSpaceExplorationTask(
     features: Option[Seq[Feature]],
     logFile: Option[String],
     debugMode: Option[String],
-    val onComplete: Boolean => Unit)
+    val onComplete: Boolean => Unit,
+    val deleteOnFail: Boolean = false)
     (implicit cfg: Configuration, tsk: Tasks) extends Task with LogTracking with ExplorationTask {
   private[this] val _logger = de.tu_darmstadt.cs.esa.tapasco.Logging.logger(getClass)
   /** Internal representation of result. **/
@@ -167,7 +168,8 @@ object DesignSpaceExplorationTask {
             features: Option[Seq[Feature]],
             logFile: Option[String],
             debugMode: Option[String],
-            onComplete: Boolean => Unit)
+            onComplete: Boolean => Unit,
+            deleteOnFail: Boolean = false)
            (implicit cfg: Configuration, tsk: Tasks): ExplorationTask = {
     new DesignSpaceExplorationTask(
         composition,
@@ -180,7 +182,8 @@ object DesignSpaceExplorationTask {
         features,
         logFile,
         debugMode,
-        onComplete)
+        onComplete,
+        deleteOnFail)
   }
   // scalastyle:on parameter.number
 }