Closes #111 - Add CLI log tracking switch

* new switch is -v/--verbose with optional MODE as quoted string

src/main/scala/tapasco/activity/composers/VivadoComposer.scala

@@ -26,6 +26,7 @@ import  de.tu_darmstadt.cs.esa.tapasco.Common
 import  de.tu_darmstadt.cs.esa.tapasco.base._
 import  de.tu_darmstadt.cs.esa.tapasco.base.json._
 import  de.tu_darmstadt.cs.esa.tapasco.base.tcl._
+import  de.tu_darmstadt.cs.esa.tapasco.filemgmt.LogTrackingFileWatcher
 import  de.tu_darmstadt.cs.esa.tapasco.util._
 import  de.tu_darmstadt.cs.esa.tapasco.reports._
 import  de.tu_darmstadt.cs.esa.tapasco.dse.Heuristics
@@ -50,6 +51,8 @@ class VivadoComposer()(implicit cfg: Configuration) extends Composer {
     logger.debug("VivadoComposer uses at most {} threads", cfg.maxThreads getOrElse "unlimited")
     // create output struct
     val files = VivadoComposer.Files(bd, target, f, archFeatures ++ platformFeatures)
+    // create log tracker
+    val lt = new LogTrackingFileWatcher(Some(logger))
     // create output directory
     java.nio.file.Files.createDirectories(files.outdir)
     // dump configuration
@@ -63,7 +66,10 @@ class VivadoComposer()(implicit cfg: Configuration) extends Composer {
                 composition  = composition(bd, target))
 
     logger.info("Vivado starting run {}: output in {}", files.runName: Any, files.logFile)
-
+    cfg.verbose foreach { mode =>
+      logger.info("verbose mode {} is active, starting to watch {}", mode: Any, files.logFile)
+      lt += files.logFile
+    }
 
     // Vivado shell command
     val vivadoCmd = Seq("vivado", "-mode", "batch", "-source", files.tclFile.toString,

src/main/scala/tapasco/activity/hls/VivadoHighLevelSynthesis.scala

 package de.tu_darmstadt.cs.esa.tapasco.activity.hls
 import  de.tu_darmstadt.cs.esa.tapasco.base._
+import  de.tu_darmstadt.cs.esa.tapasco.filemgmt.LogTrackingFileWatcher
 import  de.tu_darmstadt.cs.esa.tapasco.util._
 import  de.tu_darmstadt.cs.esa.tapasco.Common
 import  de.tu_darmstadt.cs.esa.tapasco.Logging._
@@ -26,6 +27,8 @@ private object VivadoHighLevelSynthesis extends HighLevelSynthesizer {
   }
 
   def synthesize(k: Kernel, t: Target)(implicit cfg: Configuration): Result = try {
+    // create log tracker
+    val lt = new LogTrackingFileWatcher(Some(logger))
     val outzip = outputZipFile(k, t)
     val script = cfg.outputDir(k, t).resolve("hls").resolve("%s.tcl".format(t.ad.name))
     val logfile = logFile(k, t)
@@ -34,6 +37,10 @@ private object VivadoHighLevelSynthesis extends HighLevelSynthesizer {
       new FileWriter(script.toString).append(makeScript(k, t)).close()  // write Tcl file
       val runName = "'%s' for %s".format(k.name, t.toString)
       logger.info("starting run {}: output in {}", runName: Any, logfile)
+      cfg.verbose foreach { mode =>
+        logger.info("verbose mode {} is active, starting to watch {}", mode: Any, logfile)
+        lt += logfile
+      }
 
       // execute Vivado HLS (max. runtime: 1 day)
       val vivadoRet = InterruptibleProcess(Process(Seq("vivado_hls",

src/main/scala/tapasco/base/Configuration.scala

@@ -52,6 +52,8 @@ trait Configuration {
   def maxThreads(mt: Option[Int]): Configuration
   def dryRun(cfg: Option[Path]): Configuration
   def dryRun: Option[Path]
+  def verbose(mode: Option[String]): Configuration
+  def verbose: Option[String]
 
   /** Returns the default output directory for the given kernel and target. */
   def outputDir(kernel: Kernel, target: Target): Path =

src/main/scala/tapasco/base/ConfigurationImpl.scala

@@ -45,26 +45,28 @@ private case class ConfigurationImpl (
       parallel: Boolean                               = false,
       maxThreads: Option[Int]                         = None,
       dryRun: Option[Path]                            = None,
+      verbose: Option[String]                         = None,
       jobs: Seq[Job]                                  = Seq()
     ) extends Description(descPath: Path) with Configuration {
-  def descPath(p: Path): Configuration = this.copy(descPath = p)
-  val archDir: Path = resolve(_archDir)
-  def archDir(p: Path): Configuration = this.copy(_archDir = p)
-  val compositionDir: Path = resolve(_compositionDir)
-  def compositionDir(p: Path): Configuration = this.copy(_compositionDir = p)
-  val coreDir: Path = resolve(_coreDir)
-  def coreDir(p: Path): Configuration = this.copy(_coreDir = p)
-  val kernelDir: Path = resolve(_kernelDir)
-  def kernelDir(p: Path): Configuration = this.copy(_kernelDir = p)
-  val platformDir: Path = resolve(_platformDir)
-  def platformDir(p: Path): Configuration = this.copy(_platformDir = p)
-  val logFile: Option[Path] = _logFile map (resolve _)
-  def logFile(op: Option[Path]): Configuration = this.copy(_logFile = op)
-  def slurm(enabled: Boolean): Configuration = this.copy(slurm = enabled)
-  def parallel(enabled: Boolean): Configuration = this.copy(parallel = enabled)
-  def maxThreads(mt: Option[Int]): Configuration = this.copy(maxThreads = mt)
-  def dryRun(cfg: Option[Path]): Configuration = this.copy(dryRun = cfg)
-  def jobs(js: Seq[Job]): Configuration = this.copy(jobs = js)
+  def descPath(p: Path): Configuration                = this.copy(descPath = p)
+  val archDir: Path                                   = resolve(_archDir)
+  def archDir(p: Path): Configuration                 = this.copy(_archDir = p)
+  val compositionDir: Path                            = resolve(_compositionDir)
+  def compositionDir(p: Path): Configuration          = this.copy(_compositionDir = p)
+  val coreDir: Path                                   = resolve(_coreDir)
+  def coreDir(p: Path): Configuration                 = this.copy(_coreDir = p)
+  val kernelDir: Path                                 = resolve(_kernelDir)
+  def kernelDir(p: Path): Configuration               = this.copy(_kernelDir = p)
+  val platformDir: Path                               = resolve(_platformDir)
+  def platformDir(p: Path): Configuration             = this.copy(_platformDir = p)
+  val logFile: Option[Path]                           = _logFile map (resolve _)
+  def logFile(op: Option[Path]): Configuration        = this.copy(_logFile = op)
+  def slurm(enabled: Boolean): Configuration          = this.copy(slurm = enabled)
+  def parallel(enabled: Boolean): Configuration       = this.copy(parallel = enabled)
+  def maxThreads(mt: Option[Int]): Configuration      = this.copy(maxThreads = mt)
+  def dryRun(cfg: Option[Path]): Configuration        = this.copy(dryRun = cfg)
+  def verbose(mode: Option[String]): Configuration    = this.copy(verbose = mode)
+  def jobs(js: Seq[Job]): Configuration               = this.copy(jobs = js)
 
   // these directories must exist
   for ((d, n) <- Seq((archDir, "architectures"),

src/main/scala/tapasco/base/PrettyPrinter.scala

@@ -75,6 +75,7 @@ private object PrettyPrinter {
 
   def printConfiguration(c: Configuration): String = List(
       "[Configuration @" + c.descPath + "]",
+      "Verbose = " + c.verbose,
       "KernelDir = " + c.kernelDir,
       "CoreDir = " + c.coreDir,
       "ArchDir = " + c.archDir,

src/main/scala/tapasco/base/json/package.scala

@@ -276,6 +276,7 @@ package object json {
     (JsPath \ "Parallel").readNullable[Boolean].map (_ getOrElse false) ~
     (JsPath \ "MaxThreads").readNullable[Int] ~
     (JsPath \ "DryRun").readNullable[Path] ~
+    (JsPath \ "Verbose").readNullable[String] ~
     (JsPath \ "Jobs").read[Seq[Job]]
   ) (ConfigurationImpl.apply _)
   implicit private val configurationWrites: Writes[ConfigurationImpl] = (
@@ -290,6 +291,7 @@ package object json {
     (JsPath \ "Parallel").write[Boolean] ~
     (JsPath \ "MaxThreads").writeNullable[Int] ~
     (JsPath \ "DryRun").writeNullable[Path].transform((js: JsObject) => js - "DryRun") ~
+    (JsPath \ "Verbose").writeNullable[String] ~
     (JsPath \ "Jobs").write[Seq[Job]]
   ) (unlift(ConfigurationImpl.unapply _))
   implicit object ConfigurationWrites extends Writes[Configuration] {

src/main/scala/tapasco/filemgmt/LogTrackingFileWatcher.scala

+package de.tu_darmstadt.cs.esa.tapasco.filemgmt
+import  de.tu_darmstadt.cs.esa.tapasco.Logging._
+import  de.tu_darmstadt.cs.esa.tapasco.util.Listener
+import  MultiFileWatcher._, Events._
+import  LogTrackingFileWatcher._
+import  java.nio.file.Paths
+
+/** A [[MultiFileWatcher]] which tracks a logfile:
+ *  subsequent logfiles mentioned in the log (matched via regex) are tracked recursively,
+ *  making it easy to follow complex outputs, e.g., from Vivado.
+ *  @param logger Optional logger instance to use.
+ *  @param pollInterval Optional polling interval for files.
+ **/
+class LogTrackingFileWatcher(_logger: Option[Logger] = None, pollInterval: Int = POLL_INTERVAL)
+    extends MultiFileWatcher(POLL_INTERVAL) {
+  private[this] final val logger = _logger getOrElse de.tu_darmstadt.cs.esa.tapasco.Logging.logger(getClass)
+
+  private object listener extends Listener[Event]{
+    def update(e: MultiFileWatcher.Event): Unit = e match {
+      case LinesAdded(src, ls) => ls map { l =>
+        logger.info(l)
+        newFileRegex foreach { rx => rx.findAllMatchIn(l) foreach { m =>
+          Option(m.group(1)) match {
+            case Some(p) if p.trim().nonEmpty =>
+              addPath(Paths.get(p))
+              logger.trace("adding new file: {}", p)
+            case _ => {}
+          }
+        }}
+      }
+    }
+  }
+
+  addListener(listener)
+}
+
+private object LogTrackingFileWatcher {
+  val newFileRegex = Seq(
+    """(?i)output in (\S*)$""".r.unanchored,
+    """(?i)\s*(\S*/synth_1/runme\.log)$""".r.unanchored,
+    """(?i)\s*(\S*/impl_1/runme\.log)$""".r.unanchored)
+}

src/main/scala/tapasco/filemgmt/MultiFileWatcher.scala

@@ -21,25 +21,29 @@ class MultiFileWatcher(pollInterval: Int = MultiFileWatcher.POLL_INTERVAL) exten
    * Add a file to the monitoring.
    * @param p Path to file to be monitored.
    */
-  def +=(p: Path): Unit = open(p)
+  def +=(p: Path) { open(p) }
+  @inline def addPath(p: Path) { this += p }
 
   /**
    * Add a collection of files to the monitoring.
    * @param ps Collection of Paths to files to be monitored.
    */
-  def ++=(ps: Traversable[Path]): Unit = ps foreach (open _)
+  def ++=(ps: Traversable[Path]) { ps foreach (open _) }
+  @inline def addPaths(ps: Traversable[Path]) { this ++= ps }
 
   /**
    * Remove a file from the monitoring.
    * @param p Path to file to be removed.
    */
-  def -=(p: Path): Unit = close(p)
+  def -=(p: Path) { close(p) }
+  @inline def remPath(p: Path) { this -= p }
 
   /**
    * Remove a collection of files from the monitoring.
    * @param ps Collection of Paths to files to be removed.
    */
   def --=(ps: Traversable[Path]): Unit = ps foreach (close _)
+  @inline def remPaths(ps: Traversable[Path]) { this --= ps }
 
   /** Remove and close all files. */
   def closeAll(): Unit = {

src/main/scala/tapasco/parser/GlobalOptions.scala

@@ -13,6 +13,8 @@ private object GlobalOptions {
 
   def validOption: Parser[String] = (
     longShortOption("h", "help")    |
+    longShortOption("v", "verbose") |
+    longShortOption("n", "dryRun")  |
     longOption("archDir")           |
     longOption("platformDir")       |
     longOption("coreDir")           |
@@ -23,7 +25,6 @@ private object GlobalOptions {
     longOption("logFile")           |
     longOption("parallel")          |
     longOption("slurm")             |
-    longShortOption("-n", "DryRUn") |
     longOption("maxThreads") 
   ).opaque("a global option")
 
@@ -31,6 +32,10 @@ private object GlobalOptions {
     (longShortOption("h", "help") | IgnoreCase("help") | IgnoreCase("usage")) ~
     (ws1 ~ string).? map { case (h, topic) => { Usage(topic); ("Help", Usage()) } }
 
+  def verbose: Parser[(String, String)] =
+    (longShortOption("v", "verbose", Some("Verbose")) ~/ (ws1 ~ quotedString.opaque("verbose mode as quoted string")).? ~ ws)
+      .map { case (k, mode) => (k, mode getOrElse "verbose") }
+
   def archDir: Parser[(String, Path)] =
     longOption("archDir", "Architecture") ~/ ws1 ~ path.opaque("root dir of Architectures") ~ ws
   def platformDir: Parser[(String, Path)] =
@@ -71,7 +76,7 @@ private object GlobalOptions {
     longOption("maxThreads", "MaxThreads") ~/ ws ~ posint ~ ws
 
   def globalOptionsSeq: Parser[Seq[(String, _)]] =
-    ws ~ (help | dirs | inputFiles | slurm | parallel | dryRun | maxThreads).rep
+    ws ~ (help | verbose | dirs | inputFiles | slurm | parallel | dryRun | maxThreads).rep
   
   def globalOptions: Parser[Configuration] =
     globalOptionsSeq map (as => mkConfig(as))
@@ -86,13 +91,13 @@ private object GlobalOptions {
         case ("Platform", p: Path)     => mkConfig(as, Some(c getOrElse Configuration() platformDir p))
         case ("Slurm", e: Boolean)     => mkConfig(as, Some(c getOrElse Configuration() slurm e))
         case ("Parallel", e: Boolean)  => mkConfig(as, Some(c getOrElse Configuration() parallel e))
-        //case ("Features", fs)          => TODO
         case ("JobsFile", p: Path)     => mkConfig(as, Some(c getOrElse Configuration() jobs readJobsFile(p)))
         case ("LogFile", p: Path)      => mkConfig(as, Some(c getOrElse Configuration() logFile Some(p)))
         case ("ConfigFile", p: Path)   => mkConfig(as, Some(loadConfigFromFile(p)))
         case ("DryRun", p: Path)       => mkConfig(as, Some(c getOrElse Configuration() dryRun Some(p)))
         case ("MaxThreads", i: Int)    => mkConfig(as, Some(c getOrElse Configuration() maxThreads Some(i)))
-        case _ => c getOrElse Configuration()
+        case ("Verbose", m: String)    => mkConfig(as, Some(c getOrElse Configuration() verbose Some(m)))
+        case _                         => c getOrElse Configuration()
       }
       case x => c getOrElse Configuration()
     }

src/main/scala/tapasco/parser/Usage.scala

@@ -50,6 +50,9 @@ object Usage {
 
   private def globals()    = """
   Global Options:
+    -v | --verbose [MODE]     Verbose mode, log outputs of subprocesses;
+                              Optional MODE is a quoted string selecting the
+                              output mode (default: 'verbose')
     -n | --dryRun FILE        Dry run, do not execute, only dump Json into FILE.
     --archDir PATH            Base directory for architecture descriptions
     --compositionDir PATH     Output base directory for Compose jobs