Commit 3959bfb3 authored by Jens Korinth's avatar Jens Korinth

Closes #111 - Add CLI log tracking switch

* new switch is -v/--verbose with optional MODE as quoted string
parent ab773490
......@@ -26,6 +26,7 @@ import de.tu_darmstadt.cs.esa.tapasco.Common
import de.tu_darmstadt.cs.esa.tapasco.base._
import de.tu_darmstadt.cs.esa.tapasco.base.json._
import de.tu_darmstadt.cs.esa.tapasco.base.tcl._
import de.tu_darmstadt.cs.esa.tapasco.filemgmt.LogTrackingFileWatcher
import de.tu_darmstadt.cs.esa.tapasco.util._
import de.tu_darmstadt.cs.esa.tapasco.reports._
import de.tu_darmstadt.cs.esa.tapasco.dse.Heuristics
......@@ -50,6 +51,8 @@ class VivadoComposer()(implicit cfg: Configuration) extends Composer {
logger.debug("VivadoComposer uses at most {} threads", cfg.maxThreads getOrElse "unlimited")
// create output struct
val files = VivadoComposer.Files(bd, target, f, archFeatures ++ platformFeatures)
// create log tracker
val lt = new LogTrackingFileWatcher(Some(logger))
// create output directory
java.nio.file.Files.createDirectories(files.outdir)
// dump configuration
......@@ -63,7 +66,10 @@ class VivadoComposer()(implicit cfg: Configuration) extends Composer {
composition = composition(bd, target))
logger.info("Vivado starting run {}: output in {}", files.runName: Any, files.logFile)
cfg.verbose foreach { mode =>
logger.info("verbose mode {} is active, starting to watch {}", mode: Any, files.logFile)
lt += files.logFile
}
// Vivado shell command
val vivadoCmd = Seq("vivado", "-mode", "batch", "-source", files.tclFile.toString,
......
package de.tu_darmstadt.cs.esa.tapasco.activity.hls
import de.tu_darmstadt.cs.esa.tapasco.base._
import de.tu_darmstadt.cs.esa.tapasco.filemgmt.LogTrackingFileWatcher
import de.tu_darmstadt.cs.esa.tapasco.util._
import de.tu_darmstadt.cs.esa.tapasco.Common
import de.tu_darmstadt.cs.esa.tapasco.Logging._
......@@ -26,6 +27,8 @@ private object VivadoHighLevelSynthesis extends HighLevelSynthesizer {
}
def synthesize(k: Kernel, t: Target)(implicit cfg: Configuration): Result = try {
// create log tracker
val lt = new LogTrackingFileWatcher(Some(logger))
val outzip = outputZipFile(k, t)
val script = cfg.outputDir(k, t).resolve("hls").resolve("%s.tcl".format(t.ad.name))
val logfile = logFile(k, t)
......@@ -34,6 +37,10 @@ private object VivadoHighLevelSynthesis extends HighLevelSynthesizer {
new FileWriter(script.toString).append(makeScript(k, t)).close() // write Tcl file
val runName = "'%s' for %s".format(k.name, t.toString)
logger.info("starting run {}: output in {}", runName: Any, logfile)
cfg.verbose foreach { mode =>
logger.info("verbose mode {} is active, starting to watch {}", mode: Any, logfile)
lt += logfile
}
// execute Vivado HLS (max. runtime: 1 day)
val vivadoRet = InterruptibleProcess(Process(Seq("vivado_hls",
......
......@@ -52,6 +52,8 @@ trait Configuration {
def maxThreads(mt: Option[Int]): Configuration
def dryRun(cfg: Option[Path]): Configuration
def dryRun: Option[Path]
def verbose(mode: Option[String]): Configuration
def verbose: Option[String]
/** Returns the default output directory for the given kernel and target. */
def outputDir(kernel: Kernel, target: Target): Path =
......
......@@ -45,26 +45,28 @@ private case class ConfigurationImpl (
parallel: Boolean = false,
maxThreads: Option[Int] = None,
dryRun: Option[Path] = None,
verbose: Option[String] = None,
jobs: Seq[Job] = Seq()
) extends Description(descPath: Path) with Configuration {
def descPath(p: Path): Configuration = this.copy(descPath = p)
val archDir: Path = resolve(_archDir)
def archDir(p: Path): Configuration = this.copy(_archDir = p)
val compositionDir: Path = resolve(_compositionDir)
def compositionDir(p: Path): Configuration = this.copy(_compositionDir = p)
val coreDir: Path = resolve(_coreDir)
def coreDir(p: Path): Configuration = this.copy(_coreDir = p)
val kernelDir: Path = resolve(_kernelDir)
def kernelDir(p: Path): Configuration = this.copy(_kernelDir = p)
val platformDir: Path = resolve(_platformDir)
def platformDir(p: Path): Configuration = this.copy(_platformDir = p)
val logFile: Option[Path] = _logFile map (resolve _)
def logFile(op: Option[Path]): Configuration = this.copy(_logFile = op)
def slurm(enabled: Boolean): Configuration = this.copy(slurm = enabled)
def parallel(enabled: Boolean): Configuration = this.copy(parallel = enabled)
def maxThreads(mt: Option[Int]): Configuration = this.copy(maxThreads = mt)
def dryRun(cfg: Option[Path]): Configuration = this.copy(dryRun = cfg)
def jobs(js: Seq[Job]): Configuration = this.copy(jobs = js)
def descPath(p: Path): Configuration = this.copy(descPath = p)
val archDir: Path = resolve(_archDir)
def archDir(p: Path): Configuration = this.copy(_archDir = p)
val compositionDir: Path = resolve(_compositionDir)
def compositionDir(p: Path): Configuration = this.copy(_compositionDir = p)
val coreDir: Path = resolve(_coreDir)
def coreDir(p: Path): Configuration = this.copy(_coreDir = p)
val kernelDir: Path = resolve(_kernelDir)
def kernelDir(p: Path): Configuration = this.copy(_kernelDir = p)
val platformDir: Path = resolve(_platformDir)
def platformDir(p: Path): Configuration = this.copy(_platformDir = p)
val logFile: Option[Path] = _logFile map (resolve _)
def logFile(op: Option[Path]): Configuration = this.copy(_logFile = op)
def slurm(enabled: Boolean): Configuration = this.copy(slurm = enabled)
def parallel(enabled: Boolean): Configuration = this.copy(parallel = enabled)
def maxThreads(mt: Option[Int]): Configuration = this.copy(maxThreads = mt)
def dryRun(cfg: Option[Path]): Configuration = this.copy(dryRun = cfg)
def verbose(mode: Option[String]): Configuration = this.copy(verbose = mode)
def jobs(js: Seq[Job]): Configuration = this.copy(jobs = js)
// these directories must exist
for ((d, n) <- Seq((archDir, "architectures"),
......
......@@ -75,6 +75,7 @@ private object PrettyPrinter {
def printConfiguration(c: Configuration): String = List(
"[Configuration @" + c.descPath + "]",
"Verbose = " + c.verbose,
"KernelDir = " + c.kernelDir,
"CoreDir = " + c.coreDir,
"ArchDir = " + c.archDir,
......
......@@ -276,6 +276,7 @@ package object json {
(JsPath \ "Parallel").readNullable[Boolean].map (_ getOrElse false) ~
(JsPath \ "MaxThreads").readNullable[Int] ~
(JsPath \ "DryRun").readNullable[Path] ~
(JsPath \ "Verbose").readNullable[String] ~
(JsPath \ "Jobs").read[Seq[Job]]
) (ConfigurationImpl.apply _)
implicit private val configurationWrites: Writes[ConfigurationImpl] = (
......@@ -290,6 +291,7 @@ package object json {
(JsPath \ "Parallel").write[Boolean] ~
(JsPath \ "MaxThreads").writeNullable[Int] ~
(JsPath \ "DryRun").writeNullable[Path].transform((js: JsObject) => js - "DryRun") ~
(JsPath \ "Verbose").writeNullable[String] ~
(JsPath \ "Jobs").write[Seq[Job]]
) (unlift(ConfigurationImpl.unapply _))
implicit object ConfigurationWrites extends Writes[Configuration] {
......
package de.tu_darmstadt.cs.esa.tapasco.filemgmt
import de.tu_darmstadt.cs.esa.tapasco.Logging._
import de.tu_darmstadt.cs.esa.tapasco.util.Listener
import MultiFileWatcher._, Events._
import LogTrackingFileWatcher._
import java.nio.file.Paths
/** A [[MultiFileWatcher]] which tracks a logfile:
* subsequent logfiles mentioned in the log (matched via regex) are tracked recursively,
* making it easy to follow complex outputs, e.g., from Vivado.
* @param logger Optional logger instance to use.
* @param pollInterval Optional polling interval for files.
**/
class LogTrackingFileWatcher(_logger: Option[Logger] = None, pollInterval: Int = POLL_INTERVAL)
extends MultiFileWatcher(POLL_INTERVAL) {
private[this] final val logger = _logger getOrElse de.tu_darmstadt.cs.esa.tapasco.Logging.logger(getClass)
private object listener extends Listener[Event]{
def update(e: MultiFileWatcher.Event): Unit = e match {
case LinesAdded(src, ls) => ls map { l =>
logger.info(l)
newFileRegex foreach { rx => rx.findAllMatchIn(l) foreach { m =>
Option(m.group(1)) match {
case Some(p) if p.trim().nonEmpty =>
addPath(Paths.get(p))
logger.trace("adding new file: {}", p)
case _ => {}
}
}}
}
}
}
addListener(listener)
}
private object LogTrackingFileWatcher {
val newFileRegex = Seq(
"""(?i)output in (\S*)$""".r.unanchored,
"""(?i)\s*(\S*/synth_1/runme\.log)$""".r.unanchored,
"""(?i)\s*(\S*/impl_1/runme\.log)$""".r.unanchored)
}
......@@ -21,25 +21,29 @@ class MultiFileWatcher(pollInterval: Int = MultiFileWatcher.POLL_INTERVAL) exten
* Add a file to the monitoring.
* @param p Path to file to be monitored.
*/
def +=(p: Path): Unit = open(p)
def +=(p: Path) { open(p) }
@inline def addPath(p: Path) { this += p }
/**
* Add a collection of files to the monitoring.
* @param ps Collection of Paths to files to be monitored.
*/
def ++=(ps: Traversable[Path]): Unit = ps foreach (open _)
def ++=(ps: Traversable[Path]) { ps foreach (open _) }
@inline def addPaths(ps: Traversable[Path]) { this ++= ps }
/**
* Remove a file from the monitoring.
* @param p Path to file to be removed.
*/
def -=(p: Path): Unit = close(p)
def -=(p: Path) { close(p) }
@inline def remPath(p: Path) { this -= p }
/**
* Remove a collection of files from the monitoring.
* @param ps Collection of Paths to files to be removed.
*/
def --=(ps: Traversable[Path]): Unit = ps foreach (close _)
@inline def remPaths(ps: Traversable[Path]) { this --= ps }
/** Remove and close all files. */
def closeAll(): Unit = {
......
......@@ -13,6 +13,8 @@ private object GlobalOptions {
def validOption: Parser[String] = (
longShortOption("h", "help") |
longShortOption("v", "verbose") |
longShortOption("n", "dryRun") |
longOption("archDir") |
longOption("platformDir") |
longOption("coreDir") |
......@@ -23,7 +25,6 @@ private object GlobalOptions {
longOption("logFile") |
longOption("parallel") |
longOption("slurm") |
longShortOption("-n", "DryRUn") |
longOption("maxThreads")
).opaque("a global option")
......@@ -31,6 +32,10 @@ private object GlobalOptions {
(longShortOption("h", "help") | IgnoreCase("help") | IgnoreCase("usage")) ~
(ws1 ~ string).? map { case (h, topic) => { Usage(topic); ("Help", Usage()) } }
def verbose: Parser[(String, String)] =
(longShortOption("v", "verbose", Some("Verbose")) ~/ (ws1 ~ quotedString.opaque("verbose mode as quoted string")).? ~ ws)
.map { case (k, mode) => (k, mode getOrElse "verbose") }
def archDir: Parser[(String, Path)] =
longOption("archDir", "Architecture") ~/ ws1 ~ path.opaque("root dir of Architectures") ~ ws
def platformDir: Parser[(String, Path)] =
......@@ -71,7 +76,7 @@ private object GlobalOptions {
longOption("maxThreads", "MaxThreads") ~/ ws ~ posint ~ ws
def globalOptionsSeq: Parser[Seq[(String, _)]] =
ws ~ (help | dirs | inputFiles | slurm | parallel | dryRun | maxThreads).rep
ws ~ (help | verbose | dirs | inputFiles | slurm | parallel | dryRun | maxThreads).rep
def globalOptions: Parser[Configuration] =
globalOptionsSeq map (as => mkConfig(as))
......@@ -86,13 +91,13 @@ private object GlobalOptions {
case ("Platform", p: Path) => mkConfig(as, Some(c getOrElse Configuration() platformDir p))
case ("Slurm", e: Boolean) => mkConfig(as, Some(c getOrElse Configuration() slurm e))
case ("Parallel", e: Boolean) => mkConfig(as, Some(c getOrElse Configuration() parallel e))
//case ("Features", fs) => TODO
case ("JobsFile", p: Path) => mkConfig(as, Some(c getOrElse Configuration() jobs readJobsFile(p)))
case ("LogFile", p: Path) => mkConfig(as, Some(c getOrElse Configuration() logFile Some(p)))
case ("ConfigFile", p: Path) => mkConfig(as, Some(loadConfigFromFile(p)))
case ("DryRun", p: Path) => mkConfig(as, Some(c getOrElse Configuration() dryRun Some(p)))
case ("MaxThreads", i: Int) => mkConfig(as, Some(c getOrElse Configuration() maxThreads Some(i)))
case _ => c getOrElse Configuration()
case ("Verbose", m: String) => mkConfig(as, Some(c getOrElse Configuration() verbose Some(m)))
case _ => c getOrElse Configuration()
}
case x => c getOrElse Configuration()
}
......
......@@ -50,6 +50,9 @@ object Usage {
private def globals() = """
Global Options:
-v | --verbose [MODE] Verbose mode, log outputs of subprocesses;
Optional MODE is a quoted string selecting the
output mode (default: 'verbose')
-n | --dryRun FILE Dry run, do not execute, only dump Json into FILE.
--archDir PATH Base directory for architecture descriptions
--compositionDir PATH Output base directory for Compose jobs
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment