Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
tapasco
tapasco
Commits
3959bfb3
Commit
3959bfb3
authored
Jul 14, 2017
by
Jens Korinth
Browse files
Closes
#111
- Add CLI log tracking switch
* new switch is -v/--verbose with optional MODE as quoted string
parent
ab773490
Changes
10
Hide whitespace changes
Inline
Side-by-side
src/main/scala/tapasco/activity/composers/VivadoComposer.scala
View file @
3959bfb3
...
...
@@ -26,6 +26,7 @@ import de.tu_darmstadt.cs.esa.tapasco.Common
import
de.tu_darmstadt.cs.esa.tapasco.base._
import
de.tu_darmstadt.cs.esa.tapasco.base.json._
import
de.tu_darmstadt.cs.esa.tapasco.base.tcl._
import
de.tu_darmstadt.cs.esa.tapasco.filemgmt.LogTrackingFileWatcher
import
de.tu_darmstadt.cs.esa.tapasco.util._
import
de.tu_darmstadt.cs.esa.tapasco.reports._
import
de.tu_darmstadt.cs.esa.tapasco.dse.Heuristics
...
...
@@ -50,6 +51,8 @@ class VivadoComposer()(implicit cfg: Configuration) extends Composer {
logger
.
debug
(
"VivadoComposer uses at most {} threads"
,
cfg
.
maxThreads
getOrElse
"unlimited"
)
// create output struct
val
files
=
VivadoComposer
.
Files
(
bd
,
target
,
f
,
archFeatures
++
platformFeatures
)
// create log tracker
val
lt
=
new
LogTrackingFileWatcher
(
Some
(
logger
))
// create output directory
java
.
nio
.
file
.
Files
.
createDirectories
(
files
.
outdir
)
// dump configuration
...
...
@@ -63,7 +66,10 @@ class VivadoComposer()(implicit cfg: Configuration) extends Composer {
composition
=
composition
(
bd
,
target
))
logger
.
info
(
"Vivado starting run {}: output in {}"
,
files
.
runName
:
Any
,
files
.
logFile
)
cfg
.
verbose
foreach
{
mode
=>
logger
.
info
(
"verbose mode {} is active, starting to watch {}"
,
mode
:
Any
,
files
.
logFile
)
lt
+=
files
.
logFile
}
// Vivado shell command
val
vivadoCmd
=
Seq
(
"vivado"
,
"-mode"
,
"batch"
,
"-source"
,
files
.
tclFile
.
toString
,
...
...
src/main/scala/tapasco/activity/hls/VivadoHighLevelSynthesis.scala
View file @
3959bfb3
package
de.tu_darmstadt.cs.esa.tapasco.activity.hls
import
de.tu_darmstadt.cs.esa.tapasco.base._
import
de.tu_darmstadt.cs.esa.tapasco.filemgmt.LogTrackingFileWatcher
import
de.tu_darmstadt.cs.esa.tapasco.util._
import
de.tu_darmstadt.cs.esa.tapasco.Common
import
de.tu_darmstadt.cs.esa.tapasco.Logging._
...
...
@@ -26,6 +27,8 @@ private object VivadoHighLevelSynthesis extends HighLevelSynthesizer {
}
def
synthesize
(
k
:
Kernel
,
t
:
Target
)(
implicit
cfg
:
Configuration
)
:
Result
=
try
{
// create log tracker
val
lt
=
new
LogTrackingFileWatcher
(
Some
(
logger
))
val
outzip
=
outputZipFile
(
k
,
t
)
val
script
=
cfg
.
outputDir
(
k
,
t
).
resolve
(
"hls"
).
resolve
(
"%s.tcl"
.
format
(
t
.
ad
.
name
))
val
logfile
=
logFile
(
k
,
t
)
...
...
@@ -34,6 +37,10 @@ private object VivadoHighLevelSynthesis extends HighLevelSynthesizer {
new
FileWriter
(
script
.
toString
).
append
(
makeScript
(
k
,
t
)).
close
()
// write Tcl file
val
runName
=
"'%s' for %s"
.
format
(
k
.
name
,
t
.
toString
)
logger
.
info
(
"starting run {}: output in {}"
,
runName
:
Any
,
logfile
)
cfg
.
verbose
foreach
{
mode
=>
logger
.
info
(
"verbose mode {} is active, starting to watch {}"
,
mode
:
Any
,
logfile
)
lt
+=
logfile
}
// execute Vivado HLS (max. runtime: 1 day)
val
vivadoRet
=
InterruptibleProcess
(
Process
(
Seq
(
"vivado_hls"
,
...
...
src/main/scala/tapasco/base/Configuration.scala
View file @
3959bfb3
...
...
@@ -52,6 +52,8 @@ trait Configuration {
def
maxThreads
(
mt
:
Option
[
Int
])
:
Configuration
def
dryRun
(
cfg
:
Option
[
Path
])
:
Configuration
def
dryRun
:
Option
[
Path
]
def
verbose
(
mode
:
Option
[
String
])
:
Configuration
def
verbose
:
Option
[
String
]
/** Returns the default output directory for the given kernel and target. */
def
outputDir
(
kernel
:
Kernel
,
target
:
Target
)
:
Path
=
...
...
src/main/scala/tapasco/base/ConfigurationImpl.scala
View file @
3959bfb3
...
...
@@ -45,26 +45,28 @@ private case class ConfigurationImpl (
parallel
:
Boolean
=
false
,
maxThreads
:
Option
[
Int
]
=
None
,
dryRun
:
Option
[
Path
]
=
None
,
verbose
:
Option
[
String
]
=
None
,
jobs
:
Seq
[
Job
]
=
Seq
()
)
extends
Description
(
descPath
:
Path
)
with
Configuration
{
def
descPath
(
p
:
Path
)
:
Configuration
=
this
.
copy
(
descPath
=
p
)
val
archDir
:
Path
=
resolve
(
_archDir
)
def
archDir
(
p
:
Path
)
:
Configuration
=
this
.
copy
(
_archDir
=
p
)
val
compositionDir
:
Path
=
resolve
(
_compositionDir
)
def
compositionDir
(
p
:
Path
)
:
Configuration
=
this
.
copy
(
_compositionDir
=
p
)
val
coreDir
:
Path
=
resolve
(
_coreDir
)
def
coreDir
(
p
:
Path
)
:
Configuration
=
this
.
copy
(
_coreDir
=
p
)
val
kernelDir
:
Path
=
resolve
(
_kernelDir
)
def
kernelDir
(
p
:
Path
)
:
Configuration
=
this
.
copy
(
_kernelDir
=
p
)
val
platformDir
:
Path
=
resolve
(
_platformDir
)
def
platformDir
(
p
:
Path
)
:
Configuration
=
this
.
copy
(
_platformDir
=
p
)
val
logFile
:
Option
[
Path
]
=
_logFile
map
(
resolve
_
)
def
logFile
(
op
:
Option
[
Path
])
:
Configuration
=
this
.
copy
(
_logFile
=
op
)
def
slurm
(
enabled
:
Boolean
)
:
Configuration
=
this
.
copy
(
slurm
=
enabled
)
def
parallel
(
enabled
:
Boolean
)
:
Configuration
=
this
.
copy
(
parallel
=
enabled
)
def
maxThreads
(
mt
:
Option
[
Int
])
:
Configuration
=
this
.
copy
(
maxThreads
=
mt
)
def
dryRun
(
cfg
:
Option
[
Path
])
:
Configuration
=
this
.
copy
(
dryRun
=
cfg
)
def
jobs
(
js
:
Seq
[
Job
])
:
Configuration
=
this
.
copy
(
jobs
=
js
)
def
descPath
(
p
:
Path
)
:
Configuration
=
this
.
copy
(
descPath
=
p
)
val
archDir
:
Path
=
resolve
(
_archDir
)
def
archDir
(
p
:
Path
)
:
Configuration
=
this
.
copy
(
_archDir
=
p
)
val
compositionDir
:
Path
=
resolve
(
_compositionDir
)
def
compositionDir
(
p
:
Path
)
:
Configuration
=
this
.
copy
(
_compositionDir
=
p
)
val
coreDir
:
Path
=
resolve
(
_coreDir
)
def
coreDir
(
p
:
Path
)
:
Configuration
=
this
.
copy
(
_coreDir
=
p
)
val
kernelDir
:
Path
=
resolve
(
_kernelDir
)
def
kernelDir
(
p
:
Path
)
:
Configuration
=
this
.
copy
(
_kernelDir
=
p
)
val
platformDir
:
Path
=
resolve
(
_platformDir
)
def
platformDir
(
p
:
Path
)
:
Configuration
=
this
.
copy
(
_platformDir
=
p
)
val
logFile
:
Option
[
Path
]
=
_logFile
map
(
resolve
_
)
def
logFile
(
op
:
Option
[
Path
])
:
Configuration
=
this
.
copy
(
_logFile
=
op
)
def
slurm
(
enabled
:
Boolean
)
:
Configuration
=
this
.
copy
(
slurm
=
enabled
)
def
parallel
(
enabled
:
Boolean
)
:
Configuration
=
this
.
copy
(
parallel
=
enabled
)
def
maxThreads
(
mt
:
Option
[
Int
])
:
Configuration
=
this
.
copy
(
maxThreads
=
mt
)
def
dryRun
(
cfg
:
Option
[
Path
])
:
Configuration
=
this
.
copy
(
dryRun
=
cfg
)
def
verbose
(
mode
:
Option
[
String
])
:
Configuration
=
this
.
copy
(
verbose
=
mode
)
def
jobs
(
js
:
Seq
[
Job
])
:
Configuration
=
this
.
copy
(
jobs
=
js
)
// these directories must exist
for
((
d
,
n
)
<-
Seq
((
archDir
,
"architectures"
),
...
...
src/main/scala/tapasco/base/PrettyPrinter.scala
View file @
3959bfb3
...
...
@@ -75,6 +75,7 @@ private object PrettyPrinter {
def
printConfiguration
(
c
:
Configuration
)
:
String
=
List
(
"[Configuration @"
+
c
.
descPath
+
"]"
,
"Verbose = "
+
c
.
verbose
,
"KernelDir = "
+
c
.
kernelDir
,
"CoreDir = "
+
c
.
coreDir
,
"ArchDir = "
+
c
.
archDir
,
...
...
src/main/scala/tapasco/base/json/package.scala
View file @
3959bfb3
...
...
@@ -276,6 +276,7 @@ package object json {
(
JsPath
\
"Parallel"
).
readNullable
[
Boolean
].
map
(
_
getOrElse
false
)
~
(
JsPath
\
"MaxThreads"
).
readNullable
[
Int
]
~
(
JsPath
\
"DryRun"
).
readNullable
[
Path
]
~
(
JsPath
\
"Verbose"
).
readNullable
[
String
]
~
(
JsPath
\
"Jobs"
).
read
[
Seq
[
Job
]]
)
(
ConfigurationImpl
.
apply
_
)
implicit
private
val
configurationWrites
:
Writes
[
ConfigurationImpl
]
=
(
...
...
@@ -290,6 +291,7 @@ package object json {
(
JsPath
\
"Parallel"
).
write
[
Boolean
]
~
(
JsPath
\
"MaxThreads"
).
writeNullable
[
Int
]
~
(
JsPath
\
"DryRun"
).
writeNullable
[
Path
].
transform
((
js
:
JsObject
)
=>
js
-
"DryRun"
)
~
(
JsPath
\
"Verbose"
).
writeNullable
[
String
]
~
(
JsPath
\
"Jobs"
).
write
[
Seq
[
Job
]]
)
(
unlift
(
ConfigurationImpl
.
unapply
_
))
implicit
object
ConfigurationWrites
extends
Writes
[
Configuration
]
{
...
...
src/main/scala/tapasco/filemgmt/LogTrackingFileWatcher.scala
0 → 100644
View file @
3959bfb3
package
de.tu_darmstadt.cs.esa.tapasco.filemgmt
import
de.tu_darmstadt.cs.esa.tapasco.Logging._
import
de.tu_darmstadt.cs.esa.tapasco.util.Listener
import
MultiFileWatcher._
,
Events
.
_
import
LogTrackingFileWatcher._
import
java.nio.file.Paths
/** A [[MultiFileWatcher]] which tracks a logfile:
* subsequent logfiles mentioned in the log (matched via regex) are tracked recursively,
* making it easy to follow complex outputs, e.g., from Vivado.
* @param logger Optional logger instance to use.
* @param pollInterval Optional polling interval for files.
**/
class
LogTrackingFileWatcher
(
_logger
:
Option
[
Logger
]
=
None
,
pollInterval
:
Int
=
POLL_INTERVAL
)
extends
MultiFileWatcher
(
POLL_INTERVAL
)
{
private
[
this
]
final
val
logger
=
_logger
getOrElse
de
.
tu_darmstadt
.
cs
.
esa
.
tapasco
.
Logging
.
logger
(
getClass
)
private
object
listener
extends
Listener
[
Event
]{
def
update
(
e
:
MultiFileWatcher.Event
)
:
Unit
=
e
match
{
case
LinesAdded
(
src
,
ls
)
=>
ls
map
{
l
=>
logger
.
info
(
l
)
newFileRegex
foreach
{
rx
=>
rx
.
findAllMatchIn
(
l
)
foreach
{
m
=>
Option
(
m
.
group
(
1
))
match
{
case
Some
(
p
)
if
p
.
trim
().
nonEmpty
=>
addPath
(
Paths
.
get
(
p
))
logger
.
trace
(
"adding new file: {}"
,
p
)
case
_
=>
{}
}
}}
}
}
}
addListener
(
listener
)
}
private
object
LogTrackingFileWatcher
{
val
newFileRegex
=
Seq
(
"""(?i)output in (\S*)$"""
.
r
.
unanchored
,
"""(?i)\s*(\S*/synth_1/runme\.log)$"""
.
r
.
unanchored
,
"""(?i)\s*(\S*/impl_1/runme\.log)$"""
.
r
.
unanchored
)
}
src/main/scala/tapasco/filemgmt/MultiFileWatcher.scala
View file @
3959bfb3
...
...
@@ -21,25 +21,29 @@ class MultiFileWatcher(pollInterval: Int = MultiFileWatcher.POLL_INTERVAL) exten
* Add a file to the monitoring.
* @param p Path to file to be monitored.
*/
def
+=
(
p
:
Path
)
:
Unit
=
open
(
p
)
def
+=
(
p
:
Path
)
{
open
(
p
)
}
@inline
def
addPath
(
p
:
Path
)
{
this
+=
p
}
/**
* Add a collection of files to the monitoring.
* @param ps Collection of Paths to files to be monitored.
*/
def
++=
(
ps
:
Traversable
[
Path
])
:
Unit
=
ps
foreach
(
open
_
)
def
++=
(
ps
:
Traversable
[
Path
])
{
ps
foreach
(
open
_
)
}
@inline
def
addPaths
(
ps
:
Traversable
[
Path
])
{
this
++=
ps
}
/**
* Remove a file from the monitoring.
* @param p Path to file to be removed.
*/
def
-=
(
p
:
Path
)
:
Unit
=
close
(
p
)
def
-=
(
p
:
Path
)
{
close
(
p
)
}
@inline
def
remPath
(
p
:
Path
)
{
this
-=
p
}
/**
* Remove a collection of files from the monitoring.
* @param ps Collection of Paths to files to be removed.
*/
def
--=
(
ps
:
Traversable
[
Path
])
:
Unit
=
ps
foreach
(
close
_
)
@inline
def
remPaths
(
ps
:
Traversable
[
Path
])
{
this
--=
ps
}
/** Remove and close all files. */
def
closeAll
()
:
Unit
=
{
...
...
src/main/scala/tapasco/parser/GlobalOptions.scala
View file @
3959bfb3
...
...
@@ -13,6 +13,8 @@ private object GlobalOptions {
def
validOption
:
Parser
[
String
]
=
(
longShortOption
(
"h"
,
"help"
)
|
longShortOption
(
"v"
,
"verbose"
)
|
longShortOption
(
"n"
,
"dryRun"
)
|
longOption
(
"archDir"
)
|
longOption
(
"platformDir"
)
|
longOption
(
"coreDir"
)
|
...
...
@@ -23,7 +25,6 @@ private object GlobalOptions {
longOption
(
"logFile"
)
|
longOption
(
"parallel"
)
|
longOption
(
"slurm"
)
|
longShortOption
(
"-n"
,
"DryRUn"
)
|
longOption
(
"maxThreads"
)
).
opaque
(
"a global option"
)
...
...
@@ -31,6 +32,10 @@ private object GlobalOptions {
(
longShortOption
(
"h"
,
"help"
)
|
IgnoreCase
(
"help"
)
|
IgnoreCase
(
"usage"
))
~
(
ws1
~
string
).?
map
{
case
(
h
,
topic
)
=>
{
Usage
(
topic
);
(
"Help"
,
Usage
())
}
}
def
verbose
:
Parser
[(
String
,
String
)]
=
(
longShortOption
(
"v"
,
"verbose"
,
Some
(
"Verbose"
))
~/
(
ws1
~
quotedString
.
opaque
(
"verbose mode as quoted string"
)).?
~
ws
)
.
map
{
case
(
k
,
mode
)
=>
(
k
,
mode
getOrElse
"verbose"
)
}
def
archDir
:
Parser
[(
String
,
Path
)]
=
longOption
(
"archDir"
,
"Architecture"
)
~/
ws1
~
path
.
opaque
(
"root dir of Architectures"
)
~
ws
def
platformDir
:
Parser
[(
String
,
Path
)]
=
...
...
@@ -71,7 +76,7 @@ private object GlobalOptions {
longOption
(
"maxThreads"
,
"MaxThreads"
)
~/
ws
~
posint
~
ws
def
globalOptionsSeq
:
Parser
[
Seq
[(
String
,
_
)]]
=
ws
~
(
help
|
dirs
|
inputFiles
|
slurm
|
parallel
|
dryRun
|
maxThreads
).
rep
ws
~
(
help
|
verbose
|
dirs
|
inputFiles
|
slurm
|
parallel
|
dryRun
|
maxThreads
).
rep
def
globalOptions
:
Parser
[
Configuration
]
=
globalOptionsSeq
map
(
as
=>
mkConfig
(
as
))
...
...
@@ -86,13 +91,13 @@ private object GlobalOptions {
case
(
"Platform"
,
p
:
Path
)
=>
mkConfig
(
as
,
Some
(
c
getOrElse
Configuration
()
platformDir
p
))
case
(
"Slurm"
,
e
:
Boolean
)
=>
mkConfig
(
as
,
Some
(
c
getOrElse
Configuration
()
slurm
e
))
case
(
"Parallel"
,
e
:
Boolean
)
=>
mkConfig
(
as
,
Some
(
c
getOrElse
Configuration
()
parallel
e
))
//case ("Features", fs) => TODO
case
(
"JobsFile"
,
p
:
Path
)
=>
mkConfig
(
as
,
Some
(
c
getOrElse
Configuration
()
jobs
readJobsFile
(
p
)))
case
(
"LogFile"
,
p
:
Path
)
=>
mkConfig
(
as
,
Some
(
c
getOrElse
Configuration
()
logFile
Some
(
p
)))
case
(
"ConfigFile"
,
p
:
Path
)
=>
mkConfig
(
as
,
Some
(
loadConfigFromFile
(
p
)))
case
(
"DryRun"
,
p
:
Path
)
=>
mkConfig
(
as
,
Some
(
c
getOrElse
Configuration
()
dryRun
Some
(
p
)))
case
(
"MaxThreads"
,
i
:
Int
)
=>
mkConfig
(
as
,
Some
(
c
getOrElse
Configuration
()
maxThreads
Some
(
i
)))
case
_
=>
c
getOrElse
Configuration
()
case
(
"Verbose"
,
m
:
String
)
=>
mkConfig
(
as
,
Some
(
c
getOrElse
Configuration
()
verbose
Some
(
m
)))
case
_
=>
c
getOrElse
Configuration
()
}
case
x
=>
c
getOrElse
Configuration
()
}
...
...
src/main/scala/tapasco/parser/Usage.scala
View file @
3959bfb3
...
...
@@ -50,6 +50,9 @@ object Usage {
private
def
globals
()
=
"""
Global Options:
-v | --verbose [MODE] Verbose mode, log outputs of subprocesses;
Optional MODE is a quoted string selecting the
output mode (default: 'verbose')
-n | --dryRun FILE Dry run, do not execute, only dump Json into FILE.
--archDir PATH Base directory for architecture descriptions
--compositionDir PATH Output base directory for Compose jobs
...
...
Jens Korinth
@jk
mentioned in commit
a831b501
·
Jul 14, 2017
mentioned in commit
a831b501
mentioned in commit a831b5015df6bd8f8d23ddd334a3b1159392de92
Toggle commit list
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment