Skip to content

GitLab

Commit 53d35035 authored by Jens Korinth's avatar Jens Korinth
Browse files

Benchmarks: Add new job throughput measure 

* added Json serdes
* fixed unit test cases
parent 9ff38dee
Hide whitespace changes
Inline Side-by-side
json-examples/correct-benchmark.json
View file @ 53d35035
{
"Timestamp" : "2017-05-30 13:08:55",
"Host" : {
"Machine" : "x86_64",
"Node" : "mountdoom",
"Machine" : "armv7l",
"Node" : "pynq",
"Operating System" : "Linux",
"Release" : "3.19.8-100.fc20.x86_64",
"Version" : "#1 SMP Tue May 12 17:08:50 UTC 2015"
"Release" : "4.6.0-tapasco",
"Version" : "#1 SMP PREEMPT Fri May 26 16:26:16 CEST 2017"
},
"Library Versions" : {
"Platform API" : "1.2.1",
"TPC API" : "1.2"
"Tapasco API" : "1.2"
},
"Timestamp" : "2016-04-20 16:33:49",
"Transfer Speed" : [ {
"Chunk Size" : 256,
"Read" : 49.329030801393962,
"ReadWrite" : 55.568662824206989,
"Write" : 48.829431984156649
}, {
"Chunk Size" : 512,
"Read" : 99.711073192785094,
"ReadWrite" : 106.41911067833198,
"Write" : 91.843507830807283
}, {
"Chunk Size" : 1024,
"Read" : 194.52247367498106,
"ReadWrite" : 206.27983738461907,
"Write" : 174.38052151696462
"Read" : 20.078207396701792,
"Write" : 21.68177044056579,
"ReadWrite" : 37.784625316971535
}, {
"Chunk Size" : 2048,
"Read" : 371.6782428456554,
"ReadWrite" : 380.91467182463464,
"Write" : 325.40049755718161
}, {
"Chunk Size" : 4096,
"Read" : 684.745320993988,
"ReadWrite" : 686.36036417201785,
"Write" : 586.86389287107374
}, {
"Chunk Size" : 8192,
"Read" : 1250.5008574526241,
"ReadWrite" : 1218.1428227833037,
"Write" : 1030.7088569409982
}, {
"Chunk Size" : 16384,
"Read" : 2026.1898750593398,
"ReadWrite" : 1950.3061380522922,
"Write" : 1742.6575694856201
}, {
"Chunk Size" : 32768,
"Read" : 2968.1108195912561,
"ReadWrite" : 2829.2075588814191,
"Write" : 2634.9623567740964
}, {
"Chunk Size" : 65536,
"Read" : 3574.6895965870904,
"ReadWrite" : 3507.9793903906839,
"Write" : 3264.0111182613077
}, {
"Chunk Size" : 131072,
"Read" : 4045.004247835624,
"ReadWrite" : 4001.5013015672289,
"Write" : 3815.5274786894447
}, {
"Chunk Size" : 262144,
"Read" : 4384.3815829308041,
"ReadWrite" : 4200.9501087840708,
"Write" : 3998.2952464074478
}, {
"Chunk Size" : 524288,
"Read" : 4577.7959688291603,
"ReadWrite" : 4289.9518289243551,
"Write" : 2948.3745818585521
}, {
"Chunk Size" : 1048576,
"Read" : 4178.4256565802325,
"ReadWrite" : 3983.4937552016727,
"Write" : 3995.3812092607886
}, {
"Chunk Size" : 2097152,
"Read" : 4428.582495387981,
"ReadWrite" : 3738.9455160871075,
"Write" : 3867.7089852089889
"Read" : 36.913918454834466,
"Write" : 42.449135015215866,
"ReadWrite" : 69.51085992539018
}, {
"Chunk Size" : 67108864,
"Read" : 101.10065452266623,
"Write" : 178.37930005081756,
"ReadWrite" : 175.57383237856067
} ],
"Interrupt Latency" : [ {
"Cycle Count" : 1,
"Avg Latency" : 10.01592947927347,
"Min Latency" : 9,
"Max Latency" : 156
}, {
"Cycle Count" : 2,
"Avg Latency" : 164.85714285714286,
"Min Latency" : 162,
"Max Latency" : 171
}, {
"Cycle Count" : 2147483648,
"Avg Latency" : 276.69565217391295,
"Min Latency" : 269,
"Max Latency" : 282
} ],
"Job Throughput" : [ {
"Number of threads" : 1,
"Jobs per second" : 119074.73866666667
}, {
"Chunk Size" : 4194304,
"Read" : 3889.6524963123138,
"ReadWrite" : 3545.0135744438817,
"Write" : 3583.2993572493579
"Number of threads" : 2,
"Jobs per second" : 246078.71600000001
}, {
"Chunk Size" : 8388608,
"Read" : 3598.3527292972121,
"ReadWrite" : 3293.1267171672935,
"Write" : 3382.630246092891
"Number of threads" : 3,
"Jobs per second" : 237654.358
}, {
"Chunk Size" : 16777216,
"Read" : 3429.053370509484,
"ReadWrite" : 3334.438865174192,
"Write" : 3332.0624094774985
"Number of threads" : 4,
"Jobs per second" : 231967.13166666668
}, {
"Chunk Size" : 33554432,
"Read" : 3322.1144740468026,
"ReadWrite" : 3182.0719464635727,
"Write" : 3178.9272608933888
} ],
"Interrupt Latency": [ {
"Cycle Count": 1,
"Latency": 7.0
"Number of threads" : 5,
"Jobs per second" : 226561.87117590103
}, {
"Cycle Count": 2,
"Latency": 7.5
"Number of threads" : 6,
"Jobs per second" : 222900.86777777775
}, {
"Cycle Count": 4,
"Latency": 8.0
"Number of threads" : 7,
"Jobs per second" : 218548.3080264527
}, {
"Cycle Count": 123456,
"Latency": 10.0
"Number of threads" : 8,
"Jobs per second" : 215949.0373333333
} ]
}
src/main/scala/tapasco/base/Benchmark.scala
View file @ 53d35035
......@@ -36,6 +36,8 @@ final case class Host(machine: String, node: String, operatingSystem: String, re
final case class TransferSpeedMeasurement(chunkSize: Long, read: Double, write: Double, readWrite: Double)
/** Interrupt latency in us at given PE runtime (in clock cycles). */
final case class InterruptLatency(clockCycles: Long, latency: Double, min: Double, max: Double)
/** Jobs per second with given number of threads. */
final case class JobThroughput(numberOfThreads: Int, jobsPerSecond: Double)
/** Defines an interpolation on [[InterruptLatency]] elements. */
final class LatencyInterpolator(data: Seq[InterruptLatency])
extends LinearInterpolator[Long, Double](data map { il => (il.clockCycles, il.latency) }) {
......@@ -75,7 +77,8 @@ final case class Benchmark (
host: Host,
libraryVersions: LibraryVersions,
transferSpeed: Seq[TransferSpeedMeasurement],
interruptLatency: Seq[InterruptLatency]
interruptLatency: Seq[InterruptLatency],
jobThroughput: Seq[JobThroughput]
) extends Description(descPath) {
/** Function to compute interpolated latency values. */
lazy val latency = new LatencyInterpolator(interruptLatency)
......
src/main/scala/tapasco/base/json/package.scala
View file @ 53d35035
......@@ -93,13 +93,19 @@ package object json {
(JsPath \ "Max Latency").format[Double]
) (InterruptLatency.apply _, unlift(InterruptLatency.unapply _))
implicit val jobThroughputFormat: Format[JobThroughput] = (
(JsPath \ "Number of threads").format[Int] ~
(JsPath \ "Jobs per second").format[Double]
) (JobThroughput.apply _, unlift(JobThroughput.unapply _))
implicit val benchmarkReads: Reads[Benchmark] = (
(JsPath \ "DescPath").readNullable[Path].map(_ getOrElse Paths.get("N/A")) ~
(JsPath \ "Timestamp").read[LocalDateTime] ~
(JsPath \ "Host").read[Host] ~
(JsPath \ "Library Versions").read[LibraryVersions] ~
(JsPath \ "Transfer Speed").read[Seq[TransferSpeedMeasurement]] ~
(JsPath \ "Interrupt Latency").read[Seq[InterruptLatency]]
(JsPath \ "Interrupt Latency").read[Seq[InterruptLatency]] ~
(JsPath \ "Job Throughput").read[Seq[JobThroughput]]
) (Benchmark. apply _)
implicit val benchmarkWrites: Writes[Benchmark] = (
(JsPath \ "DescPath").write[Path].transform((js: JsObject) => js - "DescPath") ~
......@@ -107,7 +113,8 @@ package object json {
(JsPath \ "Host").write[Host] ~
(JsPath \ "Library Versions").write[LibraryVersions] ~
(JsPath \ "Transfer Speed").write[Seq[TransferSpeedMeasurement]] ~
(JsPath \ "Interrupt Latency").write[Seq[InterruptLatency]]
(JsPath \ "Interrupt Latency").write[Seq[InterruptLatency]] ~
(JsPath \ "Job Throughput").write[Seq[JobThroughput]]
) (unlift(Benchmark.unapply _))
/* Benchmark @} */
......
src/test/scala/tapasco/base/BenchmarkTest.scala
View file @ 53d35035
......@@ -105,31 +105,45 @@ class BenchmarkSpec extends FlatSpec with Matchers with Checkers {
lazy val c = oc.right.get
assert(oc.isRight)
// host data
c.host.machine should be ("x86_64")
c.host.node should equal ("mountdoom")
c.host.machine should be ("armv7l")
c.host.node should equal ("pynq")
c.host.operatingSystem should equal ("Linux")
c.host.release should equal ("3.19.8-100.fc20.x86_64")
c.host.version should equal ("#1 SMP Tue May 12 17:08:50 UTC 2015")
c.host.release should equal ("4.6.0-tapasco")
c.host.version should equal ("#1 SMP PREEMPT Fri May 26 16:26:16 CEST 2017")
// job throughput
c.jobThroughput should have length (8)
c.jobThroughput should equal (List(
JobThroughput(1, 119074.73866666667),
JobThroughput(2, 246078.71600000001),
JobThroughput(3, 237654.358),
JobThroughput(4, 231967.13166666668),
JobThroughput(5, 226561.87117590103),
JobThroughput(6, 222900.86777777775),
JobThroughput(7, 218548.3080264527),
JobThroughput(8, 215949.0373333333)))
// interrupt latency
c.interruptLatency should equal (List(InterruptLatency(1,7.0), InterruptLatency(2,7.5), InterruptLatency(4,8.0),
InterruptLatency(123456,10.0)))
c.interruptLatency should have length (3)
c.interruptLatency should equal (List(
InterruptLatency(1, 10.01592947927347, 9, 156),
InterruptLatency(2, 164.85714285714286, 162, 171),
InterruptLatency(2147483648L, 276.69565217391295, 269, 282)))
// library versions
c.libraryVersions.platform should equal ("1.2.1")
c.libraryVersions.tapasco should equal ("1.2")
// timestamp
c.timestamp should equal (LocalDate.of(2016, 4, 20).atTime(16,33,49))
c.timestamp should equal (LocalDate.of(2017, 5, 30).atTime(13,8,55))
// transfer speed
c.transferSpeed should have length (18)
var ce = c.transferSpeed(17)
ce.chunkSize should equal (33554432)
ce.read should equal (3322.1144740468026)
ce.write should equal (3178.9272608933888)
ce.readWrite should equal (3182.0719464635727)
c.transferSpeed should have length (3)
var ce = c.transferSpeed(1)
ce.chunkSize should equal (2048)
ce.read should equal (36.913918454834466)
ce.write should equal (42.449135015215866)
ce.readWrite should equal (69.51085992539018)
ce = c.transferSpeed(0)
ce.chunkSize should equal (256)
ce.read should equal (49.329030801393962)
ce.write should equal (48.829431984156649)
ce.readWrite should equal (55.568662824206989)
ce.chunkSize should equal (1024)
ce.read should equal (20.078207396701792)
ce.write should equal (21.68177044056579)
ce.readWrite should equal (37.784625316971535)
}
"An invalid Benchmark file" should "not be parsed" in {
......@@ -145,14 +159,14 @@ class BenchmarkSpec extends FlatSpec with Matchers with Checkers {
val r = if (a.chunkSize < b.chunkSize) b else a
val mbm = bm.copy(transferSpeed = data)
val cs = Gen.choose(0, r.chunkSize + 1)
def interpolate(cs: Int, lcs: Int, ls: Double, rcs: Int, rs: Double): Double =
def interpolate(cs: Long, lcs: Long, ls: Double, rcs: Long, rs: Double): Double =
(((cs - lcs).toDouble / (rcs - lcs).toDouble)) * (rs - ls) + ls
forAll(cs) { n => n match {
case n if n <= l.chunkSize => mbm.speed(n) equals (l.read, l.write, l.readWrite)
case n if n >= r.chunkSize => mbm.speed(n) equals (r.read, r.write, r.readWrite)
case n => mbm.speed(n) equals (interpolate(n, l.chunkSize, l.read, r.chunkSize, r.read),
interpolate(n, l.chunkSize, l.write, r.chunkSize, r.write),
interpolate(n, l.chunkSize, l.readWrite, r.chunkSize, r.readWrite))
case n => mbm.speed(n) equals ((interpolate(n, l.chunkSize, l.read, r.chunkSize, r.read),
interpolate(n, l.chunkSize, l.write, r.chunkSize, r.write),
interpolate(n, l.chunkSize, l.readWrite, r.chunkSize, r.readWrite)))
}}
}})
}
......@@ -167,7 +181,7 @@ class BenchmarkSpec extends FlatSpec with Matchers with Checkers {
val top = data.last
val mbm = bm.copy(transferSpeed = data)
val cs = Gen.choose(0, top.chunkSize + 1)
def interpolate(cs: Int, lcs: Int, ls: Double, rcs: Int, rs: Double): Double =
def interpolate(cs: Long, lcs: Long, ls: Double, rcs: Long, rs: Double): Double =
(((cs - lcs).toDouble / (rcs - lcs).toDouble)) * (rs - ls) + ls
forAll(cs) { n => {
val l = if (n <= mid.chunkSize) bot else mid
......@@ -241,10 +255,19 @@ class BenchmarkSpec extends FlatSpec with Matchers with Checkers {
val ilmGen: Gen[InterruptLatency] = for {
cl <- posIntsPowerTwo
l <- Gen.posNum[Double]
} yield InterruptLatency(cl, l)
v1 <- Gen.posNum[Double]
v2 <- Gen.posNum[Double]
v3 <- Gen.posNum[Double]
vs = Seq(v1, v2, v3).sorted
} yield InterruptLatency(cl, vs(1), vs(0), vs(1))
implicit val arbIlm: Arbitrary[InterruptLatency] = Arbitrary(ilmGen)
val jtGen: Gen[JobThroughput] = for {
t <- Gen.posNum[Int]
j <- Gen.posNum[Double]
} yield JobThroughput(t, j)
implicit val arbJt: Arbitrary[JobThroughput] = Arbitrary(jtGen)
val hostGen = for {
machine <- Arbitrary.arbitrary[String]
node <- Arbitrary.arbitrary[String]
......@@ -266,11 +289,12 @@ class BenchmarkSpec extends FlatSpec with Matchers with Checkers {
val benchmarkGen = for {
timestamp <- Arbitrary.arbitrary[LocalDateTime]
host <- Arbitrary.arbitrary[Host]
lv <- Arbitrary.arbitrary[LibraryVersions]
tsm <- Arbitrary.arbitrary[Seq[TransferSpeedMeasurement]]
il <- Arbitrary.arbitrary[Seq[InterruptLatency]]
} yield Benchmark(java.nio.file.Paths.get("N/A"), timestamp, host, lv, tsm, il)
host <- Arbitrary.arbitrary[Host]
lv <- Arbitrary.arbitrary[LibraryVersions]
tsm <- Arbitrary.arbitrary[Seq[TransferSpeedMeasurement]]
il <- Arbitrary.arbitrary[Seq[InterruptLatency]]
jtp <- Arbitrary.arbitrary[Seq[JobThroughput]]
} yield Benchmark(java.nio.file.Paths.get("N/A"), timestamp, host, lv, tsm, il, jtp)
implicit val arbBenchmark: Arbitrary[Benchmark] = Arbitrary(benchmarkGen)
/* Generators and Arbitraries @} */
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment