From 47a9483cf6c6af654c22acfece7bc20af22cc3f0 Mon Sep 17 00:00:00 2001
From: Rene Milk <rene.milk@wwu.de>
Date: Fri, 21 Oct 2016 10:31:53 +0200
Subject: [PATCH] [cmake] make work dist script use multiprocess
---
cmake/scripts/distribute_testing.py | 70 ++++++++++++++++-------------
1 file changed, 38 insertions(+), 32 deletions(-)
diff --git a/cmake/scripts/distribute_testing.py b/cmake/scripts/distribute_testing.py
index 5ef273651..a3c6f03d8 100644
--- a/cmake/scripts/distribute_testing.py
+++ b/cmake/scripts/distribute_testing.py
@@ -8,6 +8,8 @@ import subprocess
import time
from contextlib import contextmanager
import binpacking
+from multiprocessing import Pool
+
MAXTIME = 45*60
pickle_file = 'totals.pickle'
@@ -23,52 +25,56 @@ def elapsed_timer():
elapser = lambda: end-start
-def redo_timings(argv):
- builddir = argv[1]
- builder_count = argv[2]
- binaries = argv[3].split(';')
- # list comes with a leading empty entry
- testnames = argv[4].split('/')[1:]
+def _compile(binary):
+ with elapsed_timer() as timer:
+ subprocess.check_call(['ninja', '-j1', binary])
+ return timer()
+
+
+def _run_tests(tpl):
+ binary, teststrings = tpl
+ testtimes = 0
+ for test in teststrings.split('/'):
+ with elapsed_timer() as timer:
+ subprocess.check_call(['ctest', '-j1', '-R', test])
+ testtimes += timer()
+ return testtimes
- os.chdir(builddir)
- compiles = {}
- testtimes = {}
+def redo_timings(builddir, binaries, testnames, processes):
+ os.chdir(builddir)
testlimit = -1
binaries = binaries[:testlimit]
testnames = testnames[:testlimit]
- totals = {}
- for binary in binaries:
- with elapsed_timer() as timer:
- subprocess.check_call(['ninja', '-j1', binary])
- compiles[binary] = timer()
- totals[binary] = compiles[binary]
- for binary, teststrings in zip(binaries, testnames):
- testtimes[binary] = 0
- pprint(teststrings.split('/'))
- for test in teststrings.split('/'):
- with elapsed_timer() as timer:
- subprocess.check_call(['ctest', '-j1', '-R', test])
- testtimes[binary] += timer()
- totals[binary] = testtimes[binary]
- print('compiles')
- pprint(compiles)
- print('testtimes')
- pprint(testtimes)
+ with Pool(processes=processes) as pool:
+ compiles = pool.map(_compile, binaries)
+ with Pool(processes=processes) as pool:
+ testruns = pool.map(_run_tests, zip(binaries, testnames))
+
+ totals = [a+b for a,b in zip(compiles, testruns)]
+
print('totals')
pprint(totals)
pickle.dump(totals, open(os.path.join(builddir, pickle_file), 'wb'))
- return totals
+ return {b: t for b, t in zip(binaries, totals)}
+# list comes with a leading empty entry
+testnames = sys.argv[4].split('/')[1:]
+builddir = sys.argv[1]
+binaries = sys.argv[3].split(';')
+processes = 4
+
try:
- builddir = sys.argv[1]
totals = pickle.load(open(os.path.join(builddir, pickle_file), 'rb'))
+ if totals.keys() != binaries:
+ totals = redo_timings(builddir, binaries, testnames, processes)
except FileNotFoundError:
- totals = redo_timings(sys.argv)
+ totals = redo_timings(builddir, binaries, testnames, processes)
-b = list(totals.values())
-bins = binpacking.to_constant_volume(b,MAXTIME)
+builder_count = sys.argv[2]
+b = list(totals.keys())
+bins = binpacking.to_constant_volume(b,MAXTIME)
pprint(bins)
\ No newline at end of file
--
GitLab