diff --git a/cmake/scripts/distribute_testing.py b/cmake/scripts/distribute_testing.py
index 89d44285ce0ea42f11703baead5ad6284d8e8ae4..e50d678d63674678bf838c1cdf88a81ded15527f 100644
--- a/cmake/scripts/distribute_testing.py
+++ b/cmake/scripts/distribute_testing.py
@@ -8,10 +8,10 @@ import subprocess
import time
from contextlib import contextmanager
import binpacking
-from multiprocessing import Pool
+from multiprocessing import Pool, cpu_count
-MAXTIME = 15*60
+MAXTIME = 4*60
pickle_file = 'totals.pickle'
@@ -28,9 +28,9 @@ def elapsed_timer():
def _compile(binary):
with elapsed_timer() as timer:
try:
- subprocess.check_call(['ninja', '-j1', binary])
+ _ = subprocess.check_output(['ninja', '-j1', binary], universal_newlines=True, stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as cpe:
- if not 'Timeout' in cpe:
+ if 'Timeout' not in cpe.output:
raise cpe
print('Timeout in compile {}'.format(binary))
return timer()
@@ -39,54 +39,80 @@ def _compile(binary):
def _run_tests(tpl):
binary, teststrings = tpl
testtimes = 0
- for test in teststrings.split('/'):
+ for test in teststrings.split(';'):
with elapsed_timer() as timer:
try:
- out = subprocess.check_output(['ctest', '-j1', '-R', test])
+ _ = subprocess.check_output(['ctest', '-j1', '-N', '-R', test], universal_newlines=True,
+ stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as cpe:
- if not 'Timeout' in cpe:
+ if 'Timeout' not in cpe.output:
raise cpe
+ # be pessimistic and double the timeout value as time for this run
+ testtimes += timer()
print('Timeout in {} from {}'.format(test, binary))
testtimes += timer()
- return testtimes
-
-
-def redo_timings(builddir, binaries, testnames, processes):
+ return testtimes
+
+
+def _redo(processes, keys, *args):
+ try:
+ with Pool(processes=processes) as pool:
+ result = pool.map(*args)
+ return {k: v for k,v in zip(keys, result)}
+ except subprocess.CalledProcessError as cpe:
+ print('*'*79)
+ print(cpe.stdout)
+ print(cpe.stderr)
+ print('*' * 79)
+ raise cpe
+
+def do_timings(builddir, pickledir, binaries, testnames, processes):
os.chdir(builddir)
testlimit = -1
- binaries = binaries[:testlimit]
- testnames = testnames[:testlimit]
- with Pool(processes=processes) as pool:
- compiles = pool.map(_compile, binaries)
- with Pool(processes=processes) as pool:
- testruns = pool.map(_run_tests, zip(binaries, testnames))
-
- totals = [a+b for a,b in zip(compiles, testruns)]
+ binaries = binaries[:testlimit]
+ compiles_fn = os.path.join(pickledir, 'compiles_' + pickle_file)
+ try:
+ compiles = pickle.load(open(compiles_fn, 'rb'))
+ if set(compiles.keys()) != set(binaries):
+ print('redoing compiles due to mismatched binaries')
+ compiles = _redo(processes, binaries, _compile, binaries)
+ except FileNotFoundError:
+ print('redoing compiles due to missing pickle')
+ compiles = _redo(processes, binaries, _compile, binaries)
+ pickle.dump(compiles, open(compiles_fn, 'wb'))
+ testnames = testnames[:testlimit]
+ testruns_fn = os.path.join(pickledir, 'testruns_' + pickle_file)
+ try:
+ loaded_testnames, testruns = pickle.load(open(testruns_fn, 'rb'))
+ if set(compiles.keys()) != set(binaries) or loaded_testnames != testnames:
+ print('redoing tests due to mismatched binaries/testnames')
+ testruns = _redo(processes, binaries, _run_tests, zip(binaries, testnames))
+ except FileNotFoundError:
+ print('redoing tests due to missing pickle')
+ testruns = _redo(processes, binaries, _run_tests, zip(binaries, testnames))
+ pickle.dump((testnames, testruns), open(testruns_fn, 'wb'))
+
+ totals = {n: compiles[n]+testruns[n] for n in binaries}
+ pickle.dump(totals, open(os.path.join(pickledir, pickle_file), 'wb'))
print('totals')
pprint(totals)
- pickle.dump(totals, open(os.path.join(builddir, pickle_file), 'wb'))
- pickle.dump(compiles, open(os.path.join(builddir, 'compile_'+pickle_file), 'wb'))
- pickle.dump(testruns, open(os.path.join(builddir, 'tests_' + pickle_file), 'wb'))
- return {b: t for b, t in zip(binaries, totals)}
+ return totals
# list comes with a leading empty entry
testnames = sys.argv[4].split('/')[1:]
builddir = sys.argv[1]
+pickledir = sys.argv[2]
binaries = sys.argv[3].split(';')
-processes = 4
-
-try:
- totals = pickle.load(open(os.path.join(builddir, pickle_file), 'rb'))
- if totals.keys() != binaries:
- totals = redo_timings(builddir, binaries, testnames, processes)
-except FileNotFoundError:
- totals = redo_timings(builddir, binaries, testnames, processes)
+processes = cpu_count()
+totals = do_timings(builddir, pickledir, binaries, testnames, processes)
builder_count = sys.argv[2]
b = list(totals.keys())
-bins = binpacking.to_constant_volume(b,MAXTIME)
-pprint(bins)
\ No newline at end of file
+bins = binpacking.to_constant_volume(totals, MAXTIME)
+for idx, bin in enumerate(bins):
+ pprint('Bin {} vol: {}'.format(idx, sum(bin.values())))
+ pprint(bin)
\ No newline at end of file
diff --git a/dune/gdt/test/CMakeLists.txt b/dune/gdt/test/CMakeLists.txt
index 4d8baa8478255c726737bb5006bf6f6337fa317a..4d4c1f767c493f7cb90205b62f2256f28f0b2a4f 100644
--- a/dune/gdt/test/CMakeLists.txt
+++ b/dune/gdt/test/CMakeLists.txt
@@ -22,7 +22,7 @@ endforeach (target ${dxt_test_binaries})
# message(STATUS "LL ${all_sorted_testnames}")
add_custom_target(refresh_test_timings python3 ${CMAKE_SOURCE_DIR}/cmake/scripts/distribute_testing.py
- "${CMAKE_BINARY_DIR}" ${DXT_TRAVIS_BUILDER} "${dxt_test_binaries}" "${all_sorted_testnames}" VERBATIM)
+ "${CMAKE_BINARY_DIR}" "${CMAKE_SOURCE_DIR}/cmake/scripts/" "${dxt_test_binaries}" "${all_sorted_testnames}" VERBATIM)
# link spe10 data file if present
if (NOT ${SPE10MODEL1DATA} STREQUAL "SPE10MODEL1DATA-NOTFOUND")