Try to parallelize test docker builds

docker/update_test_dockers.py

@@ -15,7 +15,7 @@
 Update docker images and templated scripts in dune-xt-*
 
 Usage:
-  update_test_dockers.py [options] MODULE_NAME
+  update_test_dockers.py [options] MODULE_NAME IMAGE_NAME
 
 Options:
    -h --help       Show this message.
@@ -24,7 +24,6 @@ Options:
 
 from docopt import docopt
 from os import path, environ
-import importlib
 import os
 import contextlib
 import subprocess
@@ -120,26 +119,12 @@ def _docker_build(client, logger, **kwargs):
     logger.fatal(f"Failed docker build command: {kwargs}")
     raise docker.errors.BuildError(last_event or 'Unknown', resp)
 
-
-def _cmd(cmd, logger):
-    logger.debug(' '.join(cmd))
-    try:
-        out = subprocess.check_output(cmd, stderr=subprocess.STDOUT, universal_newlines=True)
-        logger.debug(out)
-    except subprocess.CalledProcessError as cp:
-        logger.error(cp.output)
-        logger.error('Failed: {}'.format(' '.join(cmd)))
-        logger.error('Make sure the pushers group has write access to this repo on hub.cocker.com!')
-        raise cp
-
-
 def _build_base(scriptdir, distro, cc, cxx, commit, refname, superurl):
     client = docker.from_env(version='auto')
     base_postfix = '{}_{}'.format(distro, cc)
     slug_postfix = 'base_{}'.format(base_postfix)
     logger = logging.getLogger('{}'.format(slug_postfix))
     dockerdir = path.join(scriptdir, 'shared_base')
-    dockerfile = path.join(dockerdir, 'Dockerfile')
     repo = f'{PROJECT}/{slug_postfix}'
     with Timer('docker build ', logger.info):
         buildargs = {'COMMIT': commit, 'CC': cc, 'CXX': cxx, 'SUPERURL': superurl, 'BASE': distro}
@@ -157,40 +142,38 @@ def _build_base(scriptdir, distro, cc, cxx, commit, refname, superurl):
     return img
 
 
-def _build_combination(tag_matrix, dockerdir, module, commit, refname):
+def _build_image(image, dockerdir, module, commit, refname):
     client = docker.from_env(version='auto')
     imgs = []
-    for tag, settings in tag_matrix.items():
-        cc = settings['cc']
-        cxx = settings['cxx']
-        tmp_dir = path.join(path.dirname(path.abspath(__file__)), module, tag)
-        logger = logging.getLogger('{} - {}'.format(module, tag))
-        repo = f'{PROJECT}/{module}-testing_{tag}'
-
-        with Timer('docker build ', logger.info):
-            buildargs = {
-                'COMMIT': commit,
-                'CC': cc,
-                'project_name': module,
-                'BASE': settings['base'],
-                'PROJECT': PROJECT
-            }
-            img = _docker_build(client,
-                                logger,
-                                rm=True,
-                                buildargs=buildargs,
-                                pull=True,
-                                tag='{}:{}'.format(repo, commit),
-                                path=dockerdir)
-            img.tag(repo, refname)
-        with Timer('docker push {}:{}|{}'.format(repo, refname, commit), logger.info):
-            client.images.push(repo, tag=refname)
-            client.images.push(repo, tag=commit)
-        imgs.append(img)
+    tag, settings = image, TAG_MATRIX[image]
+    cc = settings['cc']
+    logger = logging.getLogger('{} - {}'.format(module, tag))
+    repo = f'{PROJECT}/{module}-testing_{tag}'
+
+    with Timer('docker build ', logger.info):
+        buildargs = {
+            'COMMIT': commit,
+            'CC': cc,
+            'project_name': module,
+            'BASE': settings['base'],
+            'PROJECT': PROJECT
+        }
+        img = _docker_build(client,
+                            logger,
+                            rm=True,
+                            buildargs=buildargs,
+                            pull=True,
+                            tag='{}:{}'.format(repo, commit),
+                            path=dockerdir)
+        img.tag(repo, refname)
+    with Timer('docker push {}:{}|{}'.format(repo, refname, commit), logger.info):
+        client.images.push(repo, tag=refname)
+        client.images.push(repo, tag=commit)
+    imgs.append(img)
     return imgs
 
 
-def _get_superurl(superdir):
+def _get_superurl():
     superurl = os.environ.get('CI_REPOSITORY_URL', None)
     if superurl:
         return superurl
@@ -217,19 +200,20 @@ if __name__ == '__main__':
     logging.basicConfig(level=level)
     scriptdir = path.dirname(path.abspath(__file__))
     superdir = path.join(scriptdir, '..', '..', '..')
-    superurl = _get_superurl(superdir)
+    superurl = _get_superurl()
 
     commit, refname = _get_ci_setup()
     all_compilers = {(f['base'], f['cc'], f['cxx']) for f in TAG_MATRIX.values()}
 
     module = arguments['MODULE_NAME']
+    image = arguments['IMAGE_NAME']
 
     if module == 'BASE':
         for base, cc, cxx in all_compilers:
             _build_base(scriptdir, base, cc, cxx, commit, refname, superurl)
     else:
         module_dir = os.path.join(superdir, module)
-        _build_combination(tag_matrix=TAG_MATRIX,
+        _build_image(image=image,
                            dockerdir=os.path.join(scriptdir, 'individual_base'),
                            module=module,
                            commit=commit,