moved super module scripts here

azure-job-template.yml

@@ -43,20 +43,20 @@ jobs:
     displayName: 'env setup'
 
   - script: |
-      docker build --build-arg BASE=$(docker.baseimage) -t $(docker.image) -f .ci/docker/Dockerfile .
+      docker build --build-arg BASE=$(docker.baseimage) -t $(docker.image) -f .ci/shared/ci_run/Dockerfile .
       docker inspect $(docker.image)
     displayName: 'build image'
 
   - script: |
-      python3 ./.ci/shared/make_env_file.py
-      $(docker.run) /home/dune-ci/src/${MY_MODULE}/.ci/shared/test_cpp.bash
+      python3 ./.ci/shared/scripts/make_env_file.py
+      $(docker.run) /home/dune-ci/src/${MY_MODULE}/.ci/shared/scripts/test_cpp.bash
     displayName: 'test cpp'
     env:
       CODECOV_TOKEN: $(codecov.token)
 
   - script: |
-      python3 ./.ci/shared/make_env_file.py
-      $(docker.run) /home/dune-ci/src/${MY_MODULE}/.ci/shared/test_python.bash
+      python3 ./.ci/shared/scripts/make_env_file.py
+      $(docker.run) /home/dune-ci/src/${MY_MODULE}/.ci/shared/scripts/test_python.bash
     displayName: 'test python'
     env:
       CODECOV_TOKEN: $(codecov.token)

docker/ci_run/Dockerfile

+ARG BASE
+FROM dunecommunity/$BASE
+
+ADD . ${SUPERDIR}/${MY_MODULE}/
+RUN cd ${SUPERDIR} && \
+    echo ${SRC_DCTRL} ${BLD} --only=${MY_MODULE} all && \
+    ${SRC_DCTRL} ${BLD} --only=${MY_MODULE} all

docker/individual_base/Dockerfile

+ARG CC
+ARG COMMIT
+ARG BASE
+FROM dunecommunity/dune-xt-docker_base_${BASE}_${CC}:${COMMIT}
+
+WORKDIR ${SUPERDIR}
+
+ARG project_name
+ARG modules_to_delete
+ENV MY_MODULE=${project_name}
+
+# run our local scripts look for an OPTS env entry
+# and ensure ${MY_MODULE} from checkout is used via clean mount point
+# and removes xt-la, xt-grid, xt-functions for xt-common, etc
+RUN set -eu && \
+    rm -rf ${modules_to_delete} && \
+    ${SRC_DCTRL} ${BLD} all && \
+    ${SRC_DCTRL} ${BLD} bexec "${BUILD_CMD} bindings || echo no bindings" && \
+    rm -rf ${MY_MODULE}  && \
+    mkdir ${MY_MODULE}
+
+
+# THE END
+ENV DEBIAN_FRONTEND teletype

docker/shared_base/Dockerfile

+ARG BASE
+FROM dunecommunity/testing-base_${BASE}
+
+ARG COMMIT
+
+RUN git clone https://github.com/dune-community/dune-xt-super.git ${SUPERDIR} && \
+    cd ${SUPERDIR} && \
+    git checkout ${COMMIT} && \
+    git submodule foreach git reset --hard && \
+    git submodule update --init --recursive && \
+    git submodule status
+WORKDIR ${SUPERDIR}
+
+ARG CC
+ARG CXX
+
+ENV CC=${CC} \
+    CXX=${CXX}
+
+# run our local scripts look for an OPTS env entry
+RUN ./bin/download_external_libraries.py 1> /dev/null && \
+    ./bin/build_external_libraries.py 1> /dev/null
+
+RUN . environments/${DXT_ENVIRONMENT}/PATH.sh

docker/super_docker/Dockerfile

+ARG BASE
+FROM dunecommunity/testing-base_${BASE}:latest
+
+ENV CC=gcc \
+    CXX=g++
+
+USER root
+ADD . ${SUPERDIR}/
+WORKDIR ${SUPERDIR}
+RUN chown dune-ci -R ${SUPERDIR}
+USER dune-ci
+
+# run our local scripts look for an OPTS env entry
+RUN ./bin/download_external_libraries.py 1> /dev/null && \
+    ./bin/build_external_libraries.py 1> /dev/null && \
+    . environments/${DXT_ENVIRONMENT}/PATH.sh && \
+    ${SRC_DCTRL} ${BLD} all

docker/update_test_dockers.py

+#!/usr/bin/env python3
+
+"""
+Update docker images and templated scripts in dune-xt-*
+
+Usage:
+  update_test_dockers.py [options] MODULE_NAME
+
+Options:
+   -h --help       Show this message.
+   -v --verbose    Set logging level to debug.
+"""
+
+from docopt import docopt
+from os import path
+import importlib
+import os
+import contextlib
+import subprocess
+import sys
+import logging
+import time
+import six
+import re
+try:
+    import docker
+except ImportError:
+    print('missing module: pip install docker')
+    sys.exit(1)
+from docker.utils.json_stream import json_stream
+
+TAG_MATRIX = {'debian-unstable_gcc_full': {'cc': 'gcc', 'cxx': 'g++', 'deletes': "", 'base': 'debian-unstable'},
+        'debian_gcc_full': {'cc': 'gcc', 'cxx': 'g++', 'deletes': "", 'base': 'debian'},
+        'debian_clang_full': {'cc': 'clang', 'cxx': 'clang++', 'deletes':"", 'base': 'debian'},}
+        #'arch_gcc_full': {'cc': 'gcc', 'cxx': 'g++', 'deletes': "", 'base': 'arch'},}
+
+
+@contextlib.contextmanager
+def remember_cwd(dirname):
+    curdir = os.getcwd()
+    try:
+        os.chdir(dirname)
+        yield curdir
+    finally:
+        os.chdir(curdir)
+
+
+class Timer(object):
+    def __init__(self, section, log):
+        self._section = section
+        self._start = 0
+        self._log = log
+        self.time_func = time.time
+
+    def start(self):
+        self.dt = -1
+        self._start = self.time_func()
+
+    def stop(self):
+        self.dt = self.time_func() - self._start
+
+    def __enter__(self):
+        self.start()
+
+    def __exit__(self, type_, value, traceback):
+        self.stop()
+        self._log('Execution of {} took {} (s)'.format(self._section, self.dt))
+
+def _docker_build(client, **kwargs):
+    resp = client.api.build(**kwargs)
+    if isinstance(resp, six.string_types):
+        return client.images.get(resp)
+    last_event = None
+    image_id = None
+    output = []
+    for chunk in json_stream(resp):
+        if 'error' in chunk:
+            msg = chunk['error'] + '\n' + ''.join(output)
+            raise docker.errors.BuildError(msg, resp)
+        if 'stream' in chunk:
+            output.append(chunk['stream'])
+            match = re.search(
+                r'(^Successfully built |sha256:)([0-9a-f]+)$',
+                chunk['stream']
+            )
+            if match:
+                image_id = match.group(2)
+        last_event = chunk
+    if image_id:
+        return client.images.get(image_id)
+    raise docker.errors.BuildError(last_event or 'Unknown', resp)
+
+
+def _cmd(cmd, logger):
+    logger.debug(' '.join(cmd))
+    try:
+        out = subprocess.check_output(cmd, stderr=subprocess.STDOUT, universal_newlines=True)
+        logger.debug(out)
+    except subprocess.CalledProcessError as cp:
+        logger.error(cp.output)
+        logger.error('Failed: {}'.format(' '.join(cmd)))
+        logger.error('Make sure the pushers group has write access to this repo on hub.cocker.com!')
+        raise cp
+
+
+def _build_base(scriptdir, distro, cc, cxx, commit, refname):
+    client = docker.from_env(version='auto')
+    base_postfix = '{}_{}'.format(distro, cc)
+    slug_postfix = 'base_{}'.format(base_postfix)
+    logger = logging.getLogger('{}'.format(slug_postfix))
+    dockerdir = path.join(scriptdir, 'dune-xt-docker_base')
+    dockerfile = path.join(dockerdir, 'Dockerfile')
+    repo = 'dunecommunity/dune-xt-docker_{}'.format(slug_postfix)
+    with Timer('docker build ', logger.info):
+        buildargs = {'COMMIT': commit, 'CC': cc, 'CXX': cxx, 'BASE': distro}
+        img = _docker_build(client, rm=False, buildargs=buildargs, pull=True,
+                            tag='{}:{}'.format(repo, commit), path=dockerdir)
+        img.tag(repo, refname)
+    with Timer('docker push {}:{}|{}'.format(repo, refname, commit), logger.info):
+        client.images.push(repo, tag=refname)
+        client.images.push(repo, tag=commit)
+    return img
+
+
+def _build_combination(tag_matrix, dockerdir, module, commit, refname):
+    client = docker.from_env(version='auto')
+    imgs = []
+    for tag, settings in tag_matrix.items():
+        cc = settings['cc']
+        cxx = settings['cxx']
+        vars = importlib.import_module(module)
+        tmp_dir = path.join(path.dirname(path.abspath(__file__)), module, tag)
+        modules = settings['deletes']
+        logger = logging.getLogger('{} - {}'.format(module, tag))
+        modules_to_delete = '{} {}'.format(modules, vars.modules_to_delete)
+        logger.debug('delete: ' + modules_to_delete)
+        repo = 'dunecommunity/{}-testing_{}'.format(module, tag)
+
+        with Timer('docker build ', logger.info):
+            buildargs = {'COMMIT': commit, 'CC': cc, 'project_name': module,
+                         'modules_to_delete': modules_to_delete, 'BASE': settings['base']}
+            img = _docker_build(client, rm=False, buildargs=buildargs,
+                    tag='{}:{}'.format(repo, commit), path=dockerdir)
+            img.tag(repo, refname)
+        with Timer('docker push {}:{}|{}'.format(repo, refname, commit), logger.info):
+            client.images.push(repo, tag=refname)
+            client.images.push(repo, tag=commit)
+        imgs.append(img)
+    return imgs
+
+
+if __name__ == '__main__':
+    arguments = docopt(__doc__)
+    level = logging.DEBUG if arguments['--verbose'] else logging.INFO
+    logging.basicConfig(level=level)
+    scriptdir = path.dirname(path.abspath(__file__))
+    superdir = path.join(scriptdir, '..', '..')
+    names = ['common', 'functions', 'la', 'grid', 'data'] if 'XT_MODULE_NAME' not in os.environ else [os.environ['XT_MODULE_NAME']]
+
+    head = subprocess.check_output(['git', 'rev-parse', 'HEAD'], universal_newlines=True).strip()
+    commit = os.environ.get('CI_COMMIT_SHA', head)
+    refname = os.environ.get('CI_COMMIT_REF_NAME', 'master').replace('/', '_')
+
+    all_compilers = {(f['base'], f['cc'], f['cxx']) for f in TAG_MATRIX.values()}
+    base_imgs = [_build_base(scriptdir, base, cc, cxx, commit, refname) for base, cc, cxx in all_compilers]
+    for i in names:
+        module = 'dune-xt-{}'.format(i)
+        module_dir = os.path.join(superdir, module)
+        _build_combination(tag_matrix=TAG_MATRIX,
+                            dockerdir=os.path.join(scriptdir, 'dune-xt-docker'),
+                            module=module,
+                            commit=commit, refname=refname)