Commit 8b391d02 authored by René Fritze's avatar René Fritze Committed by probot-auto-merge[bot]

[ci] hook tutorials into pytest

- included in the "notebooks" step
- failed code is written to disc and uploaded as ci artefact
parent 24a707d5
......@@ -48,6 +48,7 @@ stages:
expire_in: 3 months
paths:
- src/pymortests/testdata/check_results/*/*_changed
- docs/source/*_extracted.py
- coverage*
- memory_usage.txt
- .hypothesis
......@@ -357,6 +358,78 @@ pip_installed 3 8:
- ./.ci/gitlab/test_pip_installed.bash
- find . -name "coverage*"
- ls -la *
tutorials 3 6:
extends: .pytest
rules:
- if: $CI_PIPELINE_SOURCE == "schedule"
when: never
- when: on_success
variables:
COVERAGE_FILE: coverage_tutorials__3.6
services:
- name: zivgitlab.wwu.io/pymor/docker/pymor/pypi-mirror_stable_py3.6:f6acb8bdf750d965c15f7f90b01fac4b745b58bd
alias: pypi_mirror
image: zivgitlab.wwu.io/pymor/docker/pymor/testing_py3.6:f6acb8bdf750d965c15f7f90b01fac4b745b58bd
script:
- |
if [[ "$CI_COMMIT_REF_NAME" == *"github/PR_"* ]]; then
echo selecting hypothesis profile "ci_pr" for branch $CI_COMMIT_REF_NAME
export PYMOR_HYPOTHESIS_PROFILE="ci_pr"
else
echo selecting hypothesis profile "ci" for branch $CI_COMMIT_REF_NAME
export PYMOR_HYPOTHESIS_PROFILE="ci"
fi
- ./.ci/gitlab/test_tutorials.bash
- find . -name "coverage*"
- ls -la *
tutorials 3 7:
extends: .pytest
rules:
- if: $CI_PIPELINE_SOURCE == "schedule"
when: never
- when: on_success
variables:
COVERAGE_FILE: coverage_tutorials__3.7
services:
- name: zivgitlab.wwu.io/pymor/docker/pymor/pypi-mirror_stable_py3.7:f6acb8bdf750d965c15f7f90b01fac4b745b58bd
alias: pypi_mirror
image: zivgitlab.wwu.io/pymor/docker/pymor/testing_py3.7:f6acb8bdf750d965c15f7f90b01fac4b745b58bd
script:
- |
if [[ "$CI_COMMIT_REF_NAME" == *"github/PR_"* ]]; then
echo selecting hypothesis profile "ci_pr" for branch $CI_COMMIT_REF_NAME
export PYMOR_HYPOTHESIS_PROFILE="ci_pr"
else
echo selecting hypothesis profile "ci" for branch $CI_COMMIT_REF_NAME
export PYMOR_HYPOTHESIS_PROFILE="ci"
fi
- ./.ci/gitlab/test_tutorials.bash
- find . -name "coverage*"
- ls -la *
tutorials 3 8:
extends: .pytest
rules:
- if: $CI_PIPELINE_SOURCE == "schedule"
when: never
- when: on_success
variables:
COVERAGE_FILE: coverage_tutorials__3.8
services:
- name: zivgitlab.wwu.io/pymor/docker/pymor/pypi-mirror_stable_py3.8:f6acb8bdf750d965c15f7f90b01fac4b745b58bd
alias: pypi_mirror
image: zivgitlab.wwu.io/pymor/docker/pymor/testing_py3.8:f6acb8bdf750d965c15f7f90b01fac4b745b58bd
script:
- |
if [[ "$CI_COMMIT_REF_NAME" == *"github/PR_"* ]]; then
echo selecting hypothesis profile "ci_pr" for branch $CI_COMMIT_REF_NAME
export PYMOR_HYPOTHESIS_PROFILE="ci_pr"
else
echo selecting hypothesis profile "ci" for branch $CI_COMMIT_REF_NAME
export PYMOR_HYPOTHESIS_PROFILE="ci"
fi
- ./.ci/gitlab/test_tutorials.bash
- find . -name "coverage*"
- ls -la *
vanilla 3 6:
extends: .pytest
rules:
......
......@@ -50,6 +50,7 @@ stages:
expire_in: 3 months
paths:
- src/pymortests/testdata/check_results/*/*_changed
- docs/source/*_extracted.py
- coverage*
- memory_usage.txt
- .hypothesis
......@@ -428,7 +429,7 @@ tpl = jinja2.Template(tpl)
pythons = ['3.6', '3.7', '3.8']
oldest = [pythons[0]]
newest = [pythons[-1]]
test_scripts = [("mpi", pythons, 1), ("pip_installed", pythons, 1),
test_scripts = [("mpi", pythons, 1), ("pip_installed", pythons, 1), ("tutorials", pythons, 1),
("vanilla", pythons, 1), ("numpy_git", newest, 1), ("oldest", oldest, 1),]
# these should be all instances in the federation
binder_urls = [f'https://{sub}.mybinder.org/build/gh/pymor/pymor' for sub in ('gke', 'ovh', 'gesis')]
......
#!/bin/bash
THIS_DIR="$(cd "$(dirname ${BASH_SOURCE[0]})" ; pwd -P )"
source ${THIS_DIR}/common_test_setup.bash
xvfb-run -a py.test ${COMMON_PYTEST_OPTS} docs/test_tutorials.py
coverage xml
......@@ -50,7 +50,7 @@ doc_requires = ['sphinx>=1.7', 'jupyter_sphinx', 'matplotlib', 'PySide2', 'ipypa
'ipywidgets', 'sphinx-qt-documentation', 'bash_kernel', 'sphinx-material'] + install_requires
ci_requires = [_PYTEST, 'pytest-cov', 'pytest-xdist', 'check-manifest', 'nbconvert', 'pytest-parallel',
'readme_renderer[md]', 'rstcheck', 'codecov', 'twine', 'pytest-memprof', 'pytest-timeout',
'testipynb', "pypi-oldest-requirements>=2020.2", 'hypothesis[numpy,pytest]>=5.19', 'PyQt5']
'docutils', "pypi-oldest-requirements>=2020.2", 'hypothesis[numpy,pytest]>=5.19', 'PyQt5']
import_names = {'ipython': 'IPython',
'pytest-cache': 'pytest_cache',
'pytest-instafail': 'pytest_instafail',
......
*rst_extracted.py
......@@ -2,6 +2,10 @@
:hide-code:
:hide-output:
from IPython import get_ipython
ip = get_ipython()
if ip is not None:
ip.run_line_magic('load_ext', 'pymor.discretizers.builtin.gui.jupyter')
%matplotlib inline
import warnings
......@@ -9,10 +13,6 @@
import pymor.tools.random
pymor.tools.random._default_random_state = None
from IPython import get_ipython
ip = get_ipython()
if ip is not None:
ip.run_line_magic('load_ext', 'pymor.discretizers.builtin.gui.jupyter')
..
named .txt to avoid sphinx warning since it's not included in any toc
# This file is part of the pyMOR project (http://www.pymor.org).
# Copyright 2013-2020 pyMOR developers and contributors. All rights reserved.
# License: BSD 2-Clause License (http://opensource.org/licenses/BSD-2-Clause)
import os
import sys
from pathlib import Path
import io
import importlib.machinery
import importlib.util
from docutils.core import publish_doctree
from docutils.parsers.rst import Directive
from docutils.parsers.rst.directives import flag, register_directive
import pytest
from pymortests.base import runmodule
from pymortests.demos import _test_demo
TUT_DIR = Path(os.path.dirname(__file__)).resolve() / 'source'
EXCLUDE = [TUT_DIR / t for t in ['tutorial_external_solver.rst']]
TUTORIALS = [t for t in TUT_DIR.glob('tutorial_*rst') if t not in EXCLUDE]
class CodeCell(Directive):
required_arguments = 0
optional_arguments = 0
final_argument_whitespace = True
option_spec = {'hide-output': flag,
'hide-code': flag,
'raises': flag}
has_content = True
def run(self):
self.assert_has_content()
if 'raises' in self.options:
text = 'try:\n ' + '\n '.join(
self.content) + '\nexcept:\n import traceback; traceback.print_exc()'
else:
text = '\n'.join(self.content)
print('# %%')
print(text)
print()
return []
@pytest.fixture(params=TUTORIALS, ids=[t.name for t in TUTORIALS])
def tutorial_code(request):
filename = request.param
current_dir = os.getcwd()
os.chdir(TUT_DIR)
code = io.StringIO()
register_directive('jupyter-execute', CodeCell)
with open(filename, 'rt') as f:
original = sys.stdout
sys.stdout = code
publish_doctree(f.read(), settings_overrides={'report_level': 42})
sys.stdout = original
os.chdir(current_dir)
code.seek(0)
source_fn = Path(f'{str(filename).replace(".rst", "_rst")}_extracted.py')
with open(source_fn, 'wt') as source:
# filter line magics
source.write(''.join([line for line in code.readlines() if not line.startswith('%')]))
return request.param, source_fn
def test_tutorial(tutorial_code):
filename, source_module_path = tutorial_code
# make sure (picture) resources can be loaded as in sphinx-build
current_dir = os.getcwd()
os.chdir(TUT_DIR)
def _run():
loader = importlib.machinery.SourceFileLoader(source_module_path.stem, str(source_module_path))
spec = importlib.util.spec_from_loader(loader.name, loader)
mod = importlib.util.module_from_spec(spec)
loader.exec_module(mod)
try:
# wrap module execution in hacks to auto-close Qt-Apps, etc.
_test_demo(_run)
except Exception as e:
print(f'Failed: {source_module_path}')
raise e
os.chdir(current_dir)
if __name__ == "__main__":
runmodule(filename=__file__)
......@@ -3,6 +3,7 @@
PyQt5
check-manifest
codecov
docutils
hypothesis[numpy,pytest]>=5.19
nbconvert
pypi-oldest-requirements>=2020.2
......@@ -14,5 +15,4 @@ pytest-xdist
pytest>=4.4
readme_renderer[md]
rstcheck
testipynb
twine
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment