Commit 0699985e authored by René Fritze's avatar René Fritze
Browse files

[ci] simplify tutorial testing

and fix coverage generation
parent 78581d29
......@@ -3,6 +3,11 @@
THIS_DIR="$(cd "$(dirname ${BASH_SOURCE[0]})" ; pwd -P )"
source ${THIS_DIR}/common_test_setup.bash
xvfb-run -a py.test ${COMMON_PYTEST_OPTS} docs/test_tutorials.py
for fn in ${PYMOR_ROOT}/docs/source/tutorial*md ; do
mystnb-to-jupyter -o "${fn}" "${fn/tutorial/..\/converted_tutorial}".ipynb
done
xvfb-run -a py.test ${COMMON_PYTEST_OPTS} -s --cov= -p no:pycharm \
docs/test_tutorials.py
coverage xml
......@@ -78,6 +78,7 @@ ci_requires = ['check-manifest==0.48',
_PYTEST,
'pytest-cov==3.0.0',
'pytest-memprof==0.2.0',
'pytest-notebook==0.8.0',
'pytest-parallel==0.1.1',
'pytest-regressions==2.3.1',
'pytest-xdist==2.5.0',
......
......@@ -2,86 +2,36 @@
# Copyright pyMOR developers and contributors. All rights reserved.
# License: BSD 2-Clause License (https://opensource.org/licenses/BSD-2-Clause)
import os
import sys
from pathlib import Path
import io
import importlib.machinery
import importlib.util
from docutils.core import publish_doctree
from docutils.parsers.rst import Directive
from docutils.parsers.rst.directives import flag, register_directive
import pytest
from pytest_notebook.nb_regression import NBRegressionFixture
from pytest_notebook.plugin import gather_config_options
from pymor.tools.io import change_to_directory
from pymortests.base import runmodule
from pymortests.demos import _test_demo
TUT_DIR = Path(os.path.dirname(__file__)).resolve() / 'source'
TUT_DIR = Path(os.path.dirname(__file__)).resolve().absolute()
_exclude_files = []
EXCLUDE = [TUT_DIR / t for t in _exclude_files]
TUTORIALS = [t for t in TUT_DIR.glob('tutorial_*rst') if t not in EXCLUDE]
TUTORIALS += [t for t in TUT_DIR.glob('tutorial_*md') if t not in EXCLUDE]
TUTORIALS = [t for t in TUT_DIR.glob('converted_tutorial_*ipynb') if t not in EXCLUDE]
class CodeCell(Directive):
class NBLaxFixture(NBRegressionFixture):
"""Same functionality as base class, but result comparison for regressions is skipped"""
required_arguments = 0
optional_arguments = 0
final_argument_whitespace = True
option_spec = {'hide-output': flag,
'hide-code': flag,
'raises': flag}
has_content = True
def check(self, path):
return super().check(path=path, raise_errors=False)
def run(self):
self.assert_has_content()
if 'raises' in self.options:
text = 'try:\n ' + '\n '.join(
self.content) + '\nexcept:\n import traceback; traceback.print_exc()'
else:
text = '\n'.join(self.content)
print('# %%')
print(text)
print()
return []
@pytest.fixture(scope="function")
def nb_lax(pytestconfig):
kwargs, other_args = gather_config_options(pytestconfig)
return NBLaxFixture(**kwargs)
@pytest.fixture(params=TUTORIALS, ids=[t.name for t in TUTORIALS])
def tutorial_code(request):
filename = request.param
with change_to_directory(TUT_DIR):
code = io.StringIO()
register_directive('jupyter-execute', CodeCell)
with open(filename, 'rt') as f:
original = sys.stdout
sys.stdout = code
publish_doctree(f.read(), settings_overrides={'report_level': 42})
sys.stdout = original
code.seek(0)
source_fn = Path(f'{str(filename).replace(".rst", "_rst")}_extracted.py')
with open(source_fn, 'wt') as source:
# filter line magics
source.write(''.join([line for line in code.readlines() if not line.startswith('%')]))
return request.param, source_fn
def test_tutorial(tutorial_code):
filename, source_module_path = tutorial_code
# make sure (picture) resources can be loaded as in sphinx-build
with change_to_directory(TUT_DIR):
def _run():
loader = importlib.machinery.SourceFileLoader(source_module_path.stem, str(source_module_path))
spec = importlib.util.spec_from_loader(loader.name, loader)
mod = importlib.util.module_from_spec(spec)
loader.exec_module(mod)
try:
# wrap module execution in hacks to auto-close Qt-Apps, etc.
_test_demo(_run)
except Exception as e:
print(f'Failed: {source_module_path}')
raise e
@pytest.mark.parametrize("filename", TUTORIALS, ids=[t.name for t in TUTORIALS])
def test_check(filename, nb_lax):
nb_lax.check(filename)
if __name__ == "__main__":
......
......@@ -13,6 +13,7 @@ pyqt5-qt5==5.15.2
pyqt5==5.15.7
pytest-cov==3.0.0
pytest-memprof==0.2.0
pytest-notebook==0.8.0
pytest-parallel==0.1.1
pytest-regressions==2.3.1
pytest-xdist==2.5.0
......
......@@ -58,7 +58,7 @@ tag_prefix = ''
testpaths = src/pymortests/
python_files = src/pymortests/*.py
# exclude misbehaving plugin from auto load
addopts=-p no:nb_regression -p no:notebook -p no:memprof -p no:pytest-qt --durations 10
addopts=-p no:nb_regression -p no:memprof -p no:pytest-qt --durations 10
junit_family=xunit2
[metadata]
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment