Merge pull request #1827 from pymor/ci_more_cov

Improve test coverage

.codecov.yml

@@ -15,12 +15,25 @@ coverage:
   status:
     changes: false
     project:
-      pymor:
-        # since we have no policy on this might as well turn this off entirely
-        target: 0%
-        flags:
-          - gitlab_ci
-          - github_actions
+        default: false
+        pymor:
+            # since we have no policy on this might as well turn this off entirely
+            target: 0%
+            flags:
+                - gitlab_ci
+                - github_actions
+            paths:
+                - "src/"
+                - "!src/pymortests/"
+                - "!src/pymor/version.py"
+        tests:
+            # if test coverage decreases, something very wrong is happening in CI
+            target: 95%
+            flags:
+                - gitlab_ci
+                - github_actions
+            paths:
+                - "src/pymortests/"
     patch:
       pymor:
         target: 0%

.gitignore

@@ -74,3 +74,4 @@ Pipfile.lock
 .hypothesis
 .vscode
 .ci/create_conda_env.log
+src/pymortests/testdata/check_results/test_check_check_results_miss*

.pre-commit-config.yaml

@@ -18,8 +18,7 @@ repos:
   rev: 'v0.0.219'
   hooks:
     - id: ruff
-      # Respect `exclude` and `extend-exclude` settings.
-      args: ["--force-exclude", "--fix"]
+      args: ["--fix"]
 - repo: https://github.com/DavidAnson/markdownlint-cli2
   rev: v0.4.0
   hooks:

setup.cfg

@@ -70,6 +70,8 @@ license_file = LICENSE.txt
 omit =
   expression
   */expression
+  src/pymor/version.py
+  pymor/version
 
 [coverage:run]
 relative_files = True
@@ -78,6 +80,8 @@ source_pkgs = pymor
 omit =
   expression
   */expression
+  src/pymor/version.py
+  pymor/version
 
 # each entry is a list of paths that coverage will consider to be equivalent
 # this allows to map source files if pymor is imported from installed package

src/pymor/core/exceptions.py

@@ -105,3 +105,9 @@ class UnpicklableError(Exception):
 
     def __str__(self):
         return f'{self.cls} cannot be pickled.'
+
+
+class NoResultDataError(Exception):
+    def __init__(self, msg=None):
+        msg = msg or 'missing testdata, re-run test'
+        super().__init__(msg)

src/pymor/discretizers/fenics/cg.py

@@ -66,7 +66,10 @@ def discretize_stationary_cg(analytical_problem, diameter=None, degree=1, preass
     if p.dirichlet_data is not None and p.dirichlet_data.parametric:
         raise NotImplementedError
 
-    mesh, (boundary_mask, boundary_ids) = discretize_domain(p.domain, diameter=diameter)
+    if diameter is None:
+        mesh, (boundary_mask, boundary_ids) = discretize_domain(p.domain)
+    else:
+        mesh, (boundary_mask, boundary_ids) = discretize_domain(p.domain, diameter=diameter)
 
     V = df.FunctionSpace(mesh, 'Lagrange', degree)
     bc = df.DirichletBC(V, 0. if p.dirichlet_data is None else p.dirichlet_data.to_fenics(mesh)[0].item(),

src/pymor/scripts/pymor_demo.py

@@ -3,7 +3,6 @@
 # Copyright pyMOR developers and contributors. All rights reserved.
 # License: BSD 2-Clause License (https://opensource.org/licenses/BSD-2-Clause)
 
-from __future__ import absolute_import, division, print_function
 
 import importlib
 import pkgutil

src/pymordemos/neural_networks_instationary.py

@@ -178,7 +178,7 @@ def create_fom(problem_number, grid_intervals, time_steps):
         fom, _ = discretize_instationary_fv(problem, diameter=1. / grid_intervals, nt=time_steps)
         plot_function = fom.visualize
     else:
-        assert False
+        raise ValueError(f'Unknown problem number {problem_number}')
 
     return fom, plot_function
 

src/pymortests/base.py

@@ -16,7 +16,7 @@ from pytest import skip
 
 from pymor.algorithms.basic import almost_equal, relative_error
 from pymor.core import config
-from pymor.core.exceptions import DependencyMissingError
+from pymor.core.exceptions import DependencyMissingError, NoResultDataError
 
 
 def runmodule(filename):
@@ -61,8 +61,8 @@ def check_results(test_name, params, results, *args):
         if not os.path.exists(testname_dir):
             os.mkdir(testname_dir)
         _dump_results(filename, results)
-        assert False, \
-            f'No results found for test {test_name} ({params}), saved current results. Remember to check in {filename}.'
+        raise NoResultDataError(msg=f'No results found for test {test_name} ({params}), saved current results.'
+                                        f'Remember to check in {filename}.')
 
     for k, (atol, rtol) in keys.items():
         if not np.all(np.allclose(old_results[k], results[k], atol=atol, rtol=rtol)):

src/pymortests/demos.py

 # This file is part of the pyMOR project (https://www.pymor.org).
 # Copyright pyMOR developers and contributors. All rights reserved.
 # License: BSD 2-Clause License (https://opensource.org/licenses/BSD-2-Clause)
-
+import math
 import os
 import shutil
 from importlib import import_module
@@ -13,7 +13,13 @@ from typer.testing import CliRunner
 
 import pymordemos  # noqa: F401
 from pymor.core.config import is_macos_platform, is_windows_platform
-from pymor.core.exceptions import GmshMissingError, MeshioMissingError, QtMissingError, TorchMissingError
+from pymor.core.exceptions import (
+    GmshMissingError,
+    MeshioMissingError,
+    NoResultDataError,
+    QtMissingError,
+    TorchMissingError,
+)
 from pymor.tools.mpi import parallel
 from pymortests.base import check_results, runmodule
 
@@ -211,14 +217,14 @@ def _test_demo(demo):
         pass
 
     try:
-        from matplotlib import pyplot
+        from matplotlib import pyplot as plt
         if sys.version_info[:2] > (3, 7) or (
                 sys.version_info[0] == 3 and sys.version_info[1] == 6):
-            pyplot.ion()
+            plt.ion()
         else:
             # the ion switch results in interpreter segfaults during multiple
             # demo tests on 3.7 -> fall back on old monkeying solution
-            pyplot.show = nop
+            plt.show = nop
     except ImportError:
         pass
     try:
@@ -244,8 +250,8 @@ def _test_demo(demo):
         from pymor.parallel.default import _cleanup
         _cleanup()
         try:
-            from matplotlib import pyplot
-            pyplot.close('all')
+            from matplotlib import pyplot as plt
+            plt.close('all')
         except ImportError:
             pass
 
@@ -367,5 +373,18 @@ def test_parabolic_mor_results():
                   'min_effectivities', 'max_effectivities', 'errors')
 
 
+def test_check_check_results_missing(tmp_path):
+    test_name = tmp_path.name
+    args = ['NONE', tmp_path]
+    results = {'error': math.pi}
+    with pytest.raises(NoResultDataError):
+        check_results(test_name, args, results, 'error')
+    # running same check again against now recored data must be fine
+    check_results(test_name, args, results, 'error')
+    with pytest.raises(AssertionError):
+        results['error'] += 1
+        check_results(test_name, args, results, 'error')
+
+
 if __name__ == '__main__':
     runmodule(filename=__file__)