[tests] turn missing records for check_results into an exception

This allows to exercise the code in a test function

.gitignore

@@ -74,3 +74,4 @@ Pipfile.lock
 .hypothesis
 .vscode
 .ci/create_conda_env.log
+src/pymortests/testdata/check_results/test_check_check_results_miss*

src/pymor/core/exceptions.py

@@ -105,3 +105,9 @@ class UnpicklableError(Exception):
 
     def __str__(self):
         return f'{self.cls} cannot be pickled.'
+
+
+class NoResultDataError(Exception):
+    def __init__(self, msg=None):
+        msg = msg or 'missing testdata, re-run test'
+        super().__init__(msg)

src/pymortests/base.py

@@ -16,7 +16,7 @@ from pytest import skip
 
 from pymor.algorithms.basic import almost_equal, relative_error
 from pymor.core import config
-from pymor.core.exceptions import DependencyMissingError
+from pymor.core.exceptions import DependencyMissingError, NoResultDataError
 
 
 def runmodule(filename):
@@ -61,8 +61,8 @@ def check_results(test_name, params, results, *args):
         if not os.path.exists(testname_dir):
             os.mkdir(testname_dir)
         _dump_results(filename, results)
-        assert False, \
-            f'No results found for test {test_name} ({params}), saved current results. Remember to check in {filename}.'
+        raise NoResultDataError(msg=f'No results found for test {test_name} ({params}), saved current results.'
+                                        f'Remember to check in {filename}.')
 
     for k, (atol, rtol) in keys.items():
         if not np.all(np.allclose(old_results[k], results[k], atol=atol, rtol=rtol)):

src/pymortests/demos.py

 # This file is part of the pyMOR project (https://www.pymor.org).
 # Copyright pyMOR developers and contributors. All rights reserved.
 # License: BSD 2-Clause License (https://opensource.org/licenses/BSD-2-Clause)
-
+import math
 import os
 import shutil
 from importlib import import_module
@@ -13,7 +13,13 @@ from typer.testing import CliRunner
 
 import pymordemos  # noqa: F401
 from pymor.core.config import is_macos_platform, is_windows_platform
-from pymor.core.exceptions import GmshMissingError, MeshioMissingError, QtMissingError, TorchMissingError
+from pymor.core.exceptions import (
+    GmshMissingError,
+    MeshioMissingError,
+    NoResultDataError,
+    QtMissingError,
+    TorchMissingError,
+)
 from pymor.tools.mpi import parallel
 from pymortests.base import check_results, runmodule
 
@@ -367,5 +373,18 @@ def test_parabolic_mor_results():
                   'min_effectivities', 'max_effectivities', 'errors')
 
 
+def test_check_check_results_missing(tmp_path):
+    test_name = tmp_path.name
+    args = ['NONE', tmp_path]
+    results = {"error": math.pi}
+    with pytest.raises(NoResultDataError):
+        check_results(test_name, args, results, "error")
+    # running same check again against now recored data must be fine
+    check_results(test_name, args, results, "error")
+    with pytest.raises(AssertionError):
+        results["error"] += 1
+        check_results(test_name, args, results, "error")
+
+
 if __name__ == '__main__':
     runmodule(filename=__file__)