pytest-regtest 2.2.0__py2.py3-none-any.whl

pytest_regtest/__init__.py

@@ -0,0 +1,96 @@
+from importlib.metadata import version as _version
+
+import pytest
+
+from .pytest_regtest import clear_converters  # noqa: F401
+from .pytest_regtest import patch_terminal_size  # noqa: F401
+from .pytest_regtest import register_converter_post  # noqa: F401
+from .pytest_regtest import register_converter_pre  # noqa: F401
+from .pytest_regtest import (
+    PytestRegtestCommonHooks,
+    PytestRegtestPlugin,
+    RegtestStream,
+    Snapshot,
+    SnapshotPlugin,
+)
+from .register_third_party_handlers import (
+    register_numpy_handler,
+    register_pandas_handler,
+)
+
+__version__ = _version(__package__)
+
+
+def pytest_addoption(parser):
+    """Add options to control the timeout plugin"""
+    group = parser.getgroup("regtest", "regression test plugin")
+    group.addoption(
+        "--regtest-reset",
+        action="store_true",
+        help="do not run regtest but record current output",
+    )
+    group.addoption(
+        "--regtest-tee",
+        action="store_true",
+        default=False,
+        help="print recorded results to console too",
+    )
+    group.addoption(
+        "--regtest-consider-line-endings",
+        action="store_true",
+        default=False,
+        help="do not strip whitespaces at end of recorded lines",
+    )
+    group.addoption(
+        "--regtest-nodiff",
+        action="store_true",
+        default=False,
+        help="do not show diff output for failed regresson tests",
+    )
+    group.addoption(
+        "--regtest-disable-stdconv",
+        action="store_true",
+        default=False,
+        help=(
+            "do not apply standard output converters to clean up indeterministic output"
+        ),
+    )
+
+
+def pytest_configure(config):
+    common = PytestRegtestCommonHooks()
+    config.pluginmanager.register(common)
+    config.pluginmanager.register(PytestRegtestPlugin(common))
+    config.pluginmanager.register(SnapshotPlugin(common))
+
+
+@pytest.fixture
+def regtest(request):
+    yield RegtestStream(request)
+
+
+@pytest.fixture
+def snapshot(request):
+    yield Snapshot(request)
+
+
+@pytest.fixture
+def regtest_all(regtest):
+    yield regtest
+
+
+snapshot_all_output = regtest_all
+
+try:
+    import pandas  # noqa: F401
+
+    register_pandas_handler()
+except ImportError:
+    pass
+
+try:
+    import numpy  # noqa: F401
+
+    register_numpy_handler()
+except ImportError:
+    pass

pytest_regtest/numpy_handler.py

@@ -0,0 +1,209 @@
+import difflib
+import io
+import os.path
+import warnings
+
+import numpy as np
+
+from .snapshot_handler import BaseSnapshotHandler
+from .utils import highlight_mismatches
+
+
+class NumpyHandler(BaseSnapshotHandler):
+    def __init__(self, handler_options, pytest_config, tw):
+        self.print_options = np.get_printoptions() | handler_options.get(
+            "print_options", {}
+        )
+        self.atol = handler_options.get("atol", 0.0)
+        self.rtol = handler_options.get("rtol", 0.0)
+        self.equal_nan = handler_options.get("equal_nan", True)
+
+    def _filename(self, folder):
+        return os.path.join(folder, "arrays.npy")
+
+    def save(self, folder, obj):
+        np.save(self._filename(folder), obj)
+
+    def load(self, folder):
+        return np.load(self._filename(folder))
+
+    def show(self, obj):
+        stream = io.StringIO()
+        with np.printoptions(**self.print_options):
+            print(obj, file=stream)
+        return stream.getvalue().splitlines()
+
+    def compare(self, current_obj, recorded_obj):
+        return (
+            isinstance(current_obj, np.ndarray)
+            and current_obj.shape == recorded_obj.shape
+            and current_obj.dtype == recorded_obj.dtype
+            and np.allclose(
+                recorded_obj,
+                current_obj,
+                atol=self.atol,
+                rtol=self.rtol,
+                equal_nan=self.equal_nan,
+            )
+        )
+
+    def show_differences(self, current_obj, recorded_obj, has_markup):
+        lines = []
+
+        if recorded_obj.dtype != current_obj.dtype:
+            lines.extend(
+                [
+                    f"dtype mismatch: current dtype: {current_obj.dtype}",
+                    f"               recorded dtype: {recorded_obj.dtype}",
+                ]
+            )
+
+        recorded_as_text = self.show(recorded_obj)
+        current_as_text = self.show(current_obj)
+
+        if recorded_obj.shape == current_obj.shape:
+            if np.allclose(current_obj, recorded_obj, rtol=self.rtol, atol=self.atol):
+                return lines or None
+
+            lines.extend(self.error_diagnostics(recorded_obj, current_obj))
+
+        else:
+            lines.extend(
+                [
+                    f"shape mismatch: current shape: {current_obj.shape}",
+                    f"               recorded shape: {recorded_obj.shape}",
+                ]
+            )
+
+        if recorded_obj.ndim > 2:
+            return lines
+
+        if recorded_obj.ndim == 1:
+            diff_lines = list(
+                difflib.unified_diff(
+                    current_as_text,
+                    recorded_as_text,
+                    "current",
+                    "expected",
+                    lineterm="",
+                )
+            )
+            lines.append("")
+            lines.extend(diff_lines)
+
+        else:
+            diff_lines = self.error_diagnostics_2d_linewise(
+                current_obj,
+                current_as_text,
+                recorded_obj,
+                recorded_as_text,
+                has_markup,
+            )
+            lines.extend(diff_lines)
+
+        if not diff_lines:
+            lines.append("diff is empty, you may want to change the print options")
+
+        return lines
+
+    def error_diagnostics(self, recorded_obj, current_obj):
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore", RuntimeWarning)
+            rel_err = np.abs(current_obj - recorded_obj) / recorded_obj
+        rel_err[(recorded_obj == 0) * (current_obj == recorded_obj)] = 0.0
+        rel_err_max_1 = np.max(rel_err)
+        rel_err_max_2 = np.max(rel_err[recorded_obj != 0])
+
+        abs_err = np.abs(current_obj - recorded_obj)
+        abs_err_max = np.max(abs_err)
+
+        lines = []
+
+        if rel_err_max_1 == rel_err_max_2:
+            lines.append(f"max relative deviation: {rel_err_max_1:e}")
+        else:
+            lines.append(f"max relative deviation: {rel_err_max_1:e}")
+            lines.append(f"max relative deviation except inf: {rel_err_max_2:e}")
+
+        lines.append(f"max absolute deviation: {abs_err_max:e}")
+
+        n_diff = np.sum(
+            np.logical_not(
+                np.isclose(current_obj, recorded_obj, rtol=self.rtol, atol=self.atol)
+            )
+        )
+
+        lines.append(
+            f"both arrays differ in {n_diff} out of {np.prod(recorded_obj.shape)}"
+            " entries"
+        )
+        lines.append(
+            f"up to given precision settings rtol={self.rtol:e} and"
+            f" atol={self.atol:e}"
+        )
+
+        return lines
+
+    def error_diagnostics_2d_linewise(
+        self, current_obj, current_as_text, recorded_obj, recorded_as_text, has_markup
+    ):
+        sub_diff = []
+
+        for i, (l1, l2, r1, r2) in enumerate(
+            zip(current_as_text, recorded_as_text, current_obj, recorded_obj)
+        ):
+            if r1.shape == r2.shape and np.allclose(
+                r1, r2, rtol=self.rtol, atol=self.atol
+            ):
+                continue
+
+            if r1.shape == r2.shape:
+                # enforces more uniform formatting of both lines:
+                rows_together = np.vstack((r1, r2))
+                lines_together = self.show(rows_together)
+                line_diff = list(
+                    difflib.unified_diff(
+                        [lines_together[0][1:].strip()],
+                        [lines_together[1][:-1].strip()],
+                        "current",
+                        "expected",
+                        lineterm="",
+                    )
+                )
+            else:
+                row_1 = self.show(r1)
+                row_2 = self.show(r2)
+                line_diff = list(
+                    difflib.unified_diff(
+                        row_1,
+                        row_2,
+                        "current",
+                        "expected",
+                        lineterm="",
+                    )
+                )
+
+            if line_diff:
+                if not sub_diff:
+                    sub_diff = line_diff[:2]
+
+                l1, l2 = line_diff[-2], line_diff[-1]
+                if has_markup:
+                    l1, l2 = highlight_mismatches(l1, l2)
+
+                sub_diff.append(f"row {i:3d}: {l1}")
+                sub_diff.append(f"         {l2}")
+
+        missing = len(current_as_text) - len(recorded_as_text)
+        if missing > 0:
+            for i, row in enumerate(current_as_text[-missing:], len(recorded_as_text)):
+                # remove duplicate brackets
+                row = row.rstrip("]") + "]"
+                sub_diff.append(f"row {i:3d}: -{row.lstrip()}")
+        if missing < 0:
+            for i, row in enumerate(recorded_as_text[missing:], len(current_as_text)):
+                # remove duplicate brackets
+                row = row.rstrip("]") + "]"
+                sub_diff.append(f"row {i:3d}: +{row.lstrip()}")
+
+        return sub_diff

pytest_regtest/pandas_handler.py

@@ -0,0 +1,135 @@
+import difflib
+import io
+import os.path
+
+import numpy as np
+import pandas as pd
+
+from .snapshot_handler import BaseSnapshotHandler
+
+
+class DataFrameHandler(BaseSnapshotHandler):
+    def __init__(self, handler_options, pytest_config, tw):
+        # default contains a few nested dicts and we flatten those, e.g.
+        # { "html": {"border": 1} } -> { "html.border": 1 }
+        default = list(pd.options.display.d.items())
+        default_flattened = {}
+        for k, v in default:
+            if isinstance(v, dict):
+                for k0, v0 in v.items():
+                    default_flattened[f"{k}.{k0}"] = v0
+            else:
+                default_flattened[k] = v
+
+        # overwrite with user settings:
+        items = (default_flattened | handler_options.get("display_options", {})).items()
+
+        # flatten items as required by pandas.option_context:
+        self.display_options_flat = [
+            entry for item in items for entry in (f"display.{item[0]}", item[1])
+        ]
+        self.atol = handler_options.get("atol", 0.0)
+        self.rtol = handler_options.get("rtol", 0.0)
+
+    def _filename(self, folder):
+        return os.path.join(folder, "dataframe.pkl")
+
+    def save(self, folder, obj):
+        obj.to_pickle(self._filename(folder), compression="gzip")
+
+    def load(self, folder):
+        return pd.read_pickle(self._filename(folder), compression="gzip")
+
+    def show(self, obj):
+        stream = io.StringIO()
+        with pd.option_context(*self.display_options_flat):
+            print(obj, file=stream)
+        return stream.getvalue().splitlines()
+
+    def compare(self, current, recorded):
+        missing = set(
+            n
+            for (n, t) in set(zip(recorded.columns, recorded.dtypes))
+            ^ set(zip(current.columns, current.dtypes))
+        )
+
+        if missing:
+            return False
+
+        common = set(
+            n
+            for (n, t) in set(zip(recorded.columns, recorded.dtypes))
+            & set(zip(current.columns, current.dtypes))
+        )
+        current_reduced = current[[n for n in current.columns if n in common]]
+        recorded_reduced = recorded[[n for n in recorded.columns if n in common]]
+
+        def extract(df, selector):
+            return df[[n for (n, t) in zip(df.columns, df.dtypes) if selector(t)]]
+
+        current_reduced_floats = extract(
+            current_reduced, lambda t: t.type is np.float64
+        ).to_numpy()
+
+        current_reduced_other = extract(
+            current_reduced, lambda t: t.type is not np.float64
+        )
+
+        recorded_reduced_floats = extract(
+            recorded_reduced, lambda t: t.type is np.float64
+        ).to_numpy()
+
+        recorded_reduced_other = extract(
+            recorded_reduced, lambda t: t.type is not np.float64
+        )
+
+        return np.allclose(
+            current_reduced_floats,
+            recorded_reduced_floats,
+            atol=self.atol,
+            rtol=self.rtol,
+            equal_nan=True,
+        ) and (current_reduced_other == recorded_reduced_other).all(axis=None)
+
+    def show_differences(self, current, recorded, has_markup):
+        lines = []
+
+        stream = io.StringIO()
+        current.info(buf=stream, verbose=True, memory_usage=False)
+        current_info = stream.getvalue().splitlines()[2:][:-1]
+
+        stream = io.StringIO()
+        recorded.info(buf=stream, verbose=True, memory_usage=False)
+        recorded_info = stream.getvalue().splitlines()[2:][:-1]
+
+        info_diff = list(
+            difflib.unified_diff(
+                current_info,
+                recorded_info,
+                "current",
+                "expected",
+                lineterm="",
+            )
+        )
+        lines.extend(info_diff)
+
+        recorded_as_text = self.show(recorded)
+        current_as_text = self.show(current)
+
+        diffs = list(
+            difflib.unified_diff(
+                current_as_text,
+                recorded_as_text,
+                "current",
+                "expected",
+                lineterm="",
+            )
+        )
+
+        lines.append("")
+        if diffs:
+            lines.extend(diffs)
+        else:
+            lines.append("diff is empty, you may want to change the print options")
+
+        return lines

pytest_regtest/pytest_regtest.py

@@ -0,0 +1,610 @@
+import difflib
+import functools
+import inspect
+import os
+import re
+import shutil
+import sys
+import tempfile
+from hashlib import sha512
+from io import StringIO
+
+import pytest
+from _pytest._code.code import TerminalRepr
+from _pytest._io import TerminalWriter
+
+from .snapshot_handler import PythonObjectHandler  # noqa: F401
+from .snapshot_handler import snapshot_handlers
+
+IS_WIN = sys.platform == "win32"
+
+
+def patch_terminal_size(w, h):
+    def get_terminal_size(fallback=None):
+        return w, h
+
+    shutil.get_terminal_size = get_terminal_size
+
+
+# we determine actual terminal size before pytest changes this:
+tw, _ = shutil.get_terminal_size()
+
+
+class RegtestException(Exception):
+    pass
+
+
+class RecordedOutputException(RegtestException):
+    pass
+
+
+class SnapshotException(RegtestException):
+    pass
+
+
+class PytestRegtestCommonHooks:
+    def __init__(self):
+        self._reset_snapshots = []
+        self._reset_regtest_outputs = []
+        self._failed_snapshots = []
+        self._failed_regtests = []
+
+    @pytest.hookimpl(hookwrapper=False)
+    def pytest_terminal_summary(self, terminalreporter, exitstatus, config):
+        tr = terminalreporter
+        tr.ensure_newline()
+        tr.section("pytest-regtest report", sep="-", blue=True, bold=True)
+        tr.write("total number of failed regression tests: ", bold=True)
+        tr.line(str(len(self._failed_regtests)))
+        tr.write("total number of failed snapshot tests  : ", bold=True)
+        tr.line(str(len(self._failed_snapshots)))
+
+        if config.getvalue("--regtest-reset"):
+            if config.option.verbose:
+                tr.line("the following output files have been reset:", bold=True)
+                for path in self._reset_regtest_outputs:
+                    rel_path = os.path.relpath(path)
+                    tr.line("  " + rel_path)
+                for path in self._reset_snapshots:
+                    rel_path = os.path.relpath(path)
+                    tr.line("  " + rel_path)
+            else:
+                tr.write("total number of reset output files: ", bold=True)
+                tr.line(
+                    str(len(self._reset_regtest_outputs) + len(self._reset_snapshots))
+                )
+
+    @pytest.hookimpl(hookwrapper=True)
+    def pytest_pyfunc_call(self, pyfuncitem):
+        stdout = sys.stdout
+        if "regtest_all" in pyfuncitem.fixturenames and hasattr(
+            pyfuncitem, "regtest_stream"
+        ):
+            sys.stdout = pyfuncitem.regtest_stream
+        yield
+        sys.stdout = stdout
+
+    @pytest.hookimpl(hookwrapper=True)
+    def pytest_report_teststatus(self, report, config):
+        outcome = yield
+        if report.when == "call" and "uses-regtest" in report.keywords:
+            if config.getvalue("--regtest-reset"):
+                result = outcome.get_result()
+                if result[0] != "failed":
+                    outcome.force_result((result[0], "R", "RESET"))
+
+
+class PytestRegtestPlugin:
+    def __init__(self, recorder):
+        self.recorder = recorder
+
+    @pytest.hookimpl(trylast=True)
+    def pytest_runtest_call(self, item):
+        if hasattr(item, "regtest_stream"):
+            output_exception = self.check_recorded_output(item)
+            if output_exception is not None:
+                raise output_exception
+
+        if item.get_closest_marker("xfail") and item.config.getvalue("--regtest-reset"):
+            # enforce consistency with xfail:
+            assert False
+
+    def check_recorded_output(self, item):
+        test_folder = item.fspath.dirname
+        regtest_stream = item.regtest_stream
+        version = regtest_stream.version
+        if not isinstance(regtest_stream, RegtestStream):
+            return
+
+        orig_identifer, recorded_output_path = result_file_paths(
+            test_folder, item.nodeid, version
+        )
+        config = item.config
+
+        consider_line_endings = config.getvalue("--regtest-consider-line-endings")
+        reset = config.getvalue("--regtest-reset")
+
+        if reset:
+            os.makedirs(os.path.dirname(recorded_output_path), exist_ok=True)
+            with open(recorded_output_path + ".out", "w", encoding="utf-8") as fh:
+                fh.write("".join(regtest_stream.get_lines()))
+            if orig_identifer is not None:
+                self.recorder._reset_regtest_outputs.append(
+                    recorded_output_path + ".item"
+                )
+                with open(recorded_output_path + ".item", "w") as fh:
+                    print(orig_identifer, file=fh)
+            self.recorder._reset_regtest_outputs.append(recorded_output_path + ".out")
+            return
+
+        if os.path.exists(recorded_output_path + ".out"):
+            with open(recorded_output_path + ".out", "r", encoding="utf-8") as fh:
+                tobe = fh.readlines()
+            recorded_output_file_exists = True
+        else:
+            tobe = []
+            recorded_output_file_exists = False
+
+        current = regtest_stream.get_lines()
+        if consider_line_endings:
+            current = [repr(line.rstrip("\n")) for line in current]
+            tobe = [repr(line.rstrip("\n")) for line in tobe]
+        else:
+            current = [line.rstrip() for line in current]
+            tobe = [line.rstrip() for line in tobe]
+
+        if current != tobe:
+            self.recorder._failed_regtests.append(item)
+            return RecordedOutputException(
+                current,
+                tobe,
+                recorded_output_path,
+                regtest_stream,
+                recorded_output_file_exists,
+            )
+
+    @pytest.hookimpl(hookwrapper=True)
+    def pytest_runtest_makereport(self, item, call):
+        outcome = yield
+        result = outcome.get_result()
+        if call.when == "teardown" and hasattr(item, "regtest_stream"):
+            if item.config.getvalue("--regtest-tee"):
+                tw = TerminalWriter()
+                output = item.regtest_stream.get_output()
+
+                if output:
+                    tw.line()
+                    line = "recorded raw output to regtest fixture: "
+                    line = line.ljust(tw.fullwidth, "-")
+                    tw.line(line, green=True)
+                    tw.write(item.regtest_stream.get_output() + "\n", cyan=True)
+                    line = "-" * tw.fullwidth
+                    tw.line(line, green=True)
+
+        if call.when != "call" or not getattr(item, "regtest", False):
+            return
+
+        result.keywords["uses-regtest"] = True
+
+        if call.excinfo is not None:
+            all_lines, all_colors = [], []
+            if call.excinfo.type is RecordedOutputException:
+                output_exception = call.excinfo
+                if output_exception is not None:
+                    lines, colors = self._handle_regtest_exception(
+                        item, output_exception.value.args, result
+                    )
+                    all_lines.extend(lines)
+                    all_colors.extend(colors)
+
+            else:
+                return
+
+            result.longrepr = CollectErrorRepr(all_lines, all_colors)
+
+    def _handle_regtest_exception(self, item, exc_args, result):
+        (
+            current,
+            recorded,
+            recorded_output_path,
+            regtest_stream,
+            recorded_output_file_exists,
+        ) = exc_args
+
+        nodeid = item.nodeid + (
+            "" if regtest_stream.version is None else "__" + regtest_stream.version
+        )
+        if not recorded_output_file_exists:
+            msg = "\nregression test output not recorded yet for {}:\n".format(nodeid)
+            return (
+                [msg] + current,
+                [dict()] + len(current) * [dict(red=True, bold=True)],
+            )
+
+        nodiff = item.config.getvalue("--regtest-nodiff")
+        diffs = list(
+            difflib.unified_diff(current, recorded, "current", "expected", lineterm="")
+        )
+
+        msg = "\nregression test output differences for {}:\n".format(nodeid)
+
+        if nodiff:
+            msg_diff = f"    {len(diffs)} lines in diff"
+        else:
+            recorded_output_path = os.path.relpath(recorded_output_path)
+            msg += f"    (recorded output from {recorded_output_path})\n"
+            msg_diff = "    >   " + "\n    >   ".join(diffs)
+
+        return [msg, msg_diff + "\n"], [dict(), dict(red=True, bold=True)]
+
+
+class SnapshotPlugin:
+    def __init__(self, recorder):
+        self.recorder = recorder
+
+    @pytest.hookimpl(trylast=True)
+    def pytest_runtest_call(self, item):
+        if hasattr(item, "snapshot"):
+            snapshot_exception = self.check_snapshots(item)
+            if snapshot_exception is not None:
+                raise snapshot_exception
+
+        if item.get_closest_marker("xfail") and item.config.getvalue("--regtest-reset"):
+            # enforce fail
+            assert False
+
+    def check_snapshots(self, item):
+        results = []
+
+        any_failed = False
+        for idx, snapshot in enumerate(item.snapshot.snapshots):
+            is_recorded, ok, msg = self.check_snapshot(idx, item, snapshot)
+            if not ok:
+                any_failed = True
+            results.append((ok, snapshot, is_recorded, msg))
+
+        if any_failed:
+            self.recorder._failed_snapshots.append(item)
+            return SnapshotException(results)
+
+    def check_snapshot(self, idx, item, snapshot):
+        obj, version, handler_options, _ = snapshot
+        handler = get_snapshot_handler(obj, handler_options, item.config)
+
+        test_folder = item.fspath.dirname
+        if version is not None:
+            identifier = str(version) + "__" + str(idx)
+        else:
+            identifier = str(idx)
+
+        config = item.config
+
+        orig_identifer, recorded_output_path = result_file_paths(
+            test_folder, item.nodeid, identifier
+        )
+
+        reset = config.getvalue("--regtest-reset")
+
+        if reset:
+            os.makedirs(recorded_output_path, exist_ok=True)
+            handler.save(recorded_output_path, obj)
+            if orig_identifer is not None:
+                self.recorder._reset_snapshots.append(recorded_output_path + ".item")
+                with open(recorded_output_path + ".item", "w") as fh:
+                    print(orig_identifer, file=fh)
+            self.recorder._reset_snapshots.append(recorded_output_path)
+            return True, True, None
+
+        has_markup = item.config.get_terminal_writer().hasmarkup
+        if os.path.exists(recorded_output_path):
+            recorded_obj = handler.load(recorded_output_path)
+            ok = handler.compare(obj, recorded_obj)
+            if ok:
+                return True, True, None
+            msg = handler.show_differences(obj, recorded_obj, has_markup)
+            return True, False, msg
+
+        msg = handler.show(obj)
+        return False, False, msg
+
+    @pytest.hookimpl(hookwrapper=True)
+    def pytest_runtest_makereport(self, item, call):
+        outcome = yield
+        result = outcome.get_result()
+        if call.when == "teardown" and hasattr(item, "snapshot"):
+            if item.config.getvalue("--regtest-tee"):
+                tw = TerminalWriter()
+                snapshots = item.snapshot.snapshots
+                if not snapshots:
+                    return
+
+                tw.line()
+                line = "recorded snapshots: "
+                line = line.ljust(tw.fullwidth, "-")
+                tw.line(line, green=True)
+                path = item.fspath.relto(item.session.fspath)
+                code_lines = item.fspath.readlines()
+
+                for obj, version, handler_options, line_no in snapshots:
+                    info = code_lines[line_no - 1].strip()
+                    tw.line(f"> {path} +{line_no}")
+                    tw.line(f"> {info}")
+                    handler = get_snapshot_handler(obj, handler_options, item.config)
+                    lines = handler.show(obj)
+                    for line in lines:
+                        tw.line(line, cyan=True)
+                tw.line("-" * tw.fullwidth, green=True)
+                tw.line()
+                tw.flush()
+
+        if call.when != "call" or not hasattr(item, "snapshot"):
+            return
+
+        result.keywords["uses-regtest"] = True
+
+        if call.excinfo is not None:
+            all_lines, all_colors = [], []
+            if call.excinfo.type is SnapshotException:
+                snapshot_exception = call.excinfo
+                if snapshot_exception is not None:
+                    lines, colors = self._handle_snapshot_exception(
+                        item, snapshot_exception.value.args, result
+                    )
+                    all_lines.extend(lines)
+                    all_colors.extend(colors)
+            else:
+                return
+
+            result.longrepr = CollectErrorRepr(all_lines, all_colors)
+
+    def _handle_snapshot_exception(self, item, exc_args, result):
+        snapshot = item.snapshot
+        lines = []
+        colors = []
+
+        code_lines = item.fspath.readlines()
+
+        NO_COLOR = dict()
+        RED = dict(red=True, bold=True)
+        GREEN = dict(green=True, bold=False)
+
+        headline = "\nsnapshot error(s) for {}:".format(item.nodeid)
+        lines.append(headline)
+        colors.append(NO_COLOR)
+
+        for ok, snapshot, is_recorded, msg in exc_args[0]:
+            obj, version, kw, line_no = snapshot
+            info = code_lines[line_no - 1].strip()
+
+            path = item.fspath.relto(item.session.fspath)
+            if ok:
+                lines.append("\nsnapshot ok:")
+                lines.append(f"    > {path} +{line_no}")
+                lines.append(f"    > {info}")
+                colors.append(GREEN)
+                colors.append(NO_COLOR)
+                colors.append(NO_COLOR)
+            elif is_recorded:
+                lines.append("\nsnapshot mismatch:")
+                lines.append(f"    > {path} +{line_no}:")
+                lines.append(f"    > {info}")
+                colors.append(RED)
+                colors.append(NO_COLOR)
+                colors.append(NO_COLOR)
+                nodiff = item.config.getvalue("--regtest-nodiff")
+                if nodiff:
+                    lines.append(f"    {len(msg)} lines in report")
+                    colors.append(RED)
+                else:
+                    lines.extend("    " + ll for ll in msg)
+                    colors.extend(len(msg) * [RED])
+            else:
+                headline = "\nsnapshot not recorded yet:"
+                lines.append(headline)
+                colors.append(NO_COLOR)
+                lines.append("    > " + info.strip())
+                colors.append(RED)
+                lines.extend("    " + ll for ll in msg)
+                colors.extend(len(msg) * [RED])
+
+        return lines, colors
+
+
+def result_file_paths(test_folder, nodeid, version):
+    file_path, __, test_function_name = nodeid.partition("::")
+    file_name = os.path.basename(file_path)
+
+    orig_test_function_identifier = f"{file_name}::{test_function_name}"
+
+    for c in "/\\:*\"'?<>|":
+        test_function_name = test_function_name.replace(c, "-")
+
+    # If file name is too long, hash parameters.
+    if len(test_function_name) > 100:
+        test_function_name = (
+            test_function_name[:88]
+            + "__"
+            + sha512(test_function_name.encode("utf-8")).hexdigest()[:10]
+        )
+    else:
+        orig_test_function_identifier = None
+
+    test_function_name = test_function_name.replace(" ", "_")
+    stem, __ = os.path.splitext(file_name)
+    if version is not None:
+        output_file_name = stem + "." + test_function_name + "__" + str(version)
+    else:
+        output_file_name = stem + "." + test_function_name
+
+    return orig_test_function_identifier, os.path.join(
+        test_folder, "_regtest_outputs", output_file_name
+    )
+
+
+class RegtestStream:
+    def __init__(self, request):
+        request.node.regtest_stream = self
+        request.node.regtest = True
+        self.request = request
+        self.buffer = StringIO()
+        self.version = None
+
+        self.snapshots = []
+
+    def write(self, what):
+        self.buffer.write(what)
+
+    def flush(self):
+        pass
+
+    def get_lines(self):
+        output = self.buffer.getvalue()
+        if not output:
+            return []
+        output = cleanup(output, self.request)
+        lines = output.splitlines(keepends=True)
+        return lines
+
+    def get_output(self):
+        return self.buffer.getvalue()
+
+    def __enter__(self):
+        sys.stdout = self
+        return self
+
+    def __exit__(self, *a):
+        sys.stdout = sys.__stdout__
+        return False  # dont suppress exception
+
+
+class Snapshot:
+    def __init__(self, request):
+        request.node.snapshot = self
+        request.node.regtest = True
+        self.request = request
+        self.buffer = StringIO()
+
+        self.snapshots = []
+
+    def check(self, obj, *, version=None, **kw):
+        line_no = inspect.currentframe().f_back.f_lineno
+        self.snapshots.append((obj, version, kw, line_no))
+
+
+def cleanup(output, request):
+    for converter in _converters_pre:
+        output = converter(output, request)
+
+    if not request.config.getvalue("--regtest-disable-stdconv"):
+        output = _std_conversion(output, request)
+
+    for converter in _converters_post:
+        output = converter(output, request)
+
+    # in python 3 a string should not contain binary symbols...:
+    if contains_binary(output):
+        request.raiseerror(
+            "recorded output for regression test contains unprintable characters."
+        )
+
+    return output
+
+
+# the function below is modified version of http://stackoverflow.com/questions/898669/
+textchars = bytearray({7, 8, 9, 10, 12, 13, 27} | set(range(0x20, 0x100)) - {0x7F})
+
+
+def contains_binary(txt):
+    return bool(txt.translate(dict(zip(textchars, " " * 9999))).replace(" ", ""))
+
+
+_converters_pre = []
+_converters_post = []
+
+
+def clear_converters():
+    _converters_pre.clear()
+    _converters_post.clear()
+
+
+def _fix_pre_v2_converter_function(function):
+    @functools.wraps(function)
+    def fixed_converter_function(output, request):
+        return function(output)
+
+    return fixed_converter_function
+
+
+def register_converter_pre(function):
+    if function not in _converters_pre:
+        signature = inspect.signature(function)
+        # keep downward compatibility:
+        if len(signature.parameters) == 1:
+            function = _fix_pre_v2_converter_function(function)
+        _converters_pre.append(function)
+
+
+def register_converter_post(function):
+    if function not in _converters_post:
+        signature = inspect.signature(function)
+        # keep downward compatibility:
+        if len(signature.parameters) == 1:
+            function = _fix_pre_v2_converter_function(function)
+        _converters_post.append(function)
+
+
+def _std_replacements(request):
+    if "tmpdir" in request.fixturenames:
+        tmpdir = request.getfixturevalue("tmpdir").strpath + os.path.sep
+        yield tmpdir, "<tmpdir_from_fixture>/"
+        tmpdir = request.getfixturevalue("tmpdir").strpath
+        yield tmpdir, "<tmpdir_from_fixture>"
+
+    regexp = os.path.join(
+        os.path.realpath(tempfile.gettempdir()), "pytest-of-.*", r"pytest-\d+/"
+    )
+    yield regexp, "<pytest_tempdir>/"
+
+    regexp = os.path.join(tempfile.gettempdir(), "tmp[_a-zA-Z0-9]+")
+
+    yield regexp, "<tmpdir_from_tempfile_module>"
+    yield (
+        os.path.realpath(tempfile.gettempdir()) + os.path.sep,
+        "<tmpdir_from_tempfile_module>/",
+    )
+    yield os.path.realpath(tempfile.gettempdir()), "<tmpdir_from_tempfile_module>"
+    yield tempfile.tempdir + os.path.sep, "<tmpdir_from_tempfile_module>/"
+    yield tempfile.tempdir, "<tmpdir_from_tempfile_module>"
+    yield r"var/folders/.*/pytest-of.*/", "<pytest_tempdir>/"
+
+    # replace hex object ids in output by 0x?????????
+    yield r" 0x[0-9a-fA-F]+", " 0x?????????"
+
+
+def _std_conversion(output, request):
+    fixed = []
+    for line in output.splitlines(keepends=True):
+        for regex, replacement in _std_replacements(request):
+            if IS_WIN:
+                # fix windows backwards slashes in regex
+                regex = regex.replace("\\", "\\\\")
+            line, __ = re.subn(regex, replacement, line)
+        fixed.append(line)
+    return "".join(fixed)
+
+
+class CollectErrorRepr(TerminalRepr):
+    def __init__(self, messages, colors):
+        self.messages = messages
+        self.colors = colors
+
+    def toterminal(self, out):
+        for message, color in zip(self.messages, self.colors):
+            out.line(message, **color)
+
+
+def get_snapshot_handler(obj, handler_options, pytest_config):
+    for check, handler in snapshot_handlers:
+        if check(obj):
+            return handler(handler_options, pytest_config, tw)
+    raise ValueError(f"cannot take snapshot for type {obj.__class__}")

pytest_regtest/register_third_party_handlers.py

@@ -0,0 +1,28 @@
+def register_pandas_handler():
+    def is_dataframe(obj):
+        try:
+            import pandas as pd
+
+            return isinstance(obj, pd.DataFrame)
+        except ImportError:
+            return False
+
+    from .pandas_handler import DataFrameHandler
+    from .snapshot_handler import snapshot_handlers
+
+    snapshot_handlers.append((is_dataframe, DataFrameHandler))
+
+
+def register_numpy_handler():
+    def is_numpy(obj):
+        try:
+            import numpy as np
+
+            return isinstance(obj, np.ndarray)
+        except ImportError:
+            return False
+
+    from .numpy_handler import NumpyHandler
+    from .snapshot_handler import snapshot_handlers
+
+    snapshot_handlers.append((is_numpy, NumpyHandler))

pytest_regtest/snapshot_handler.py

@@ -0,0 +1,76 @@
+import abc
+import difflib
+import os
+import pickle
+from collections.abc import Callable
+from pprint import pformat
+from typing import Any, Type, Union
+
+import pytest
+
+
+class BaseSnapshotHandler(abc.ABC):
+    def __init__(
+        self,
+        handler_options: dict[str, Any],
+        pytest_config: Type[pytest.Config],
+        tw: int,
+    ) -> None: ...
+
+    @abc.abstractmethod
+    def save(self, folder: Union[str, os.PathLike], obj: Any) -> None: ...
+
+    @abc.abstractmethod
+    def load(self, folder: Union[str, os.PathLike]) -> Any: ...
+
+    @abc.abstractmethod
+    def show(self, obj: Any) -> list[str]: ...
+
+    @abc.abstractmethod
+    def compare(self, current_obj: Any, recorded_obj: Any) -> bool: ...
+
+    @abc.abstractmethod
+    def show_differences(
+        self, current_obj: Any, recorded_obj: Any, has_markup: bool
+    ) -> list[str]: ...
+
+
+snapshot_handlers: list[tuple[Callable[[Any], bool]]] = []
+
+
+class PythonObjectHandler(BaseSnapshotHandler):
+    def __init__(self, handler_options, pytest_config, tw):
+        self.compact = handler_options.get("compact", False)
+
+    def save(self, folder, obj):
+        with open(os.path.join(folder, "object.pkl"), "wb") as fh:
+            pickle.dump(obj, fh)
+
+    def load(self, folder):
+        with open(os.path.join(folder, "object.pkl"), "rb") as fh:
+            return pickle.load(fh)
+
+    def show(self, obj):
+        return pformat(obj, compact=self.compact).splitlines()
+
+    def compare(self, current_obj, recorded_obj):
+        return recorded_obj == current_obj
+
+    def show_differences(self, current_obj, recorded_obj, has_markup):
+        return list(
+            difflib.unified_diff(
+                self.show(current_obj),
+                self.show(recorded_obj),
+                "current",
+                "expected",
+                lineterm="",
+            )
+        )
+
+
+snapshot_handlers.append(
+    (
+        lambda obj: isinstance(obj, (int, float, str, list, tuple, dict, set)),
+        PythonObjectHandler,
+    ),
+)

pytest_regtest/utils.py

@@ -0,0 +1,28 @@
+def highlight_mismatches(l1, l2):
+    if not l1 and not l2:
+        return l1, l2
+
+    l1 = l1.ljust(max(len(l1), len(l2)), " ")
+    l2 = l2.ljust(max(len(l1), len(l2)), " ")
+
+    chars_1 = [l1[0]]
+    chars_2 = [l2[0]]
+    UNDERLINE_ON = "\x1b[21m"
+    UNDERLINE_OFF = "\x1b[24m"
+
+    is_invert = False
+
+    for c1, c2 in zip(l1[1:], l2[1:]):
+        if not is_invert and c1 != c2:
+            chars_1.append(UNDERLINE_ON)
+            chars_2.append(UNDERLINE_ON)
+            is_invert = True
+        if is_invert and c1 == c2:
+            chars_1.append(UNDERLINE_OFF)
+            chars_2.append(UNDERLINE_OFF)
+            is_invert = False
+        chars_1.append(c1)
+        chars_2.append(c2)
+    chars_1.append(UNDERLINE_OFF)
+    chars_2.append(UNDERLINE_OFF)
+    return "".join(chars_1), "".join(chars_2)

pytest_regtest-2.2.0.dist-info/METADATA

@@ -0,0 +1,81 @@
+Metadata-Version: 2.3
+Name: pytest-regtest
+Version: 2.2.0
+Summary: pytest plugin for snapshot regression testing
+Project-URL: Source, https://gitlab.com/uweschmitt/pytest-regtest
+Project-URL: Documentation, https://pytest-regtest.readthedocs.org
+Author-email: Uwe Schmitt <uwe.schmitt@id.ethz.ch>
+License: MIT License
+License-File: LICENSE.txt
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Dist: pytest>7.2
+Provides-Extra: dev
+Requires-Dist: black; extra == 'dev'
+Requires-Dist: build; extra == 'dev'
+Requires-Dist: hatchling; extra == 'dev'
+Requires-Dist: mistletoe; extra == 'dev'
+Requires-Dist: mkdocs; extra == 'dev'
+Requires-Dist: mkdocs-awesome-pages-plugin; extra == 'dev'
+Requires-Dist: mkdocs-material; extra == 'dev'
+Requires-Dist: numpy; extra == 'dev'
+Requires-Dist: pandas; extra == 'dev'
+Requires-Dist: pre-commit; extra == 'dev'
+Requires-Dist: pytest-cov; extra == 'dev'
+Requires-Dist: ruff; extra == 'dev'
+Requires-Dist: twine; extra == 'dev'
+Requires-Dist: wheel; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# About
+
+## Introduction
+
+`pytest-regtest` is a plugin for [pytest](https://pytest.org) to implement **regression testing**.
+
+Unlike [functional testing](https://en.wikipedia.org/wiki/Functional_testing), [regression
+testing](https://en.wikipedia.org/wiki/Regression_testing) testing does not test whether the
+software produces the correct results, but whether it behaves as it did before changes were
+introduced.
+
+More specifically, `pytest-regtest` provides **snapshot testing**, which implements regression
+testing by recording data within a test function and comparing this recorded output to a previously
+recorded reference output.
+
+
+## Installation
+
+To install and activate this plugin execute:
+
+    $ pip install pytest-regtest
+
+
+## Use case 1: Changing code with no or little testing setup yet
+If you're working with code that has little or no unit testing, you can use regression testing to
+ensure that your changes don't break or alter previous results.
+
+**Example**: This can be useful when working with scientific data analysis scripts, which often
+start a long script and then are restructured into different functions.
+
+
+## Use case 2:  Testing complex data
+
+If your unit tests contain a lot of `assert` statements to check a complex data structure you can
+use regression tests instead.
+
+**Example**: To test code which ingests data into a database one can use regression tests on textual
+database dumps.
+
+## Use case 3: Testing numpy arrays or pandas data frames
+
+If code produces numerical results, such as `numpy` arrays or pandas` data frames, you can
+use `pytest-regtest` to simply record such results and test later with considering relative and
+absolute tolerances.
+
+**Example**: A function creates a 10 x 10 matrix. Either you have to write 100 assert statements or
+you use summary statistics to test your result. In both cases, you may get little debugging
+information if a test fails.

pytest_regtest-2.2.0.dist-info/RECORD

@@ -0,0 +1,12 @@
+pytest_regtest/__init__.py,sha256=OIo8mNzqvvQ-AiArwcuJvjNvfPOHxwgALfTHqIM1te0,2313
+pytest_regtest/numpy_handler.py,sha256=UZyp9GqO07Kq07qgNSGJq5JXw2vUlgfljGeoxco95Mk,6919
+pytest_regtest/pandas_handler.py,sha256=wlCVYLQU_CJ0x6r9Nok3ToF4CXm8eydBkoil-Nu2rEk,4249
+pytest_regtest/pytest_regtest.py,sha256=b87mqp4ih0puoiKGazHPB79KdTa3rD-D7rRyN0OfDCc,20893
+pytest_regtest/register_third_party_handlers.py,sha256=29bdboPfus_KfnJb0P26-mwF4dEf_Yd4EfdPp75vVpA,725
+pytest_regtest/snapshot_handler.py,sha256=uSngr-Mjnomj6VzDxTjbUnnHsocj00Fiez4EMU0ZMQw,2005
+pytest_regtest/utils.py,sha256=2jYTlV_qL5hH6FCeg7T1HJJvKual-Kux2scJ9_aB1lY,811
+pytest_regtest-2.2.0.dist-info/METADATA,sha256=peSQ9xOiJtyZ0JW9GvyUprtnpWsGVWVHJAsCQLsymRk,3207
+pytest_regtest-2.2.0.dist-info/WHEEL,sha256=fl6v0VwpzfGBVsGtkAkhILUlJxROXbA3HvRL6Fe3140,105
+pytest_regtest-2.2.0.dist-info/entry_points.txt,sha256=4VuIhXeMGhDo0ATbaUfyjND0atofmZjV_P-o6_uEk2s,36
+pytest_regtest-2.2.0.dist-info/licenses/LICENSE.txt,sha256=Tue36uAzpW79-9WAqzkwPhsDDVd1X-VWUmdZ0MfGYvk,1068
+pytest_regtest-2.2.0.dist-info/RECORD,,

pytest_regtest-2.2.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.25.0
+Root-Is-Purelib: true
+Tag: py2-none-any
+Tag: py3-none-any

pytest_regtest-2.2.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[pytest11]
+regtest = pytest_regtest

pytest_regtest-2.2.0.dist-info/licenses/LICENSE.txt

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2016 Uwe Schmitt
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.