pytest-regtest 2.2.0a1__py2.py3-none-any.whl

pytest_regtest/__init__.py

@@ -0,0 +1,67 @@
+from importlib.metadata import version as _version
+
+import pytest
+
+from . import register_third_party_handlers  # noqa: F401
+from .pytest_regtest import PytestRegtestPlugin  # noqa: F401
+from .pytest_regtest import RegtestStream  # noqa: F401
+from .pytest_regtest import clear_converters  # noqa: F401
+from .pytest_regtest import patch_terminal_size  # noqa: F401
+from .pytest_regtest import register_converter_post  # noqa: F401
+from .pytest_regtest import register_converter_pre  # noqa: F401
+
+__version__ = _version(__package__)
+
+
+def pytest_addoption(parser):
+    """Add options to control the timeout plugin"""
+    group = parser.getgroup("regtest", "regression test plugin")
+    group.addoption(
+        "--regtest-reset",
+        action="store_true",
+        help="do not run regtest but record current output",
+    )
+    group.addoption(
+        "--regtest-tee",
+        action="store_true",
+        default=False,
+        help="print recorded results to console too",
+    )
+    group.addoption(
+        "--regtest-consider-line-endings",
+        action="store_true",
+        default=False,
+        help="do not strip whitespaces at end of recorded lines",
+    )
+    group.addoption(
+        "--regtest-nodiff",
+        action="store_true",
+        default=False,
+        help="do not show diff output for failed regresson tests",
+    )
+    group.addoption(
+        "--regtest-disable-stdconv",
+        action="store_true",
+        default=False,
+        help="do not apply standard output converters to clean up indeterministic output",
+    )
+
+
+def pytest_configure(config):
+    config.pluginmanager.register(PytestRegtestPlugin())
+
+
+@pytest.fixture
+def regtest(request):
+    yield RegtestStream(request)
+
+
+snapshot = regtest
+
+
+@pytest.fixture
+def regtest_all(regtest):
+    yield regtest
+
+
+snapshot_all_output = regtest_all

pytest_regtest/numpy_handler.py

@@ -0,0 +1,196 @@
+import difflib
+import io
+import os.path
+import warnings
+
+import numpy as np
+
+from .snapshot_handler import BaseSnapshotHandler
+from .utils import highlight_mismatches
+
+
+class NumpyHandler(BaseSnapshotHandler):
+    def __init__(self, handler_options, pytest_config, tw):
+        self.print_options = np.get_printoptions() | handler_options.get(
+            "print_options", {}
+        )
+        self.atol = handler_options.get("atol", 0.0)
+        self.rtol = handler_options.get("rtol", 0.0)
+        self.equal_nan = handler_options.get("equal_nan", True)
+
+    def _filename(self, folder):
+        return os.path.join(folder, "arrays.npy")
+
+    def save(self, folder, obj):
+        np.save(self._filename(folder), obj)
+
+    def load(self, folder):
+        return np.load(self._filename(folder))
+
+    def show(self, obj):
+        stream = io.StringIO()
+        with np.printoptions(**self.print_options):
+            print(obj, file=stream)
+        return stream.getvalue().splitlines()
+
+    def compare(self, current_obj, recorded_obj):
+        return (
+            isinstance(current_obj, np.ndarray)
+            and current_obj.shape == recorded_obj.shape
+            and current_obj.dtype == recorded_obj.dtype
+            and np.allclose(
+                recorded_obj,
+                current_obj,
+                atol=self.atol,
+                rtol=self.rtol,
+                equal_nan=self.equal_nan,
+            )
+        )
+
+    def show_differences(self, current_obj, recorded_obj, has_markup):
+        lines = []
+
+        if recorded_obj.dtype != current_obj.dtype:
+            lines.extend(
+                [
+                    f"dtype mismatch: current dtype: {current_obj.dtype}",
+                    f"               recorded dtype: {recorded_obj.dtype}",
+                ]
+            )
+
+        recorded_as_text = self.show(recorded_obj)
+        current_as_text = self.show(current_obj)
+
+        if recorded_obj.shape == current_obj.shape:
+            if np.allclose(current_obj, recorded_obj, rtol=self.rtol, atol=self.atol):
+                return lines or None
+
+            lines.extend(self.error_diagnostics(recorded_obj, current_obj))
+
+        else:
+            lines.extend(
+                [
+                    f"shape mismatch: current shape: {current_obj.shape}",
+                    f"               recorded shape: {recorded_obj.shape}",
+                ]
+            )
+
+        if recorded_obj.ndim > 2:
+            return lines
+
+        if recorded_obj.ndim == 1:
+            diff_lines = list(
+                difflib.unified_diff(
+                    current_as_text,
+                    recorded_as_text,
+                    "current",
+                    "expected",
+                    lineterm="",
+                )
+            )
+            lines.append("")
+            lines.extend(diff_lines)
+
+        else:
+            diff_lines = self.error_diagnostics_2d_linewise(
+                current_obj,
+                current_as_text,
+                recorded_obj,
+                recorded_as_text,
+                has_markup,
+            )
+            lines.extend(diff_lines)
+
+        if not diff_lines:
+            lines.append("diff is empty, you may want to change the print options")
+
+        return lines
+
+    def error_diagnostics(self, recorded_obj, current_obj):
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore", RuntimeWarning)
+            rel_err = np.abs(current_obj - recorded_obj) / recorded_obj
+        rel_err[(recorded_obj == 0) * (current_obj == recorded_obj)] = 0.0
+        rel_err_max_1 = np.max(rel_err)
+        rel_err_max_2 = np.max(rel_err[recorded_obj != 0])
+
+        abs_err = np.abs(current_obj - recorded_obj)
+        abs_err_max = np.max(abs_err)
+
+        lines = []
+
+        if rel_err_max_1 == rel_err_max_2:
+            lines.append(f"max relative deviation: {rel_err_max_1:e}")
+        else:
+            lines.append(f"max relative deviation: {rel_err_max_1:e}")
+            lines.append(f"max relative deviation except inf: {rel_err_max_2:e}")
+
+        lines.append(f"max absolute deviation: {abs_err_max:e}")
+
+        n_diff = np.sum(
+            np.logical_not(
+                np.isclose(current_obj, recorded_obj, rtol=self.rtol, atol=self.atol)
+            )
+        )
+
+        lines.append(
+            f"both arrays differ in {n_diff} out of {np.prod(recorded_obj.shape)}"
+            " entries"
+        )
+        lines.append(
+            f"up to given precision settings rtol={self.rtol:e} and"
+            f" atol={self.atol:e}"
+        )
+
+        return lines
+
+    def error_diagnostics_2d_linewise(
+        self, current_obj, current_as_text, recorded_obj, recorded_as_text, has_markup
+    ):
+        sub_diff = []
+        for i, (l1, l2, r1, r2) in enumerate(
+            zip(current_as_text, recorded_as_text, current_obj, recorded_obj)
+        ):
+            if r1.shape == r2.shape and np.allclose(
+                r1, r2, rtol=self.rtol, atol=self.atol
+            ):
+                continue
+
+            if r1.shape == r2.shape:
+                # enforces more uniform formatting of both lines:
+                rows_together = np.vstack((r1, r2))
+                lines_together = self.show(rows_together)
+                line_diff = list(
+                    difflib.unified_diff(
+                        [lines_together[0][1:].strip()],
+                        [lines_together[1][:-1].strip()],
+                        "current",
+                        "expected",
+                        lineterm="",
+                    )
+                )
+            else:
+                row_1 = self.show(r1)
+                row_2 = self.show(r2)
+                line_diff = list(
+                    difflib.unified_diff(
+                        row_1,
+                        row_2,
+                        "current",
+                        "expected",
+                        lineterm="",
+                    )
+                )
+
+            if line_diff:
+                if not sub_diff:
+                    sub_diff = line_diff[:2]
+
+                l1, l2 = line_diff[-2], line_diff[-1]
+                if has_markup:
+                    l1, l2 = highlight_mismatches(l1, l2)
+
+                sub_diff.append(f"row {i:3d}: {l1}")
+                sub_diff.append(f"         {l2}")
+
+        return sub_diff

pytest_regtest/pandas_handler.py

@@ -0,0 +1,135 @@
+import difflib
+import io
+import os.path
+
+import numpy as np
+import pandas as pd
+
+from .snapshot_handler import BaseSnapshotHandler
+
+
+class DataFrameHandler(BaseSnapshotHandler):
+    def __init__(self, handler_options, pytest_config, tw):
+        # default contains a few nested dicts and we flatten those, e.g.
+        # { "html": {"border": 1} } -> { "html.border": 1 }
+        default = list(pd.options.display.d.items())
+        default_flattened = {}
+        for k, v in default:
+            if isinstance(v, dict):
+                for k0, v0 in v.items():
+                    default_flattened[f"{k}.{k0}"] = v0
+            else:
+                default_flattened[k] = v
+
+        # overwrite with user settings:
+        items = (default_flattened | handler_options.get("display_options", {})).items()
+
+        # flatten items as required by pandas.option_context:
+        self.display_options_flat = [
+            entry for item in items for entry in (f"display.{item[0]}", item[1])
+        ]
+        self.atol = handler_options.get("atol", 0.0)
+        self.rtol = handler_options.get("rtol", 0.0)
+
+    def _filename(self, folder):
+        return os.path.join(folder, "dataframe.pkl")
+
+    def save(self, folder, obj):
+        obj.to_pickle(self._filename(folder), compression="gzip")
+
+    def load(self, folder):
+        return pd.read_pickle(self._filename(folder), compression="gzip")
+
+    def show(self, obj):
+        stream = io.StringIO()
+        with pd.option_context(*self.display_options_flat):
+            print(obj, file=stream)
+        return stream.getvalue().splitlines()
+
+    def compare(self, current, recorded):
+        missing = set(
+            n
+            for (n, t) in set(zip(recorded.columns, recorded.dtypes))
+            ^ set(zip(current.columns, current.dtypes))
+        )
+
+        if missing:
+            return False
+
+        common = set(
+            n
+            for (n, t) in set(zip(recorded.columns, recorded.dtypes))
+            & set(zip(current.columns, current.dtypes))
+        )
+        current_reduced = current[[n for n in current.columns if n in common]]
+        recorded_reduced = recorded[[n for n in recorded.columns if n in common]]
+
+        def extract(df, selector):
+            return df[[n for (n, t) in zip(df.columns, df.dtypes) if selector(t)]]
+
+        current_reduced_floats = extract(
+            current_reduced, lambda t: t.type is np.float64
+        ).to_numpy()
+
+        current_reduced_other = extract(
+            current_reduced, lambda t: t.type is not np.float64
+        )
+
+        recorded_reduced_floats = extract(
+            recorded_reduced, lambda t: t.type is np.float64
+        ).to_numpy()
+
+        recorded_reduced_other = extract(
+            recorded_reduced, lambda t: t.type is not np.float64
+        )
+
+        return np.allclose(
+            current_reduced_floats,
+            recorded_reduced_floats,
+            atol=self.atol,
+            rtol=self.rtol,
+            equal_nan=True,
+        ) and (current_reduced_other == recorded_reduced_other).all(axis=None)
+
+    def show_differences(self, current, recorded, has_markup):
+        lines = []
+
+        stream = io.StringIO()
+        current.info(buf=stream, verbose=True, memory_usage=False)
+        current_info = stream.getvalue().splitlines()[2:][:-1]
+
+        stream = io.StringIO()
+        recorded.info(buf=stream, verbose=True, memory_usage=False)
+        recorded_info = stream.getvalue().splitlines()[2:][:-1]
+
+        info_diff = list(
+            difflib.unified_diff(
+                current_info,
+                recorded_info,
+                "current",
+                "expected",
+                lineterm="",
+            )
+        )
+        lines.extend(info_diff)
+
+        recorded_as_text = self.show(recorded)
+        current_as_text = self.show(current)
+
+        diffs = list(
+            difflib.unified_diff(
+                current_as_text,
+                recorded_as_text,
+                "current",
+                "expected",
+                lineterm="",
+            )
+        )
+
+        lines.append("")
+        if diffs:
+            lines.extend(diffs)
+        else:
+            lines.append("diff is empty, you may want to change the print options")
+
+        return lines