pytest-digline 0.2.2__tar.gz → 0.2.4__tar.gz

{pytest_digline-0.2.2 → pytest_digline-0.2.4}/.gitignore

@@ -22,6 +22,9 @@ dist/
 .idea/
 *.iml
 
+# VS Code. Not used today; ignored as prevention.
+.vscode/
+
 # Local Claude Code settings. `.claude/settings.json`, if it appears, is shared
 # and stays versioned. `CLAUDE.local.md` is the personal working agreement —
 # how I want to be worked with — as against `CLAUDE.md`, which is the project.
@@ -38,6 +41,12 @@ private/
 # artifacts are not — and the split is enforced one level down, by the
 # `.gitignore` the store itself writes into `.digline/` (`*/runs/`).
 # Ignoring `.digline/` here would take the baselines out of git with it.
+
+# Output of the release step, like dist/. .github/select_unpublished.py writes
+# into to-publish/ the distributions PyPI and TestPyPI do not already have, and
+# publish.yml uses it as packages-dir. It lands in the working copy of whoever
+# runs that step by hand. The line arrived in b5870a6 without a reason beside
+# it; the reason was read from the code and written here on 2026-10-08.
 to-publish/
 
 # The site checkout that ci.yml's `docs` job makes, and that RELEASING

{pytest_digline-0.2.2 → pytest_digline-0.2.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: pytest-digline
-Version: 0.2.2
+Version: 0.2.4
 Summary: Gate a pytest run on a digline comparison: one row per check, against the baseline committed in your repo.
 Project-URL: Homepage, https://digline.dev/
 Project-URL: Documentation, https://digline.dev/product/pytest/
@@ -17,7 +17,7 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: Software Development :: Testing
 Classifier: Typing :: Typed
 Requires-Python: >=3.12
-Requires-Dist: digline>=0.20.0
+Requires-Dist: digline>=0.31.0
 Requires-Dist: pytest>=8.0
 Description-Content-Type: text/markdown
 
@@ -40,8 +40,7 @@ eval/suite.py::gamma                    s
 
 =================================== FAILURES ===================================
 digline: alpha · llm_rubric
-  dropped from 0.910000 to 0.640000, below its threshold of 0.700000,
-  and beyond the 0.880000–0.950000 this check measured across 5 samples
+  Went from passing to failing (0.910000 → 0.640000).
   reason: signed=True, concise=False
 
 ==================================== digline ===================================
@@ -84,10 +83,14 @@ for either. The distinction survives in the report — `F` and `E` are counted
 separately, `-rE` lists the errored rows — and in `--junit-xml`. A job that
 needs `1` versus `2` runs `digline compare`, which is one command away.
 
-One exit code it does carry: a **refusal**. When digline declines the request
+What it does carry is a **refusal**. When digline declines the request
 — no baseline, a document it refuses, a comparison across tenants or across
 the projection — the session stops as pytest's usage error, exit `4`, and the
-message names the refusal, as `digline compare` names it and exits `64`.
+message names the refusal, as `digline compare` names it and exits `64`. So
+does a file that cannot be read or written, and a `sys.exit()` from a target
+under `--digline-run`. Anything else nobody anticipated prints its traceback
+and exits `3`, where the command line exits `70`, with digline's sentence in
+place of pytest's `INTERNALERROR`.
 
 ## Naming a suite
 

{pytest_digline-0.2.2 → pytest_digline-0.2.4}/README.md

@@ -17,8 +17,7 @@ eval/suite.py::gamma                    s
 
 =================================== FAILURES ===================================
 digline: alpha · llm_rubric
-  dropped from 0.910000 to 0.640000, below its threshold of 0.700000,
-  and beyond the 0.880000–0.950000 this check measured across 5 samples
+  Went from passing to failing (0.910000 → 0.640000).
   reason: signed=True, concise=False
 
 ==================================== digline ===================================
@@ -61,10 +60,14 @@ for either. The distinction survives in the report — `F` and `E` are counted
 separately, `-rE` lists the errored rows — and in `--junit-xml`. A job that
 needs `1` versus `2` runs `digline compare`, which is one command away.
 
-One exit code it does carry: a **refusal**. When digline declines the request
+What it does carry is a **refusal**. When digline declines the request
 — no baseline, a document it refuses, a comparison across tenants or across
 the projection — the session stops as pytest's usage error, exit `4`, and the
-message names the refusal, as `digline compare` names it and exits `64`.
+message names the refusal, as `digline compare` names it and exits `64`. So
+does a file that cannot be read or written, and a `sys.exit()` from a target
+under `--digline-run`. Anything else nobody anticipated prints its traceback
+and exits `3`, where the command line exits `70`, with digline's sentence in
+place of pytest's `INTERNALERROR`.
 
 ## Naming a suite
 

{pytest_digline-0.2.2 → pytest_digline-0.2.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "pytest-digline"
-version = "0.2.2"
+version = "0.2.4"
 description = "Gate a pytest run on a digline comparison: one row per check, against the baseline committed in your repo."
 readme = "README.md"
 requires-python = ">=3.12"
@@ -42,7 +42,23 @@ dependencies = [
     # 0.20.0, a fourth time, for a name the gate sees: the plugin catches
     # `digline.host.REFUSALS`, which arrived there, and turns every refusal into
     # pytest's usage error rather than an INTERNALERROR.
-    "digline>=0.20.0",
+    #
+    # 0.30.0, a fifth time, by signature: the plugin catches what the CLI's
+    # `main()` catches (#475), and refuses a `SystemExit` from code digline
+    # runs through `refused_exit`, which the gate dates to 0.28.0, passing
+    # `front_end=`, which it cannot see and which first ships in 0.30.0 (#452,
+    # ADR 0043 §2, amended). Against 0.29.0 that call raises `TypeError`. It
+    # was owed at the cut and raised in this package's own release, after the
+    # core was on PyPI, because a floor may not name a release that does not
+    # exist yet.
+    #
+    # 0.31.0, a sixth time, by name: `--digline-run` imports `suite_notes`,
+    # which first ships in 0.31.0, so the lines `digline run` prints about the
+    # suite before the first call reach a pytest session too (#396, ADR 0024
+    # §6.4). Against 0.30.0 the import raises `ImportError`. It was owed at the
+    # cut and raised in this package's own release, after the core was on
+    # PyPI, because a floor may not name a release that does not exist yet.
+    "digline>=0.31.0",
     # A floor and no cap, unlike `digline-mcp`'s `mcp<3`. That cap records a
     # rename that actually happened; pytest has not moved the `pytest11` entry
     # point in a decade, and a cap here would fight the one dependency the user

{pytest_digline-0.2.2 → pytest_digline-0.2.4}/src/pytest_digline/plugin.py

@@ -29,6 +29,7 @@ these sources to keep it that way.
 from __future__ import annotations
 
 import sys
+import traceback
 from collections.abc import Generator, Iterator, Sequence
 from dataclasses import dataclass
 from pathlib import Path
@@ -269,16 +270,47 @@ def _open(spec: str, config: pytest.Config) -> _Opened:
     one reached pytest as `INTERNALERROR` and exit 3, which reads as a crash of
     pytest or of this plugin. The class is named in the message, as the CLI
     names it. (Delta-pass over 0.25.1)
+
+    **The four branches are `digline.cli.main()`'s**, mapped onto pytest's
+    codes: what the CLI refuses with `64` is pytest's `4` here, and what it
+    reports as not anticipated with `70` is pytest's `3` with digline's own
+    sentence instead of `INTERNALERROR`, which reads as a crash of pytest or of
+    this plugin. An `OSError` is the environment, as on the CLI. A `SystemExit`
+    from code digline runs is refused with its location. Each sentence is the
+    whole one, because the reader ran the command (ADR 0043 §3), and each passes
+    through `visible()`, because a refusal can quote a document or what the
+    suite's code wrote. Before #475 only the first two branches were here.
     """
-    from digline.host import REFUSALS, UsageError
+    from digline.host import REFUSALS, UsageError, refused_exit
+    from digline.report import visible
 
     root = Path(config.getoption("digline_root") or config.rootpath)
     try:
         return _opened(spec, root, run_first=bool(config.getoption("digline_run")))
     except UsageError as exc:
-        raise pytest.UsageError(f"digline: {exc}") from exc
-    except REFUSALS as exc:
-        raise pytest.UsageError(f"digline: {type(exc).__name__}: {exc}") from exc
+        raise pytest.UsageError(visible(f"digline: {exc}")) from exc
+    except (*REFUSALS, OSError) as exc:
+        raise pytest.UsageError(
+            visible(f"digline: {type(exc).__name__}: {exc}")
+        ) from exc
+    except SystemExit as exc:
+        # This plugin's directory is handed over so that its own frames are not
+        # given as where the code was reached from. (ADR 0043 §2, amended)
+        refused = refused_exit(exc, front_end=Path(__file__).parent)
+        if refused is None:
+            raise
+        raise pytest.UsageError(visible(f"digline: {refused}")) from exc
+    except Exception:
+        # The traceback is the report, as on the CLI, a line at a time through
+        # `visible()`. `pytest.exit` rather than a raise: a raise is pytest's
+        # INTERNALERROR, the crash this replaces. (ADR 0041 §2, §5)
+        for line in traceback.format_exc().splitlines():
+            print(visible(line), file=sys.stderr)
+        pytest.exit(
+            "digline: the failure above was not anticipated. It is not a verdict "
+            "on the suite, and digline's command line exits 70 for it.",
+            returncode=pytest.ExitCode.INTERNAL_ERROR,
+        )
 
 
 def _opened(spec: str, root: Path, *, run_first: bool) -> _Opened:
@@ -338,7 +370,7 @@ def _measure(
         utc_now_iso,
     )
     from digline.report import visible
-    from digline.run import execute, planned_calls
+    from digline.run import execute, planned_calls, suite_notes
 
     target = load_target(None, loaded, spec)
     plan = planned_calls(suite)
@@ -348,6 +380,12 @@ def _measure(
     # write left in any front end when the rule was widened past `digline.cli`.
     # (from the release delta-pass over 0.15.0)
     print(f"digline: {visible(plan.sentence())}", file=sys.stderr)
+    # Beside the count, as `digline run` says them: whoever writes a suite in
+    # pytest may never run the CLI, and is the one these lines are for. Until
+    # #396 the plugin said neither, under a rule against raising its floor for
+    # a line on stderr; that rule was withdrawn on 2026-10-07. (ADR 0024 §6.4)
+    for note in suite_notes(suite):
+        print(f"digline: {visible(note)}", file=sys.stderr)
     commit = git_commit(root)
     created_at = utc_now_iso()
     artifacts = read_artifacts(suite, target, path.parent, root=root)

pytest_digline-0.2.4/tests/test_digline_run_notes.py

@@ -0,0 +1,103 @@
+"""`--digline-run` says what `digline run` says about the suite. (#396)
+
+Before the first call the CLI names a check whose class declares no `KIND` and
+a tolerance that switches its check off. Whoever writes a suite in pytest may
+never run the CLI, and is the one both lines are for. Until #396 the plugin
+said neither, under a rule against raising its floor for a line on stderr; the
+rule was withdrawn on 2026-10-07. (ADR 0024 §6.4)
+"""
+
+from __future__ import annotations
+
+import pytest
+
+NOTED = """
+from dataclasses import dataclass
+
+from digline.core import (
+    TEXT_ONLY,
+    AssertionBase,
+    Contains,
+    EvaluatorInputs,
+    JudgeReply,
+    LlmRubric,
+    OutputKind,
+    Verdict,
+)
+from digline.run import Case, Response, Suite
+
+
+@dataclass(frozen=True, slots=True)
+class NoKind(AssertionBase):
+    name: str = "no_kind"
+    threshold: float = 0.5
+    tolerance: float = 0.0
+    accepts: frozenset[OutputKind] = TEXT_ONLY
+
+    def __call__(self, inputs: EvaluatorInputs) -> Verdict:
+        return self._binary(True, "fine")
+
+
+def judge(prompt):
+    return JudgeReply(score=1.0, reason="looked")
+
+
+suite = Suite(
+    tenant="acme",
+    environment="staging",
+    name="noted",
+    assertions=[
+        Contains(needle="Acme"),
+        NoKind(),
+        LlmRubric(rubric="ok?", judge=judge, threshold=0.5, tolerance=0.5,
+                  name="loose"),
+    ],
+    cases=[Case(id="alpha")],
+)
+
+
+def target(case):
+    return Response(output="Acme", input="who?")
+"""
+
+
+def test_the_notes_follow_the_count_on_stderr(pytester: pytest.Pytester) -> None:
+    path = pytester.path / "suite.py"
+    path.write_text(NOTED, encoding="utf-8")
+
+    result = pytester.runpytest_subprocess(
+        "--digline-suite", str(path), "--digline-run"
+    )
+
+    # The order `digline run` prints: the count, the KIND note, each blind
+    # tolerance. The session itself ends on the missing baseline, which is not
+    # what is under test here.
+    result.stderr.fnmatch_lines(
+        [
+            "digline: tenant 'acme' * = 1 call to the target",
+            "digline: no_kind declares no KIND, so the shape reading leaves it out*",
+            "digline: tolerance 0.5 on 'loose' (threshold 0.5) covers every movement*",
+        ],
+        consecutive=True,
+    )
+
+
+def test_a_suite_with_nothing_to_say_says_only_the_count(
+    pytester: pytest.Pytester,
+) -> None:
+    """The control: a note on every run would be a line people learn to skip,
+    and would pass the test above."""
+    path = pytester.path / "suite.py"
+    plain = NOTED.replace("        NoKind(),\n", "").replace(
+        "tolerance=0.5", "tolerance=0.05"
+    )
+    path.write_text(plain, encoding="utf-8")
+
+    result = pytester.runpytest_subprocess(
+        "--digline-suite", str(path), "--digline-run"
+    )
+
+    text = result.stderr.str()
+    assert "= 1 call to the target" in text
+    assert "KIND" not in text
+    assert "covers every movement" not in text

pytest_digline-0.2.4/tests/test_exits.py

@@ -0,0 +1,177 @@
+"""What the CLI's `main()` catches, caught here too. (#475)
+
+`digline.cli.main()` has four branches: a usage error, a refusal or an
+`OSError`, a `SystemExit` from code digline ran, and anything else. Before #475
+this plugin had the first two without `OSError`, so after a suite had loaded the
+other three reached pytest as `INTERNALERROR` and exit 3, which reads as a crash
+of pytest or of this plugin. Each test names the row of the measurement on the
+issue it holds.
+
+Every sentence is the whole one, because the reader ran the command (ADR 0043
+§3), and passes through `visible()` (ADR 0041 §4.1, rule 3).
+"""
+
+from __future__ import annotations
+
+import os
+import sys
+from collections.abc import Callable
+from pathlib import Path
+
+import pytest
+
+Baseline = Callable[[dict[str, str], dict[str, str]], Path]
+
+FINE = {"alpha": "Acme, SATISFIED", "beta": "Acme, SATISFIED"}
+
+MARK = "ROWMARK-IT60X0542811101"
+
+without_mode_000 = pytest.mark.skipif(
+    sys.platform == "win32" or os.geteuid() == 0,
+    reason="mode 000 denies nothing to root, and Windows has no such mode",
+)
+
+
+def _append(path: Path, code: str) -> None:
+    path.write_text(path.read_text(encoding="utf-8") + "\n" + code, encoding="utf-8")
+
+
+def _preflight(path: Path, body: str) -> None:
+    """Replace the suite's target with one whose `preflight` runs `body`."""
+    _append(
+        path,
+        "class _Target:\n"
+        "    def __call__(self, case):\n"
+        "        return Response(output='Acme, SATISFIED', cost_usd=0.001, "
+        "latency_ms=1.0)\n"
+        "    def preflight(self, cases):\n"
+        f"        {body}\n"
+        "target = _Target()\n",
+    )
+
+
+def _no_crash(result: pytest.RunResult) -> str:
+    out = result.stdout.str() + result.stderr.str()
+    assert "INTERNALERROR" not in out
+    return out
+
+
+@without_mode_000
+def test_o1_a_baseline_digline_cannot_read_is_a_usage_error(
+    pytester: pytest.Pytester, baseline: Baseline
+) -> None:
+    """O1: an `OSError` digline asked for, as the CLI refuses it with 64."""
+    path = baseline(FINE, FINE)
+    (stored,) = pytester.path.glob(".digline/*/baselines/*.json")
+    stored.chmod(0)
+    try:
+        result = pytester.runpytest_subprocess("--digline-suite", str(path))
+    finally:
+        stored.chmod(0o644)
+
+    assert result.ret == pytest.ExitCode.USAGE_ERROR
+    _no_crash(result)
+    result.stderr.fnmatch_lines(["*digline: PermissionError: *Errno 13*"])
+
+
+@without_mode_000
+def test_o2_a_run_digline_cannot_write_is_a_usage_error(
+    pytester: pytest.Pytester, baseline: Baseline
+) -> None:
+    """O2: the same, from `write_run` under `--digline-run`."""
+    path = baseline(FINE, FINE)
+    (runs,) = pytester.path.glob(".digline/*/runs/*")
+    runs.chmod(0o555)
+    try:
+        result = pytester.runpytest_subprocess(
+            "--digline-suite", str(path), "--digline-run"
+        )
+    finally:
+        runs.chmod(0o755)
+
+    assert result.ret == pytest.ExitCode.USAGE_ERROR
+    _no_crash(result)
+    result.stderr.fnmatch_lines(["*digline: PermissionError: *Errno 13*"])
+
+
+def test_o4_an_oserror_the_suites_code_raises_is_a_usage_error_with_its_message(
+    pytester: pytest.Pytester, baseline: Baseline
+) -> None:
+    """O4: a `preflight`'s `OSError`. The CLI refuses it by its type, and the
+    whole sentence reaches the person who ran the command, message included."""
+    path = baseline(FINE, FINE)
+    _preflight(path, f"open('/nonexistent/{MARK}')")
+
+    result = pytester.runpytest_subprocess(
+        "--digline-suite", str(path), "--digline-run"
+    )
+
+    assert result.ret == pytest.ExitCode.USAGE_ERROR
+    _no_crash(result)
+    result.stderr.fnmatch_lines([f"*digline: FileNotFoundError: *{MARK}*"])
+
+
+def test_s2_a_target_that_calls_sys_exit_is_refused_with_its_location(
+    pytester: pytest.Pytester, baseline: Baseline
+) -> None:
+    """S2: ADR 0041 §4.3. The location is the suite's line, and none of this
+    plugin's frames is given as where the code was reached from."""
+    path = baseline(FINE, FINE)
+    _append(path, f"def target(case):\n    import sys\n    sys.exit({MARK!r})\n")
+
+    result = pytester.runpytest_subprocess(
+        "--digline-suite", str(path), "--digline-run"
+    )
+
+    assert result.ret == pytest.ExitCode.USAGE_ERROR
+    out = _no_crash(result)
+    result.stderr.fnmatch_lines(
+        [f"*digline: code digline ran raised SystemExit('{MARK}') at *suite.py:*"]
+    )
+    assert "pytest_digline" not in out
+
+
+def test_x2_anything_else_is_not_anticipated_and_not_a_crash(
+    pytester: pytest.Pytester, baseline: Baseline
+) -> None:
+    """X2: the CLI exits 70 with the traceback. Here it is pytest's 3, with the
+    traceback and digline's sentence in place of `INTERNALERROR`."""
+    path = baseline(FINE, FINE)
+    _preflight(path, f"raise ValueError({MARK!r})")
+
+    result = pytester.runpytest_subprocess(
+        "--digline-suite", str(path), "--digline-run"
+    )
+
+    assert result.ret == pytest.ExitCode.INTERNAL_ERROR
+    out = _no_crash(result)
+    result.stderr.fnmatch_lines(
+        ["Traceback (most recent call last):", f"*ValueError: {MARK}"]
+    )
+    assert "the failure above was not anticipated" in out
+
+
+@pytest.mark.parametrize(
+    "refusal",
+    [
+        pytest.param("digline.core import RefusedError as R", id="refusal"),
+        pytest.param("digline.host import UsageError as R", id="usage-error"),
+    ],
+)
+def test_esc_a_refusals_sentence_reaches_the_terminal_through_visible(
+    pytester: pytest.Pytester, baseline: Baseline, refusal: str
+) -> None:
+    """A refusal can quote what the suite's code wrote. An escape in it is
+    shown, not obeyed, as `digline run` shows it. Both branches: `UsageError`
+    has one of its own."""
+    path = baseline(FINE, FINE)
+    _preflight(path, f'from {refusal}; raise R("A\\x1b[2KFORGED")')
+
+    result = pytester.runpytest_subprocess(
+        "--digline-suite", str(path), "--digline-run"
+    )
+
+    assert result.ret == pytest.ExitCode.USAGE_ERROR
+    err = result.stderr.str()
+    assert "\x1b" not in err
+    assert "A\\x1b[2KFORGED" in err

pytest_digline-0.2.4/tests/test_plugin_one_declared_path.py

@@ -0,0 +1,93 @@
+"""ADR 0045 through pytest-digline: test plan entry 9, by its flag and its ini.
+
+A real pytest run in a subprocess, started from the working directory under
+test: `pytester` would start it from its own directory, and the directory is
+what decides which prompt a bare relative path reads. The layout is #481's,
+from `tests/_one_path.py`.
+
+The run is not promoted, so a run that gets past the files under test ends on
+"no baseline" (exit 4) after its run file is written. What these tests read is
+that file and what the provider was sent, not the gate's verdict.
+"""
+
+from __future__ import annotations
+
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+import pytest
+from tests._one_path import RELATIVE, layout, recorded, sent, sha
+
+
+def _pytest(tmp_path: Path, cwd: str, how: str) -> subprocess.CompletedProcess[str]:
+    """`--digline-suite eval/suite.py` from `R`, or the absolute suite from
+    elsewhere; or the `digline_suites` ini, which pytest makes absolute."""
+    root = tmp_path / "R"
+    args = [
+        sys.executable,
+        "-m",
+        "pytest",
+        "-p",
+        "no:cacheprovider",
+        "--digline-run",
+        "--digline-root",
+        str(root),
+        "--rootdir",
+        str(root),
+    ]
+    if how == "ini":
+        (root / "pytest.ini").write_text(
+            "[pytest]\ndigline_suites =\n    eval/suite.py\n", "utf-8"
+        )
+        args += ["-c", str(root / "pytest.ini")]
+    else:
+        spec = "eval/suite.py" if cwd == "R" else str(root / "eval" / "suite.py")
+        args += ["-c", os.devnull, "--digline-suite", spec]
+    return subprocess.run(
+        args, cwd=tmp_path / cwd, capture_output=True, text=True, check=False
+    )
+
+
+def _runs(root: Path) -> list[Path]:
+    return list((root / ".digline").rglob("runs/*/*.json"))
+
+
+@pytest.mark.parametrize("how", ["flag", "ini"])
+def test_1_from_the_root_the_run_records_the_file_it_sent(
+    tmp_path: Path, how: str
+) -> None:
+    root = layout(tmp_path)
+    done = _pytest(tmp_path, "R", how)
+    assert "no baseline" in done.stdout + done.stderr, done.stdout + done.stderr
+    assert sent(tmp_path) == ["text of R\n"]
+    (run,) = _runs(root)
+    assert recorded(root, run.stem) == {
+        "prompt.txt": {"sha": sha(root / "prompt.txt"), "text": "text of R\n"}
+    }
+
+
+@pytest.mark.parametrize("how", ["flag", "ini"])
+def test_2_from_outside_the_run_is_refused_and_nothing_is_sent(
+    tmp_path: Path, how: str
+) -> None:
+    root = layout(tmp_path)
+    done = _pytest(tmp_path, "OUT", how)
+    out = done.stdout + done.stderr
+    assert "the target Stub of suite 'qa'" in out, out
+    assert "(ADR 0042 §2)" in out
+    assert sent(tmp_path) == []
+    assert not _runs(root)
+
+
+@pytest.mark.parametrize("how", ["flag", "ini"])
+def test_4_a_relative_answer_is_refused_naming_the_target(
+    tmp_path: Path, how: str
+) -> None:
+    root = layout(tmp_path, target=RELATIVE)
+    done = _pytest(tmp_path, "R", how)
+    out = done.stdout + done.stderr
+    assert "the target Naming of suite 'qa'" in out, out
+    assert "a relative path" in out
+    assert not _runs(root)