pytest-digline 0.2.3__tar.gz → 0.2.4__tar.gz

{pytest_digline-0.2.3 → pytest_digline-0.2.4}/.gitignore

@@ -22,6 +22,9 @@ dist/
 .idea/
 *.iml
 
+# VS Code. Not used today; ignored as prevention.
+.vscode/
+
 # Local Claude Code settings. `.claude/settings.json`, if it appears, is shared
 # and stays versioned. `CLAUDE.local.md` is the personal working agreement —
 # how I want to be worked with — as against `CLAUDE.md`, which is the project.
@@ -38,6 +41,12 @@ private/
 # artifacts are not — and the split is enforced one level down, by the
 # `.gitignore` the store itself writes into `.digline/` (`*/runs/`).
 # Ignoring `.digline/` here would take the baselines out of git with it.
+
+# Output of the release step, like dist/. .github/select_unpublished.py writes
+# into to-publish/ the distributions PyPI and TestPyPI do not already have, and
+# publish.yml uses it as packages-dir. It lands in the working copy of whoever
+# runs that step by hand. The line arrived in b5870a6 without a reason beside
+# it; the reason was read from the code and written here on 2026-10-08.
 to-publish/
 
 # The site checkout that ci.yml's `docs` job makes, and that RELEASING

{pytest_digline-0.2.3 → pytest_digline-0.2.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: pytest-digline
-Version: 0.2.3
+Version: 0.2.4
 Summary: Gate a pytest run on a digline comparison: one row per check, against the baseline committed in your repo.
 Project-URL: Homepage, https://digline.dev/
 Project-URL: Documentation, https://digline.dev/product/pytest/
@@ -17,7 +17,7 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: Software Development :: Testing
 Classifier: Typing :: Typed
 Requires-Python: >=3.12
-Requires-Dist: digline>=0.30.0
+Requires-Dist: digline>=0.31.0
 Requires-Dist: pytest>=8.0
 Description-Content-Type: text/markdown
 
@@ -40,8 +40,7 @@ eval/suite.py::gamma                    s
 
 =================================== FAILURES ===================================
 digline: alpha · llm_rubric
-  dropped from 0.910000 to 0.640000, below its threshold of 0.700000,
-  and beyond the 0.880000–0.950000 this check measured across 5 samples
+  Went from passing to failing (0.910000 → 0.640000).
   reason: signed=True, concise=False
 
 ==================================== digline ===================================

{pytest_digline-0.2.3 → pytest_digline-0.2.4}/README.md

@@ -17,8 +17,7 @@ eval/suite.py::gamma                    s
 
 =================================== FAILURES ===================================
 digline: alpha · llm_rubric
-  dropped from 0.910000 to 0.640000, below its threshold of 0.700000,
-  and beyond the 0.880000–0.950000 this check measured across 5 samples
+  Went from passing to failing (0.910000 → 0.640000).
   reason: signed=True, concise=False
 
 ==================================== digline ===================================

{pytest_digline-0.2.3 → pytest_digline-0.2.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "pytest-digline"
-version = "0.2.3"
+version = "0.2.4"
 description = "Gate a pytest run on a digline comparison: one row per check, against the baseline committed in your repo."
 readme = "README.md"
 requires-python = ">=3.12"
@@ -51,7 +51,14 @@ dependencies = [
     # was owed at the cut and raised in this package's own release, after the
     # core was on PyPI, because a floor may not name a release that does not
     # exist yet.
-    "digline>=0.30.0",
+    #
+    # 0.31.0, a sixth time, by name: `--digline-run` imports `suite_notes`,
+    # which first ships in 0.31.0, so the lines `digline run` prints about the
+    # suite before the first call reach a pytest session too (#396, ADR 0024
+    # §6.4). Against 0.30.0 the import raises `ImportError`. It was owed at the
+    # cut and raised in this package's own release, after the core was on
+    # PyPI, because a floor may not name a release that does not exist yet.
+    "digline>=0.31.0",
     # A floor and no cap, unlike `digline-mcp`'s `mcp<3`. That cap records a
     # rename that actually happened; pytest has not moved the `pytest11` entry
     # point in a decade, and a cap here would fight the one dependency the user

{pytest_digline-0.2.3 → pytest_digline-0.2.4}/src/pytest_digline/plugin.py

@@ -370,7 +370,7 @@ def _measure(
         utc_now_iso,
     )
     from digline.report import visible
-    from digline.run import execute, planned_calls
+    from digline.run import execute, planned_calls, suite_notes
 
     target = load_target(None, loaded, spec)
     plan = planned_calls(suite)
@@ -380,6 +380,12 @@ def _measure(
     # write left in any front end when the rule was widened past `digline.cli`.
     # (from the release delta-pass over 0.15.0)
     print(f"digline: {visible(plan.sentence())}", file=sys.stderr)
+    # Beside the count, as `digline run` says them: whoever writes a suite in
+    # pytest may never run the CLI, and is the one these lines are for. Until
+    # #396 the plugin said neither, under a rule against raising its floor for
+    # a line on stderr; that rule was withdrawn on 2026-10-07. (ADR 0024 §6.4)
+    for note in suite_notes(suite):
+        print(f"digline: {visible(note)}", file=sys.stderr)
     commit = git_commit(root)
     created_at = utc_now_iso()
     artifacts = read_artifacts(suite, target, path.parent, root=root)

pytest_digline-0.2.4/tests/test_digline_run_notes.py

@@ -0,0 +1,103 @@
+"""`--digline-run` says what `digline run` says about the suite. (#396)
+
+Before the first call the CLI names a check whose class declares no `KIND` and
+a tolerance that switches its check off. Whoever writes a suite in pytest may
+never run the CLI, and is the one both lines are for. Until #396 the plugin
+said neither, under a rule against raising its floor for a line on stderr; the
+rule was withdrawn on 2026-10-07. (ADR 0024 §6.4)
+"""
+
+from __future__ import annotations
+
+import pytest
+
+NOTED = """
+from dataclasses import dataclass
+
+from digline.core import (
+    TEXT_ONLY,
+    AssertionBase,
+    Contains,
+    EvaluatorInputs,
+    JudgeReply,
+    LlmRubric,
+    OutputKind,
+    Verdict,
+)
+from digline.run import Case, Response, Suite
+
+
+@dataclass(frozen=True, slots=True)
+class NoKind(AssertionBase):
+    name: str = "no_kind"
+    threshold: float = 0.5
+    tolerance: float = 0.0
+    accepts: frozenset[OutputKind] = TEXT_ONLY
+
+    def __call__(self, inputs: EvaluatorInputs) -> Verdict:
+        return self._binary(True, "fine")
+
+
+def judge(prompt):
+    return JudgeReply(score=1.0, reason="looked")
+
+
+suite = Suite(
+    tenant="acme",
+    environment="staging",
+    name="noted",
+    assertions=[
+        Contains(needle="Acme"),
+        NoKind(),
+        LlmRubric(rubric="ok?", judge=judge, threshold=0.5, tolerance=0.5,
+                  name="loose"),
+    ],
+    cases=[Case(id="alpha")],
+)
+
+
+def target(case):
+    return Response(output="Acme", input="who?")
+"""
+
+
+def test_the_notes_follow_the_count_on_stderr(pytester: pytest.Pytester) -> None:
+    path = pytester.path / "suite.py"
+    path.write_text(NOTED, encoding="utf-8")
+
+    result = pytester.runpytest_subprocess(
+        "--digline-suite", str(path), "--digline-run"
+    )
+
+    # The order `digline run` prints: the count, the KIND note, each blind
+    # tolerance. The session itself ends on the missing baseline, which is not
+    # what is under test here.
+    result.stderr.fnmatch_lines(
+        [
+            "digline: tenant 'acme' * = 1 call to the target",
+            "digline: no_kind declares no KIND, so the shape reading leaves it out*",
+            "digline: tolerance 0.5 on 'loose' (threshold 0.5) covers every movement*",
+        ],
+        consecutive=True,
+    )
+
+
+def test_a_suite_with_nothing_to_say_says_only_the_count(
+    pytester: pytest.Pytester,
+) -> None:
+    """The control: a note on every run would be a line people learn to skip,
+    and would pass the test above."""
+    path = pytester.path / "suite.py"
+    plain = NOTED.replace("        NoKind(),\n", "").replace(
+        "tolerance=0.5", "tolerance=0.05"
+    )
+    path.write_text(plain, encoding="utf-8")
+
+    result = pytester.runpytest_subprocess(
+        "--digline-suite", str(path), "--digline-run"
+    )
+
+    text = result.stderr.str()
+    assert "= 1 call to the target" in text
+    assert "KIND" not in text
+    assert "covers every movement" not in text

pytest_digline-0.2.4/tests/test_plugin_one_declared_path.py

@@ -0,0 +1,93 @@
+"""ADR 0045 through pytest-digline: test plan entry 9, by its flag and its ini.
+
+A real pytest run in a subprocess, started from the working directory under
+test: `pytester` would start it from its own directory, and the directory is
+what decides which prompt a bare relative path reads. The layout is #481's,
+from `tests/_one_path.py`.
+
+The run is not promoted, so a run that gets past the files under test ends on
+"no baseline" (exit 4) after its run file is written. What these tests read is
+that file and what the provider was sent, not the gate's verdict.
+"""
+
+from __future__ import annotations
+
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+import pytest
+from tests._one_path import RELATIVE, layout, recorded, sent, sha
+
+
+def _pytest(tmp_path: Path, cwd: str, how: str) -> subprocess.CompletedProcess[str]:
+    """`--digline-suite eval/suite.py` from `R`, or the absolute suite from
+    elsewhere; or the `digline_suites` ini, which pytest makes absolute."""
+    root = tmp_path / "R"
+    args = [
+        sys.executable,
+        "-m",
+        "pytest",
+        "-p",
+        "no:cacheprovider",
+        "--digline-run",
+        "--digline-root",
+        str(root),
+        "--rootdir",
+        str(root),
+    ]
+    if how == "ini":
+        (root / "pytest.ini").write_text(
+            "[pytest]\ndigline_suites =\n    eval/suite.py\n", "utf-8"
+        )
+        args += ["-c", str(root / "pytest.ini")]
+    else:
+        spec = "eval/suite.py" if cwd == "R" else str(root / "eval" / "suite.py")
+        args += ["-c", os.devnull, "--digline-suite", spec]
+    return subprocess.run(
+        args, cwd=tmp_path / cwd, capture_output=True, text=True, check=False
+    )
+
+
+def _runs(root: Path) -> list[Path]:
+    return list((root / ".digline").rglob("runs/*/*.json"))
+
+
+@pytest.mark.parametrize("how", ["flag", "ini"])
+def test_1_from_the_root_the_run_records_the_file_it_sent(
+    tmp_path: Path, how: str
+) -> None:
+    root = layout(tmp_path)
+    done = _pytest(tmp_path, "R", how)
+    assert "no baseline" in done.stdout + done.stderr, done.stdout + done.stderr
+    assert sent(tmp_path) == ["text of R\n"]
+    (run,) = _runs(root)
+    assert recorded(root, run.stem) == {
+        "prompt.txt": {"sha": sha(root / "prompt.txt"), "text": "text of R\n"}
+    }
+
+
+@pytest.mark.parametrize("how", ["flag", "ini"])
+def test_2_from_outside_the_run_is_refused_and_nothing_is_sent(
+    tmp_path: Path, how: str
+) -> None:
+    root = layout(tmp_path)
+    done = _pytest(tmp_path, "OUT", how)
+    out = done.stdout + done.stderr
+    assert "the target Stub of suite 'qa'" in out, out
+    assert "(ADR 0042 §2)" in out
+    assert sent(tmp_path) == []
+    assert not _runs(root)
+
+
+@pytest.mark.parametrize("how", ["flag", "ini"])
+def test_4_a_relative_answer_is_refused_naming_the_target(
+    tmp_path: Path, how: str
+) -> None:
+    root = layout(tmp_path, target=RELATIVE)
+    done = _pytest(tmp_path, "R", how)
+    out = done.stdout + done.stderr
+    assert "the target Naming of suite 'qa'" in out, out
+    assert "a relative path" in out
+    assert not _runs(root)