agentprdiff 0.2.2__tar.gz → 0.2.3__tar.gz

{agentprdiff-0.2.2 → agentprdiff-0.2.3}/CHANGELOG.md

@@ -8,6 +8,43 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.2.3] — 2026-04-28
+
+### Added
+
+- **Semantic-judge banner in `check` and `review` output.** `TerminalReporter`
+  and `ReviewReporter` now print one line — `semantic judge: <description>`
+  — directly under the header whenever the suite contains at least one
+  `semantic(...)` grader. The description names the active backend
+  (`fake_judge`, `openai/<model>`, or `anthropic/<model>`) and the env-var
+  signal that selected it, with explicit `silent fallback` wording when
+  no judge is configured. Closes the most common adoption trap: shipping
+  suites whose semantic coverage is decorative because no key was set
+  and the runner stayed quiet about it. Suites without `semantic(...)`
+  are unaffected — no banner is printed. New helpers
+  `agentprdiff.graders.semantic.describe_default_judge()` and
+  `case_uses_semantic()` power the rendering and are reusable by
+  third-party tooling.
+- **Scaffolded workflow YAML now flags judge-SDK installs explicitly.**
+  `_TPL_WORKFLOW` ships commented `pip install anthropic` /
+  `pip install openai` lines tied to the chosen `AGENTGUARD_JUDGE` mode,
+  with guidance that a missing SDK raises `ImportError` rather than
+  falling back silently. Pairs with the in-band judge banner for
+  end-to-end coverage of the silent-fake_judge trap.
+
+### Fixed
+
+- Adoption checklist in `AGENTS.md` now requires the adopter to verify
+  the installed CLI supports the documented commands (`agentprdiff
+  check --help`) before writing run commands into the case dossier —
+  prevents the "source docs reference `--case` but the pinned wheel
+  predates it" confusion reported during 0.2.x adoption.
+- New `Step 5b — decide and document the semantic-judge mode` mandates
+  a `## Semantic Judge Keys` section in `suites/README.md` and an
+  explicit `AGENTGUARD_JUDGE=<mode>` line in the workflow YAML, replacing
+  the implicit "first available key wins" precedence with a deliberate
+  declaration.
+
 ## [0.2.2] — 2026-04-28
 
 ### Added

{agentprdiff-0.2.2 → agentprdiff-0.2.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentprdiff
-Version: 0.2.2
+Version: 0.2.3
 Summary: Guard your LLM agents in CI. Snapshot tests that catch behavioral regressions when models, prompts, or vendors change.
 Project-URL: Homepage, https://github.com/vnageshwaran-de/agentprdiff
 Project-URL: Documentation, https://github.com/vnageshwaran-de/agentprdiff#readme

{agentprdiff-0.2.2 → agentprdiff-0.2.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "agentprdiff"
-version = "0.2.2"
+version = "0.2.3"
 description = "Guard your LLM agents in CI. Snapshot tests that catch behavioral regressions when models, prompts, or vendors change."
 readme = "README.md"
 requires-python = ">=3.10"

{agentprdiff-0.2.2 → agentprdiff-0.2.3}/src/agentprdiff/graders/semantic.py

@@ -178,3 +178,56 @@ def _default_judge() -> Judge:
     if choice == "anthropic" or (not choice and os.environ.get("ANTHROPIC_API_KEY")):
         return anthropic_judge()
     return fake_judge
+
+
+# Default model names mirrored from `openai_judge` and `anthropic_judge`. Kept
+# in module scope so `describe_default_judge` reports the same string the
+# default judge would actually use without instantiating the judge (which
+# would import the SDK lazily).
+_DEFAULT_OPENAI_MODEL = "gpt-4o-mini"
+_DEFAULT_ANTHROPIC_MODEL = "claude-haiku-4-5-20251001"
+
+
+def describe_default_judge() -> str:
+    """Return a one-line description of the currently-selected default judge.
+
+    Mirrors the env-var precedence in :func:`_default_judge` so the string is
+    a faithful preview of what `semantic(...)` graders without an explicit
+    `judge=` argument will use at runtime. Reporters print this once per run
+    so the silent fake_judge fallback (no key set, no AGENTGUARD_JUDGE) is
+    visible at the moment a suite executes — not buried in trace JSON.
+
+    Examples:
+        - ``"fake_judge (AGENTGUARD_JUDGE=fake)"``
+        - ``"openai/gpt-4o-mini (OPENAI_API_KEY set)"``
+        - ``"anthropic/claude-haiku-4-5-20251001 (AGENTGUARD_JUDGE=anthropic)"``
+        - ``"fake_judge (no AGENTGUARD_JUDGE, no OPENAI_API_KEY/ANTHROPIC_API_KEY — silent fallback)"``
+    """
+    choice = (os.environ.get("AGENTGUARD_JUDGE") or "").lower()
+    if choice == "fake":
+        return "fake_judge (AGENTGUARD_JUDGE=fake)"
+    if choice == "openai":
+        return f"openai/{_DEFAULT_OPENAI_MODEL} (AGENTGUARD_JUDGE=openai)"
+    if choice == "anthropic":
+        return (
+            f"anthropic/{_DEFAULT_ANTHROPIC_MODEL} (AGENTGUARD_JUDGE=anthropic)"
+        )
+    if not choice and os.environ.get("OPENAI_API_KEY"):
+        return f"openai/{_DEFAULT_OPENAI_MODEL} (OPENAI_API_KEY set)"
+    if not choice and os.environ.get("ANTHROPIC_API_KEY"):
+        return f"anthropic/{_DEFAULT_ANTHROPIC_MODEL} (ANTHROPIC_API_KEY set)"
+    return (
+        "fake_judge (no AGENTGUARD_JUDGE, no OPENAI_API_KEY/ANTHROPIC_API_KEY"
+        " — silent fallback)"
+    )
+
+
+def case_uses_semantic(grader_results: list[GradeResult]) -> bool:
+    """Return True if any grader in `grader_results` was a `semantic()` grader.
+
+    Detection is based on the grader name format produced by :func:`semantic`
+    (``semantic(<rubric>)``). This is stable because the public grader name is
+    part of the user-visible grading contract; reporters and serializers
+    already depend on it.
+    """
+    return any(r.grader_name.startswith("semantic(") for r in grader_results)

{agentprdiff-0.2.2 → agentprdiff-0.2.3}/src/agentprdiff/reporters.py

@@ -17,6 +17,7 @@ from rich.table import Table
 from rich.text import Text
 
 from .differ import AssertionChange, TraceDelta
+from .graders.semantic import case_uses_semantic, describe_default_judge
 from .runner import CaseReport, RunReport
 
 
@@ -35,6 +36,7 @@ class TerminalReporter:
             style="dim",
         )
         self.console.print(header)
+        _maybe_print_judge_banner(self.console, report)
 
         table = Table(show_header=True, header_style="bold", show_lines=False, expand=True)
         table.add_column("Case", style="bold")
@@ -130,6 +132,26 @@ def _format_delta(value: float, fmt: str) -> str:
     return f"[{color}]{text}[/{color}]"
 
 
+def _maybe_print_judge_banner(console: Console, report: RunReport) -> None:
+    """Print "semantic judge: <mode>" once per run if any case used semantic().
+
+    Skipped silently when no case has a semantic grader — most suites don't
+    use them and the banner would be noise. When a suite *does* have semantic
+    coverage, the banner makes the silent fake_judge fallback (no key, no
+    AGENTGUARD_JUDGE) loud at the moment of execution rather than buried in
+    trace JSON. Coloured yellow when fake_judge would run so the warning is
+    visually distinct from real-judge runs.
+    """
+    if not any(case_uses_semantic(cr.grader_results) for cr in report.case_reports):
+        return
+    description = describe_default_judge()
+    style = "yellow" if description.startswith("fake_judge") else "dim"
+    line = Text()
+    line.append("semantic judge: ", style="dim")
+    line.append(description, style=style)
+    console.print(line)
+
+
 # ---------------------------------------------------------------------------
 # ReviewReporter — verbose per-case view for `agentprdiff review`.
 # ---------------------------------------------------------------------------
@@ -175,6 +197,7 @@ class ReviewReporter:
             style="dim",
         )
         self.console.print(header)
+        _maybe_print_judge_banner(self.console, report)
         self.console.print()
 
     def _render_footer(self, report: RunReport) -> None:

{agentprdiff-0.2.2 → agentprdiff-0.2.3}/src/agentprdiff/scaffold.py

@@ -378,6 +378,36 @@ locally (rather than the keyword-matching fake fallback), also set
 [AGENTS.md → API keys](https://github.com/vnageshwaran-de/agentprdiff/blob/main/AGENTS.md#api-keys--what-to-set-where-and-how-to-ask-the-user-about-them)
 for the full picture, including CI secret setup.
 
+## Semantic Judge Keys
+
+`semantic(...)` graders need an LLM judge to render verdicts. Without
+one, agentprdiff silently falls back to `fake_judge` (keyword matching) —
+the suite reports PASS even when no LLM ever ran. Make the judge mode
+explicit so the absence-or-presence of LLM scoring is never ambiguous in
+CI logs.
+
+**This suite's mode:** TODO — fill in one of:
+
+- `fake_judge` — free, keyword matching only. Acceptable when the rubric
+  reduces cleanly to keywords; brittle otherwise. Set
+  `AGENTGUARD_JUDGE=fake`.
+- Real Anthropic judge — recommended for cost.
+  `AGENTGUARD_JUDGE=anthropic` plus `ANTHROPIC_API_KEY=...`.
+- Real OpenAI judge — `AGENTGUARD_JUDGE=openai` plus `OPENAI_API_KEY=...`.
+- `Not applicable` — this suite has no `semantic(...)` graders. Verify with
+  `grep -n "semantic(" suites/{name}.py`. State this explicitly anyway so
+  the next reviewer knows the absence of judge config is deliberate.
+
+**Reproduce CI's mode locally:** export the same env vars CI sets and run:
+
+```bash
+agentprdiff check suites/{name}.py --case <a_case_with_semantic>
+```
+
+Confirm the trace shows non-zero `cost_usd` on the judge call (real
+judges) or a flat zero (fake_judge). A real-judge config that traces zero
+cost means the silent fallback bit you — re-check the env vars.
+
 ## Run locally
 
 ```bash
@@ -409,23 +439,110 @@ or meaningfully change a case.
 If a case fails in CI and you don't recognize the name, this file is the
 fastest path back to *what the case is pinning and why it matters*.
 
-## How to run
+## Running the suite
+
+Run every case from the repository root:
 
 ```bash
-agentprdiff check suites/{name}.py                      # full suite
-agentprdiff check suites/{name}.py --case happy_path    # one case (substring or glob)
-agentprdiff check suites/{name}.py --list               # discover case names
-agentprdiff check suites/{name}.py --skip slow          # everything except a pattern
+agentprdiff check suites/{name}.py
 ```
 
-After an *intentional* behavior change, re-record the baseline and explain
-the change in the PR description so reviewers see the before/after diff:
+If the installed wheel in your virtualenv predates the `--case`, `--list`,
+and `review` commands, use a local source checkout of agentprdiff next to
+this repository. Substitute `.venv-agentprdiff/` with whatever virtualenv
+your project actually uses:
 
 ```bash
-agentprdiff record suites/{name}.py
+.venv-agentprdiff/bin/agentprdiff check suites/{name}.py
+```
+
+### List the available cases
+
+```bash
+agentprdiff check suites/{name}.py --list
+# or, with a local source checkout:
+PYTHONPATH=../agentprdiff/src .venv-agentprdiff/bin/python -m agentprdiff.cli \\
+    check suites/{name}.py --list
+```
+
+### Run one case
+
+Filter by substring or glob (case-insensitive). A filter that matches
+nothing exits 2 — no silent zero-runs:
+
+```bash
+agentprdiff check suites/{name}.py --case happy_path
+agentprdiff check suites/{name}.py --case "*refund*"
+agentprdiff check suites/{name}.py --skip slow         # everything except a pattern
+```
+
+### Use `review` for verbose, exit-0 iteration
+
+`review` runs the same comparison `check` does but renders one verbose
+panel per case (input echo, every assertion's `was → now` verdict, cost /
+latency / token deltas, tool-sequence diff, unified output diff) and
+exits 0 even on regression — safe to wire into a watcher / `entr` / `fzf`
+loop without your shell going red on every iteration:
+
+```bash
+agentprdiff review suites/{name}.py --case happy_path
+```
+
+Reach for `check` when you want the CI gate's exit semantics; reach for
+`review` while you're working.
+
+## Seeing a regression locally
+
+These are safe local experiments to confirm the suite would catch a real
+behavior drift. Revert the edit after observing the failure — none of
+these changes should be committed.
+
+1. **Change the system prompt** in your production agent module to a
+   different sentence, then run:
+
+   ```bash
+   agentprdiff check suites/{name}.py --case happy_path
+   ```
+
+   Expected result: the prompt-text grader (e.g. `system_prompt_is(...)`)
+   fails because the recorded LLM call no longer matches the baseline
+   contract.
+
+2. **Change a stub fixture** in `suites/_stubs.py` so it returns different
+   values from the case expects, then run:
+
+   ```bash
+   agentprdiff check suites/{name}.py --case happy_path
+   ```
+
+   Expected result: the entity / value graders fail and the trace diff
+   shows the output text changed.
+
+3. **Change an exception handler** in production code (e.g. so a failure
+   path returns a string where callers expect `None`), then run:
+
+   ```bash
+   agentprdiff check suites/{name}.py --case happy_path
+   ```
+
+   Expected result: the contract grader pinning the failure-mode return
+   value fails, proving callers would no longer receive the fallback
+   signal they depend on.
+
+## Updating the baseline after an intentional change
+
+When a behavior change is intentional, re-record the baseline for only
+the affected case and commit the resulting JSON diff so reviewers see the
+before/after:
+
+```bash
+agentprdiff record suites/{name}.py --case happy_path
 git add .agentprdiff/baselines/
 ```
 
+Drop `--case` to re-record every case at once. Either form is safe to
+re-run — `record` overwrites baselines in place, no accumulation.
+
 ## Cases
 
 ### `happy_path`
@@ -453,11 +570,19 @@ affected lines without re-reading the whole agent.
 One sentence; concrete and specific. ("Refunds silently fail" not "the
 agent misbehaves.")
 
+**How to exercise this case in isolation.**
+
+```bash
+agentprdiff check  suites/{name}.py --case happy_path     # CI-style, exit 1 on regression
+agentprdiff review suites/{name}.py --case happy_path     # verbose, exit 0
+agentprdiff record suites/{name}.py --case happy_path     # re-record after intentional change
+```
+
 ---
 
 <!--
 Template for additional cases. Copy-paste this block, change the heading,
-fill in the four sections.
+fill in the five sections plus the run-commands block.
 
 ### `<case_name>`
 
@@ -475,6 +600,14 @@ fill in the four sections.
 
 **Application impact.**
 
+**How to exercise this case in isolation.**
+
+```bash
+agentprdiff check  suites/{name}.py --case <case_name>
+agentprdiff review suites/{name}.py --case <case_name>
+agentprdiff record suites/{name}.py --case <case_name>
+```
+
 -->
 '''
 
@@ -505,16 +638,34 @@ jobs:
       - run: |
           python -m pip install --upgrade pip
           pip install -r requirements.txt agentprdiff
+          # Semantic-judge SDKs. The agentprdiff wheel imports these lazily,
+          # so installing only the one matching AGENTGUARD_JUDGE below keeps
+          # the CI environment lean. If your suite has no semantic() graders
+          # (see suites/README.md "Semantic Judge Keys"), drop both lines.
+          # Uncomment exactly one to match the chosen judge mode:
+          #   pip install anthropic     # if AGENTGUARD_JUDGE=anthropic
+          #   pip install openai        # if AGENTGUARD_JUDGE=openai (already installed if your agent uses OpenAI)
+          # Without the matching SDK, semantic() raises ImportError at first
+          # use and the case fails — louder than the silent fake_judge trap.
       - env:
           # TODO: match the env var your production agent reads.
           # Common names: OPENAI_API_KEY, ANTHROPIC_API_KEY, GOOGLE_API_KEY,
           # or a project-specific name. Add the secret in:
           # Settings -> Secrets and variables -> Actions -> New repository secret.
           OPENAI_API_KEY: ${{{{ secrets.OPENAI_API_KEY }}}}
-          # Optional: real semantic judge in CI. Without this, semantic()
-          # graders fall back to fake_judge (keyword matching) — see
-          # AGENTS.md "API keys" section. Omit this line entirely to use
-          # fake_judge and keep CI free.
+          # Semantic-judge mode (see AGENTS.md "Step 5b — decide and document
+          # the semantic-judge mode"). Pick one and uncomment; do NOT rely on
+          # the implicit selection order — it makes "which provider did I
+          # get?" ambiguous in CI logs. If your suite has no semantic()
+          # graders, leave all three commented and CI runs free.
+          #
+          #   AGENTGUARD_JUDGE: fake          # keyword matching, free
+          #   AGENTGUARD_JUDGE: anthropic     # real judge, cheaper; pair with ANTHROPIC_API_KEY below
+          #   AGENTGUARD_JUDGE: openai        # real judge; pair with OPENAI_API_KEY above
+          #
+          # Pair the chosen judge with its key. Without a key, semantic()
+          # falls back to fake_judge silently and you'll see PASS without
+          # the LLM judge ever running.
           ANTHROPIC_API_KEY: ${{{{ secrets.ANTHROPIC_API_KEY }}}}
         run: |
           if [ -z "${{OPENAI_API_KEY}}" ]; then