program-context-protocol 0.12.4__py3-none-any.whl

pcp/telemetry.py

@@ -0,0 +1,308 @@
+"""Per-build-cycle telemetry — file/line/language/qa-result granularity for analysis.
+
+Distinct from token_ledger.yaml (flat call-level cost rollup feeding pcp.md).
+This is JSONL — one record per build-cycle event (a coding attempt, or a QA
+check against that attempt) — so it loads straight into pandas/duckdb/jq for
+analysis without parsing nested YAML. Auto-appended by `pcp build`. Never edit.
+"""
+
+import json
+from collections import defaultdict
+from datetime import date, datetime, timezone
+from pathlib import Path
+
+from pcp.evidence_chain import chain_entry
+
+LANGUAGE_BY_EXT = {
+    ".py": "Python", ".ts": "TypeScript", ".tsx": "TypeScript", ".js": "JavaScript",
+    ".jsx": "JavaScript", ".go": "Go", ".rs": "Rust", ".java": "Java", ".rb": "Ruby",
+    ".yaml": "YAML", ".yml": "YAML", ".json": "JSON", ".md": "Markdown",
+    ".sql": "SQL", ".sh": "Shell", ".css": "CSS", ".html": "HTML", ".c": "C",
+    ".cpp": "C++", ".h": "C/C++ header", ".swift": "Swift", ".kt": "Kotlin",
+}
+
+
+def infer_languages(file_paths: list[str]) -> list[str]:
+    langs = set()
+    for f in file_paths:
+        ext = Path(f).suffix
+        langs.add(LANGUAGE_BY_EXT.get(ext, ext.lstrip(".") or "unknown"))
+    return sorted(langs)
+
+
+def count_diff_lines(diff: str) -> tuple[int, int]:
+    """(lines_added, lines_removed) from a unified diff, excluding +++/--- headers."""
+    added = sum(1 for l in diff.splitlines() if l.startswith("+") and not l.startswith("+++"))
+    removed = sum(1 for l in diff.splitlines() if l.startswith("-") and not l.startswith("---"))
+    return added, removed
+
+
+def record(pcp_dir: Path, **fields) -> None:
+    """Append one JSONL record to .pcp/telemetry.jsonl.
+
+    Suggested fields (not enforced — callers pass whatever's available):
+    module, submodule, criterion_id, cycle ("build"|"qa"), cycle_number (attempt #),
+    check (for qa: "layer1"|"architect-review"|"gate"), result ("pass"|"block"|"error"),
+    errors (list of finding strings), files (list of paths touched), languages,
+    lines_added, lines_removed, model, session_id, token_input, token_output,
+    token_cache_read, token_cache_creation, cost_usd, duration_ms.
+    """
+    fields = {"timestamp": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), **fields}
+    path = Path(pcp_dir) / "telemetry.jsonl"
+    entry = chain_entry(_last_entry_hash(path), fields)
+    with open(path, "a") as f:
+        f.write(json.dumps(entry) + "\n")
+
+
+def _last_entry_hash(path: Path) -> str | None:
+    if not path.exists():
+        return None
+    last_line = None
+    for line in path.read_text().splitlines():
+        line = line.strip()
+        if line:
+            last_line = line
+    if not last_line:
+        return None
+    try:
+        return json.loads(last_line).get("entry_hash")
+    except json.JSONDecodeError:
+        return None
+
+
+def load(pcp_dir: Path) -> list[dict]:
+    path = Path(pcp_dir) / "telemetry.jsonl"
+    if not path.exists():
+        return []
+    records = []
+    for line in path.read_text().splitlines():
+        line = line.strip()
+        if not line:
+            continue
+        try:
+            records.append(json.loads(line))
+        except json.JSONDecodeError:
+            continue
+    return records
+
+
+def productivity_by_week(records: list[dict]) -> list[dict]:
+    """Spend and net lines written per ISO week, plus $ per net line.
+
+    Nothing in PCP reported output-per-dollar over time, so a real 6x degradation
+    went unseen. Measured by hand on Project O 2026-07-30: 07-16..07-22
+    produced +4,954 net non-test LOC for $364.77 (~$0.07/line), while 07-23..07-30
+    produced +596 for $263.89 (~$0.44/line) -- while commits/day ROSE. Commit count
+    and output per dollar were pointing in opposite directions and only the flattering
+    one was visible anywhere.
+
+    Deliberately built from telemetry's own `lines_added`/`lines_removed`, not from
+    git, so this stays a pure aggregation over data PCP already owns and needs no
+    repo access. The honest reading of `net_lines` is therefore "lines this build
+    loop wrote, net" -- it counts every attempt's diff, including work a later
+    attempt superseded, so it is an upper bound on repo growth, not a measurement
+    of it. `$/net line` is a trend signal, not an accounting figure: watch whether
+    it moves, not what it equals.
+
+    Weeks with no recorded lines report None rather than a division-by-zero-shaped
+    number -- a week that spent money and wrote nothing is a real state and must not
+    be rendered as "$0.00/line".
+    """
+    by_week: dict[str, dict] = defaultdict(
+        lambda: {"cost": 0.0, "lines_added": 0, "lines_removed": 0, "attempts": 0}
+    )
+    for r in records:
+        ts = str(r.get("timestamp") or "")
+        if len(ts) < 10:
+            continue
+        try:
+            year, week, _ = date.fromisoformat(ts[:10]).isocalendar()
+        except ValueError:
+            continue
+        w = by_week[f"{year}-W{week:02d}"]
+        w["cost"] += r.get("cost_usd") or 0
+        w["lines_added"] += r.get("lines_added") or 0
+        w["lines_removed"] += r.get("lines_removed") or 0
+        if r.get("cycle") == "build":
+            w["attempts"] += 1
+
+    out = []
+    for label in sorted(by_week):
+        w = by_week[label]
+        net = w["lines_added"] - w["lines_removed"]
+        out.append({
+            "week": label,
+            "cost_usd": round(w["cost"], 2),
+            "net_lines": net,
+            "attempts": w["attempts"],
+            "usd_per_net_line": round(w["cost"] / net, 3) if net > 0 else None,
+        })
+    return out
+
+
+_EMPTY_REPO_LINES = {"by_week": {}, "bulk_commits_skipped": {}, "bulk_threshold": 0}
+
+
+def repo_net_lines_by_week(project_root: Path, exclude_tests: bool = True,
+                           bulk_commit_threshold: int = 5000) -> dict:
+    """Net authored source lines that actually LANDED in the repo, per ISO week.
+
+    Returns {"by_week": {week: net}, "bulk_commits_skipped": {week: n},
+    "bulk_threshold": n}. Callers must surface `bulk_commits_skipped` -- a metric
+    that quietly drops data is the failure mode this whole module is trying to fix.
+
+    This exists because `productivity_by_week` alone is misleading, and shipping it
+    alone would have repeated the exact failure this session kept finding: a metric
+    that reads healthy while the situation it describes is not.
+
+    On Project O, week 2026-W31: telemetry recorded **+12,342 net lines
+    written** at $0.018/line, which looks like the most productive week of the run.
+    Git says non-test code in the repo grew by **+599** over the same window. Both
+    are true. Telemetry counts every attempt's diff, so superseded attempts, reverted
+    work, rewrites and test code all inflate it; git counts what survived.
+
+    Neither number is the interesting one. **The ratio is** — roughly 5% of what the
+    loop wrote survived as net non-test product code. That is the number worth
+    watching, and no single-source metric can express it.
+
+    **Vendored third-party source is the hard case and the reason for
+    `bulk_commit_threshold`.** Extension filtering is not enough: Project O
+    committed an entire drawio distribution under `web/drawio-site/` and
+    `web/public/drawio/` in one week and moved it the next -- ~450,000 lines of
+    third-party `.js` sitting in no conventionally-named vendor directory. That
+    produced survival rates of 107372% and -11249% before this filter existed.
+
+    No path heuristic catches that reliably, so the discriminator is size: a single
+    commit churning more than `bulk_commit_threshold` authored-source lines is a
+    vendor import, a bulk move, or a generated dump -- not one criterion's work.
+    Such commits are excluded WHOLE, and the count is returned per week so the
+    caller can say so out loud. Never silently.
+
+    Best-effort: returns empty structures when git is unavailable or the call fails,
+    so callers degrade to the telemetry-only view rather than breaking.
+    """
+    import subprocess
+
+    try:
+        proc = subprocess.run(
+            ["git", "log", "--numstat", "--date=short", "--pretty=format:__C__%cd"],
+            cwd=project_root, capture_output=True, text=True, timeout=120,
+        )
+    except (OSError, subprocess.SubprocessError):
+        return _EMPTY_REPO_LINES
+    if proc.returncode != 0:
+        return _EMPTY_REPO_LINES
+
+    # Per-commit first, so a bulk vendor import can be excluded as a whole.
+    per_commit: list[tuple[str, int, int]] = []   # (week, net, churn)
+    current: str | None = None
+    net = churn = 0
+
+    def flush():
+        if current is not None:
+            per_commit.append((current, net, churn))
+
+    for line in proc.stdout.splitlines():
+        if line.startswith("__C__"):
+            flush()
+            net = churn = 0
+            try:
+                year, week, _ = date.fromisoformat(line[5:].strip()).isocalendar()
+                current = f"{year}-W{week:02d}"
+            except ValueError:
+                current = None
+            continue
+        if not current or "\t" not in line:
+            continue
+        parts = line.split("\t")
+        # Binary files report "-" for both counts; skip rather than crash.
+        if len(parts) != 3 or not parts[0].isdigit() or not parts[1].isdigit():
+            continue
+        path = parts[2]
+        if not _is_authored_source(path):
+            continue
+        if exclude_tests and (path.startswith("tests/") or "/test_" in path
+                              or path.split("/")[-1].startswith("test_")):
+            continue
+        added, removed = int(parts[0]), int(parts[1])
+        net += added - removed
+        churn += added + removed
+    flush()
+
+    by_week: dict[str, int] = defaultdict(int)
+    skipped: dict[str, int] = defaultdict(int)
+    for week, cnet, cchurn in per_commit:
+        if cchurn > bulk_commit_threshold:
+            skipped[week] += 1
+            continue
+        by_week[week] += cnet
+    return {"by_week": dict(by_week), "bulk_commits_skipped": dict(skipped),
+            "bulk_threshold": bulk_commit_threshold}
+
+
+# Extensions a human or an agent actually authors. Everything else -- lockfiles,
+# bundles, vendored trees, snapshots, data -- is generated or copied, and counting
+# it destroys the metric rather than enriching it.
+_SOURCE_EXTS = frozenset({
+    ".py", ".ts", ".tsx", ".js", ".jsx", ".go", ".rs", ".java", ".rb", ".kt",
+    ".swift", ".c", ".cpp", ".h", ".cs", ".php", ".scala", ".ex", ".exs",
+    ".sh", ".sql", ".vue", ".svelte", ".css", ".scss", ".html",
+})
+_GENERATED_DIR_MARKERS = (
+    "node_modules/", ".venv/", "venv/", "vendor/", "dist/", "build/", "site-packages/",
+    "__pycache__/", ".next/", "coverage/", "migrations/", "__snapshots__/", "generated/",
+)
+_GENERATED_NAMES = frozenset({
+    "package-lock.json", "yarn.lock", "pnpm-lock.yaml", "poetry.lock",
+    "Cargo.lock", "go.sum", "uv.lock", "composer.lock",
+})
+
+
+def _is_authored_source(path: str) -> bool:
+    """Is this a file a person or agent wrote, rather than one a tool emitted?
+
+    Without this filter the metric is not merely noisy, it is nonsense. Run against
+    Project O it reported **+1,523,5xx lines landed** in one week and
+    **-1,470,6xx** in another, for survival rates of 362745% and -38701% -- lockfiles
+    and generated bundles swamping the signal by three orders of magnitude.
+
+    Worth stating plainly: that output was produced by a first version of this
+    function and caught only by running it against a real repo before shipping.
+    A metric that confidently reports a wrong number is worse than no metric, which
+    is the same defect class as every other finding in this session's audit.
+    """
+    name = path.split("/")[-1]
+    if name in _GENERATED_NAMES:
+        return False
+    if any(marker in path for marker in _GENERATED_DIR_MARKERS):
+        return False
+    dot = name.rfind(".")
+    return dot > 0 and name[dot:].lower() in _SOURCE_EXTS
+
+
+def aggregate(records: list[dict]) -> dict:
+    """Roll up build/qa records per module. Shared by `pcp telemetry`, end-of-build
+    summary, and the pcp.md 'Build Efficiency' section — one aggregation, three views."""
+    build_records = [r for r in records if r.get("cycle") == "build"]
+    qa_records = [r for r in records if r.get("cycle") == "qa"]
+
+    by_module = defaultdict(lambda: {
+        "attempts": 0, "criteria": set(), "tokens_in": 0, "tokens_out": 0,
+        "tokens_cache_read": 0, "cost": 0.0, "qa_blocks": 0, "qa_total": 0, "languages": set(),
+    })
+    for r in build_records:
+        m = by_module[r.get("module") or "?"]
+        m["attempts"] += 1
+        m["criteria"].add(r.get("criterion_id"))
+        m["tokens_in"] += r.get("token_input", 0)
+        m["tokens_out"] += r.get("token_output", 0)
+        m["tokens_cache_read"] += r.get("token_cache_read", 0)
+        m["cost"] += r.get("cost_usd") or 0
+        m["languages"].update(r.get("languages") or [])
+    for r in qa_records:
+        m = by_module[r.get("module") or "?"]
+        m["qa_total"] += 1
+        if r.get("result") == "block":
+            m["qa_blocks"] += 1
+
+    return {"by_module": by_module, "build_records": build_records, "qa_records": qa_records}

pcp/uat.py

@@ -0,0 +1,271 @@
+"""UAT checks — `url_responds`, `dom_contains`, and `visual` acceptance criteria,
+plus two advisory checks build.py runs on top of a rendered screenshot:
+`check_axe` (deterministic a11y scan) and `check_visual_quality` (checklist-
+anchored VLM judge).
+
+Honest scope: `url_responds`/`dom_contains` are deterministic, no browser
+involved — `dom_contains` fetches the raw HTML response and searches it as
+text, so it does NOT execute JavaScript and content only rendered
+client-side (a typical SPA) won't be found even if a real browser would
+show it. `visual` (check_visual) closes part of that gap with a real
+headless browser via Playwright — an OPTIONAL dependency
+(`pip install program-context-protocol[visual]`), never a hard requirement
+of this package. It proves the page renders without crashing/timing out
+and saves a screenshot for human review.
+
+**Updated 2026-07-18** — layout-break detection via a vision LLM is now
+built (`check_visual_quality`), closing the gap this docstring used to name
+as out of scope. What changed: `llm/client.py` gained image-input plumbing
+(`call_with_image`/`call_json_with_image`, via `claude -p`'s
+`--input-format stream-json` multimodal message shape). Deliberately
+**checklist-anchored, not a freeform "does this look good" prompt** —
+research (ArtifactsBench, 2026) found a checklist-anchored VLM judge hits
+~94% human-correlation vs. ~21% for a bare Nielsen-heuristics-style review;
+the checklist is what does the work, not the model. Same advisory,
+never-a-hard-block posture as `check_visual`'s baseline-diff note below —
+a screen scoring poorly on the checklist is a review signal, not proof the
+screen is wrong.
+
+Reconnaissance-then-action pattern (wait for `networkidle` before reading
+DOM state) is a reference-pattern borrowed from Anthropic's own
+`webapp-testing` skill (anthropics/skills) after a real prior-art miss
+found post-hoc: `check_visual` shipped without checking for it first,
+initially screenshotting right after `goto()` with no settle-wait — on a
+slow-loading SPA that can capture a blank/loading state and still report
+"rendered successfully," exactly the failure mode this check exists to
+catch. `webapp-testing` itself is a full agentic skill (server lifecycle,
+selector discovery, browser console logs) meant for an interactive Claude
+session — not adopted wholesale, since `check_visual` runs inside `pcp
+scan`'s deterministic, zero-LLM evaluation loop and turning that into an
+agent invocation per UI criterion would break Token Discipline. Only the
+underlying technique was reused; `pcp build`'s coding agent is pointed at
+the real skill separately for its own in-build UI verification.
+
+Same tool-wrapping shape as qa.py: never raises, degrades to a clear
+failure detail instead.
+"""
+
+import re
+import shutil
+import subprocess
+import urllib.error
+import urllib.request
+from pathlib import Path
+
+TIMEOUT_SEC = 10
+TIMEOUT_AXE = 120
+
+# Checklist-anchored, per the research finding above -- deliberately generic
+# and small rather than exhaustive, same "advisory signal, not proof" posture
+# as check_visual's baseline diff. A criterion's own design_justification
+# (design_system.md tokens, jtbd_framing) is NOT folded in here as additional
+# checklist items -- that field is a self-report the agent fills in, judging
+# a screen against the agent's own claims about itself would be circular.
+DEFAULT_VISUAL_CHECKLIST = [
+    "layout is not visibly broken (no overlapping elements, no obvious clipping/overflow)",
+    "text is legible (adequate contrast against its background, not truncated where it shouldn't be)",
+    "primary action or focal element is visually prominent and easy to locate",
+    "spacing/alignment reads as intentional, not haphazard",
+]
+
+
+def check_url_responds(url: str) -> tuple[bool, str]:
+    """Pass if the URL returns a 2xx/3xx status. No content is inspected."""
+    if not url:
+        return False, "no url configured for url_responds check"
+    try:
+        req = urllib.request.Request(url, method="GET")
+        with urllib.request.urlopen(req, timeout=TIMEOUT_SEC) as resp:
+            ok = 200 <= resp.status < 400
+            return ok, f"{url} responded {resp.status}"
+    except urllib.error.HTTPError as e:
+        return False, f"{url} responded {e.code}"
+    except Exception as e:
+        return False, f"{url} did not respond: {e}"
+
+
+def check_dom_contains(url: str, selector: str) -> tuple[bool, str]:
+    """Pass if `selector` (plain text, or a regex if it fails to match literally)
+    appears in the URL's raw HTML response. Static content only — see module
+    docstring for the JS-rendering limitation."""
+    if not url:
+        return False, "no url configured for dom_contains check"
+    if not selector:
+        return False, "no selector/text configured for dom_contains check"
+    try:
+        req = urllib.request.Request(url, method="GET")
+        with urllib.request.urlopen(req, timeout=TIMEOUT_SEC) as resp:
+            body = resp.read().decode(errors="replace")
+    except Exception as e:
+        return False, f"{url} did not respond: {e}"
+
+    if selector in body:
+        return True, f"'{selector}' found in {url} (static HTML)"
+    try:
+        if re.search(selector, body):
+            return True, f"pattern '{selector}' matched in {url} (static HTML)"
+    except re.error:
+        pass
+    return False, f"'{selector}' not found in {url}'s static HTML (JS-rendered content won't show here)"
+
+
+def check_visual(url: str, screenshot_path: Path | None = None) -> tuple[bool | None, str]:
+    """Loads `url` in a real headless browser (Playwright/Chromium) and
+    captures a screenshot. Returns (None, detail) — not (False, detail) —
+    when playwright isn't installed: this means "could not check", not
+    "failed". Callers must preserve whatever status a criterion already had
+    rather than downgrading it on a missing optional dependency, the same
+    posture a manual/visual criterion without this check already gets in
+    scan.py."""
+    if not url:
+        return False, "no url configured for visual check"
+    try:
+        from playwright.sync_api import sync_playwright
+    except ImportError:
+        return None, (
+            "playwright not installed -- visual check skipped, not failed. "
+            "Install with: pip install program-context-protocol[visual] && playwright install chromium"
+        )
+    try:
+        with sync_playwright() as p:
+            browser = p.chromium.launch()
+            page = browser.new_page()
+            page.goto(url, timeout=TIMEOUT_SEC * 1000)
+            # Reconnaissance-then-action pattern (reference: anthropics/skills'
+            # webapp-testing) -- goto() alone returns once the initial HTML
+            # response lands, before a typical SPA's JS has actually rendered
+            # content. Without this wait, a screenshot on a slow-loading SPA
+            # can capture a blank/loading state and still report "rendered
+            # successfully" -- exactly the failure mode this check exists to
+            # catch (dom_contains's own JS-rendering gap). networkidle waits
+            # for in-flight requests to settle before the check reads DOM state.
+            page.wait_for_load_state("networkidle", timeout=TIMEOUT_SEC * 1000)
+            if screenshot_path:
+                screenshot_path.parent.mkdir(parents=True, exist_ok=True)
+                page.screenshot(path=str(screenshot_path), full_page=True)
+            browser.close()
+    except Exception as e:
+        return False, f"{url} failed to render in a headless browser: {e}"
+
+    detail = f"{url} rendered successfully in a headless browser"
+    if screenshot_path:
+        detail += f" -- screenshot: {screenshot_path}"
+        detail += _baseline_note(screenshot_path)
+    return True, detail
+
+
+def _baseline_note(screenshot_path: Path) -> str:
+    """Baseline comparison (Chromatic reference pattern, 2026-07-17, build
+    plan 3.5) — closes part of check_visual's own stated 'not visual
+    regression testing' gap. First successful capture becomes the baseline
+    (`<name>_baseline.png`); later captures are compared by content hash.
+    HONEST SCOPE: hash inequality means "pixels changed since the accepted
+    baseline", not "layout broke" — a changed screenshot is a review signal,
+    never a failure. Accept a new baseline by deleting the old one."""
+    import hashlib
+    baseline = screenshot_path.with_name(screenshot_path.stem + "_baseline.png")
+    try:
+        current = hashlib.sha256(screenshot_path.read_bytes()).hexdigest()
+        if not baseline.exists():
+            baseline.write_bytes(screenshot_path.read_bytes())
+            return " -- baseline established (first capture)"
+        if hashlib.sha256(baseline.read_bytes()).hexdigest() == current:
+            return " -- matches accepted baseline"
+        return (f" -- CHANGED vs accepted baseline ({baseline.name}); review the two "
+                "screenshots and delete the baseline to accept the new look")
+    except OSError as e:
+        return f" -- baseline comparison skipped: {e}"
+
+
+def check_axe(url: str) -> tuple[bool | None, str]:
+    """Deterministic WCAG a11y scan via `@axe-core/cli` (npx, auto-installed
+    on first run same as the Context7 MCP entry doctor.py already scaffolds
+    via npx -- no new hard dependency added to this package). Returns
+    (None, detail) when npx isn't on PATH -- "could not check", same
+    could-not-check-vs-failed distinction check_visual already makes for a
+    missing optional dependency. --exit makes the CLI process exit 1 if any
+    rule fails; --stdout silences everything but the results/errors so the
+    tail of stdout is a usable detail message on failure."""
+    if not url:
+        return False, "no url configured for axe a11y check"
+    if not shutil.which("npx"):
+        return None, "npx not found -- axe-core a11y scan skipped, not failed"
+    try:
+        result = subprocess.run(
+            ["npx", "--yes", "@axe-core/cli", url, "--exit", "--stdout"],
+            capture_output=True, text=True, timeout=TIMEOUT_AXE,
+        )
+    except subprocess.TimeoutExpired:
+        return False, f"axe-core scan of {url} timed out after {TIMEOUT_AXE}s"
+    except Exception as e:
+        return False, f"axe-core scan of {url} failed to run: {e}"
+
+    output = (result.stdout + result.stderr).strip()
+    if result.returncode == 0:
+        return True, f"axe-core: no violations found at {url}"
+    return False, f"axe-core found violation(s) at {url}:\n{output[-3000:]}"
+
+
+def check_visual_quality(
+    screenshot_path: Path,
+    checklist: list[str] | None = None,
+    reference_image_path: Path | None = None,
+    model: str | None = None,
+    pcp_dir: Path | None = None,
+) -> tuple[bool | None, str, list[dict]]:
+    """Checklist-anchored VLM judge over a screenshot check_visual already
+    captured. Returns (None, detail, []) if the screenshot doesn't exist
+    (nothing to judge -- same could-not-check posture as a missing optional
+    dependency elsewhere in this module) or if the judge call itself errors
+    (advisory check, never let an LLM-call failure read as a verdict).
+
+    reference_image_path, if given, is attached as a second inline image so
+    the judge can compare against it -- research finding: screenshot +
+    reference beats either alone as grounding context. Comparison is
+    layout/structure, not pixel-perfect match; the prompt says so explicitly
+    so the judge doesn't fail a screen for a legitimate content difference.
+    """
+    if not screenshot_path or not screenshot_path.exists():
+        return None, "no screenshot available to judge (check_visual must run first)", []
+
+    from pcp.llm import client as llm
+
+    items = checklist or DEFAULT_VISUAL_CHECKLIST
+    checklist_text = "\n".join(f"- {item}" for item in items)
+    system = (
+        "You judge a rendered UI screenshot against a fixed checklist. For EACH "
+        "checklist item, decide pass/fail and give a one-sentence reason grounded "
+        "in what you actually see in the image -- never invent a defect that isn't "
+        "visible. If a reference image is also attached, use it only to judge "
+        "layout/structure similarity, not pixel-perfect match; a legitimate content "
+        "difference (different copy, different data) is not a failure."
+    )
+    user = (
+        f"Checklist:\n{checklist_text}\n\n"
+        "Respond with JSON: "
+        '{"items": [{"item": "<checklist item text>", "passed": true|false, "reason": "..."}], '
+        '"overall_passed": true|false}. overall_passed is true only if every item passed.'
+    )
+    if reference_image_path and reference_image_path.exists():
+        user += "\n\nA reference image is attached second, after the rendered screenshot, for comparison."
+
+    image_paths = [screenshot_path]
+    if reference_image_path and reference_image_path.exists():
+        image_paths.append(reference_image_path)
+
+    try:
+        verdict = llm.call_json_with_images(
+            system, user, image_paths,
+            model=model or llm.JUDGE_MODEL, pcp_dir=pcp_dir, command="uat.check_visual_quality",
+        )
+    except Exception as e:
+        return None, f"visual-quality judge call failed: {e}", []
+
+    checked_items = verdict.get("items", [])
+    overall = bool(verdict.get("overall_passed", all(i.get("passed") for i in checked_items)))
+    failed = [i for i in checked_items if not i.get("passed")]
+    if not overall and failed:
+        detail = "; ".join(f"{i.get('item', '?')}: {i.get('reason', '')}" for i in failed)
+    else:
+        detail = "all checklist items passed"
+    return overall, detail, checked_items