pi-dev-team 0.2.0

package/skills/code-review/scripts/contract_failure_report.py

@@ -0,0 +1,185 @@
+#!/usr/bin/env python3
+"""Per-agent contract-parse failure rate (#1998).
+
+`validate_review_output.py` logs one row to `contract-failures.jsonl` per
+review-agent output that fails the shared JSON contract. On its own that log
+answers "how many failures" but not "out of how many dispatches" — the same
+denominator problem `review_value_coverage.py` solved for `review-value.jsonl`
+by joining against `agent_dispatch_ledger.py`'s deterministic
+`boundary-events.jsonl` "record" rows. This module does the same join for the
+failure log: `rate = failures / dispatches`, per agent, so #1980/#1982 can
+read a real per-lens `$/finding` denominator instead of a lossy one, and so
+this rate is reported as a metric rather than asserted from memory (#1998
+acceptance criterion 2).
+
+The ledger-reading half of that join (`read_jsonl`, `dispatch_counts`, the
+`boundary-events.jsonl` predicate) lives in `hooks/lib/review_dispatch_ledger.py`,
+shared with `review_value_coverage.py` — both used to hand-roll an identical
+copy; see that module's docstring.
+
+Caveat on the rate: the denominator (`boundary-events.jsonl`'s
+`agent_dispatch_ledger` "record" rows) counts every dispatch of a registered
+review agent, from any caller — `/code-review`, `/build`'s inline checkpoints,
+`/repo-review` — while `validate_review_output.py` (the numerator) is only
+ever invoked from `/code-review` step 4 and sliced-mode's equivalent step.
+Dispatches this module never contract-checked still inflate the denominator
+and can never contribute to the numerator, so the reported rate is a LOWER
+bound on the true per-agent contract-failure rate, not an exact figure.
+
+Stdlib-only. See docs/python-hook-contract.md.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from collections import Counter, defaultdict
+from pathlib import Path
+
+# skills/code-review/scripts -> skills/code-review -> skills -> plugin root
+_PLUGIN_ROOT = Path(__file__).resolve().parents[3]
+_HOOKS_LIB_DIR = _PLUGIN_ROOT / "hooks" / "lib"
+if str(_HOOKS_LIB_DIR) not in sys.path:
+    sys.path.insert(0, str(_HOOKS_LIB_DIR))
+
+try:
+    import review_dispatch_ledger  # type: ignore[import-not-found]
+except ImportError:  # pragma: no cover - degraded fallback, hooks/lib unreachable
+    review_dispatch_ledger = None
+
+FAILURE_STREAM = "contract-failures.jsonl"
+LEDGER_STREAM = review_dispatch_ledger.LEDGER_STREAM if review_dispatch_ledger else "boundary-events.jsonl"
+
+
+def _resolve(stream: str, cwd: Path) -> Path:
+    # Read-only report: migrate=False so running it never migrates a legacy
+    # top-level metrics/ file or creates .claude/metrics/ as a side effect
+    # (mirrors hooks/lib/metrics_query.py::_stream_path's same reasoning).
+    if review_dispatch_ledger is not None:
+        return review_dispatch_ledger.resolve_stream("metrics", stream, cwd, migrate=False)
+    return cwd / ".claude" / "metrics" / stream
+
+
+def read_jsonl(path: Path) -> tuple[list[dict], int]:
+    """Return (rows, malformed_count). Delegates to
+    `hooks/lib/review_dispatch_ledger.py` (#1998); degrades to an empty
+    result (fail-open, same posture as `validate_review_output.log_failure`)
+    rather than re-deriving the ledger's wire shape by hand a third time if
+    `hooks/lib` is unreachable."""
+    if review_dispatch_ledger is None:  # pragma: no cover - degraded fallback
+        return [], 0
+    return review_dispatch_ledger.read_jsonl(path)
+
+
+def dispatch_counts(ledger_rows) -> Counter:
+    """Per-agent dispatch counts from the deterministic ledger. Delegates to
+    `hooks/lib/review_dispatch_ledger.py` (#1998); see `read_jsonl` above for
+    why the fallback degrades rather than re-implements."""
+    if review_dispatch_ledger is None:  # pragma: no cover - degraded fallback
+        return Counter()
+    return review_dispatch_ledger.dispatch_counts(ledger_rows)
+
+
+def failure_counts(failure_rows) -> tuple[Counter, dict]:
+    """Per-agent failure totals, plus a per-agent shape breakdown."""
+    counts: Counter = Counter()
+    shapes: dict = defaultdict(Counter)
+    for row in failure_rows:
+        agent = row.get("agent")
+        if not (isinstance(agent, str) and agent.strip()):
+            continue
+        agent = agent.strip()
+        counts[agent] += 1
+        shape = row.get("shape")
+        if isinstance(shape, str) and shape.strip():
+            shapes[agent][shape.strip()] += 1
+    return counts, shapes
+
+
+def build_report(failure_rows, ledger_rows) -> dict:
+    dispatches = dispatch_counts(ledger_rows)
+    failures, shapes = failure_counts(failure_rows)
+    total_dispatches = sum(dispatches.values())
+    total_failures = sum(failures.values())
+
+    per_agent = {}
+    for agent in sorted(set(dispatches) | set(failures)):
+        agent_dispatches = dispatches.get(agent, 0)
+        agent_failures = failures.get(agent, 0)
+        per_agent[agent] = {
+            "dispatches": agent_dispatches,
+            "failures": agent_failures,
+            "rate": (agent_failures / agent_dispatches) if agent_dispatches else None,
+            "shapes": dict(sorted(shapes.get(agent, {}).items())),
+        }
+
+    return {
+        "totals": {
+            "dispatches": total_dispatches,
+            "failures": total_failures,
+            "rate": (total_failures / total_dispatches) if total_dispatches else None,
+        },
+        "per_agent": per_agent,
+    }
+
+
+def run(cwd: Path) -> dict:
+    failure_rows, failure_malformed = read_jsonl(_resolve(FAILURE_STREAM, cwd))
+    ledger_rows, ledger_malformed = read_jsonl(_resolve(LEDGER_STREAM, cwd))
+    report = build_report(failure_rows, ledger_rows)
+    report["malformed_lines"] = {
+        FAILURE_STREAM: failure_malformed,
+        LEDGER_STREAM: ledger_malformed,
+    }
+    return report
+
+
+def _format_text(report: dict) -> str:
+    totals = report["totals"]
+    rate = totals["rate"]
+    lines = [
+        (
+            f"contract-parse failure rate: {'n/a' if rate is None else f'{rate:.1%}'} "
+            f"({totals['failures']}/{totals['dispatches']} dispatches)"
+        ),
+        (
+            "  note: dispatches counts every agent_dispatch_ledger record, including "
+            "callers this module never contract-checks (e.g. /build inline checkpoints) "
+            "— this rate is a lower bound, not exact"
+        ),
+    ]
+    for agent, row in sorted(
+        report["per_agent"].items(),
+        key=lambda kv: (kv[1]["rate"] if kv[1]["rate"] is not None else -1),
+        reverse=True,
+    ):
+        agent_rate = row["rate"]
+        lines.append(
+            f"  {agent}: {'n/a' if agent_rate is None else f'{agent_rate:.0%}'} "
+            f"({row['failures']}/{row['dispatches']}) {row['shapes']}"
+        )
+    return "\n".join(lines)
+
+
+def main(argv=None) -> int:
+    parser = argparse.ArgumentParser(
+        description="Report per-agent contract-parse failure rate from contract-failures.jsonl"
+    )
+    parser.add_argument("--cwd", default=None)
+    parser.add_argument("--json", action="store_true", dest="as_json")
+    args = parser.parse_args(argv)
+
+    report = run(Path(args.cwd) if args.cwd else Path.cwd())
+    if args.as_json:
+        print(json.dumps(report, indent=2, sort_keys=True))
+    else:
+        print(_format_text(report))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main(sys.argv[1:]))
+
+
+__all__ = ("build_report", "dispatch_counts", "failure_counts", "read_jsonl", "run")

package/skills/code-review/scripts/dispatch_reconcile.py

@@ -0,0 +1,66 @@
+#!/usr/bin/env python3
+"""Compute the dispatched-but-not-returned set difference for a review roster (#1761).
+
+`/code-review` Step 4 dispatches an agent roster and then has to notice when
+one of them didn't come back with a contract-valid result. Today that
+comparison is eyeballed by the orchestrator rather than computed — this
+module is the deterministic half of the fix, mirroring `dispatch_waves.py`'s
+own precedent of extracting a policy decision the orchestrator used to make
+by inspection into a pure, testable function.
+
+Pure policy — no filesystem, no network. Stdlib-only.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+
+
+def reconcile_dispatch(dispatched: list[str], returned: set[str]) -> list[str]:
+    """Return the names in ``dispatched`` that do not appear in ``returned``.
+
+    Preserves ``dispatched``'s order and de-duplicates: a name repeated in
+    ``dispatched`` is reported at most once, at its first-seen position. A
+    name in ``returned`` that isn't in ``dispatched`` at all is ignored —
+    this function only ever answers "who was dispatched but didn't come
+    back," never "who came back unexpectedly." An empty ``dispatched``
+    always yields ``[]``, never an error.
+    """
+    seen: set[str] = set()
+    missing: list[str] = []
+    for name in dispatched:
+        if name in seen:
+            continue
+        seen.add(name)
+        if name not in returned:
+            missing.append(name)
+    return missing
+
+
+def main(argv=None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--dispatched", required=True,
+        help="comma-separated agent names that were dispatched, in roster order",
+    )
+    parser.add_argument(
+        "--returned", required=True,
+        help="comma-separated agent names that returned a contract-valid result",
+    )
+    args = parser.parse_args(argv)
+
+    dispatched = [a.strip() for a in args.dispatched.split(",") if a.strip()]
+    returned = {a.strip() for a in args.returned.split(",") if a.strip()}
+
+    missing = reconcile_dispatch(dispatched, returned)
+    print(json.dumps({"missing": missing}, sort_keys=True))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main(sys.argv[1:]))
+
+
+__all__ = ("reconcile_dispatch",)

package/skills/code-review/scripts/dispatch_waves.py

@@ -0,0 +1,164 @@
+#!/usr/bin/env python3
+"""Split a review's agent roster into bounded, sequential dispatch waves (#1752).
+
+`/code-review` Step 4 used to spawn every eligible agent as parallel `Agent`
+tool calls in a single message, regardless of roster size. A real run with 16
+agents in one message lost its last 6 to `[Tool result missing due to
+internal error]` — the first 10 calls returned normal `agentId`s with the
+identical parameter shape, so this reads as the harness dropping calls past
+some concurrent-dispatch ceiling within that one message, not a malformed
+request. (The incident observed 10 succeeding as the first slice of a
+16-call message; it does not, on its own, prove 10 is the largest size that
+would succeed as its own dedicated message — see `DEFAULT_MAX_PARALLEL`
+below.) Nothing detected the gap: the review report simply had six fewer
+lenses' worth of findings.
+
+This module is the deterministic half of the fix — it decides *how many
+agents go out per message* and *how the roster splits into waves*, in the
+order it's given. It does not decide that order (that's the orchestrator's
+job, assembling `--agents` from Step 3's eligibility gates in `SKILL.md`) and
+it cannot decide whether a given wave's dispatch actually succeeded (that only
+the orchestrator, holding the real `Agent` tool, can observe) — `SKILL.md`'s
+retry-once-then-report handling covers that half.
+
+Pure policy — no filesystem, no network. The only ambient read is
+`os.environ`, and it's injectable via `resolve_max_parallel`'s `env`
+parameter for testability. Stdlib-only.
+
+**A note on "wave"**: this repo uses the word for three different things.
+`scripts/orchestrator.py`'s wave carries barrier-and-reconcile semantics
+(a failed slice raises and blocks state persistence); `scripts/plan_waves.py`
+derives wave membership from step *dependencies*. This module's wave is
+neither — just a fixed-size slice of an already-ordered roster, with no
+barrier or reconciliation semantics of its own.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+
+# Shares the DEV_TEAM_MAX_PARALLEL_* naming convention with build_jobs.py's
+# DEV_TEAM_MAX_PARALLEL_BUILDS, but inverts its clamping direction: that knob
+# is a worker *floor* (fan-out is opt-in, so malformed/unset -> 1, sequential);
+# this one is a *ceiling* on an already-parallel panel, so malformed/unset ->
+# DEFAULT_MAX_PARALLEL rather than 1 — collapsing to 1 here would silently
+# serialize a review that's supposed to stay parallel.
+MAX_PARALLEL_ENV_VAR = "DEV_TEAM_MAX_PARALLEL_REVIEW_AGENTS"
+
+# 10 is what stayed reliable in the #1752 incident (the first 10 of a 16-call
+# message all returned valid agentIds) — a real observation, not a measured
+# ceiling: it doesn't by itself prove 10 is the *largest* size that would
+# succeed as its own dedicated message. Revisit if real usage shows this
+# miscalibrated in either direction.
+DEFAULT_MAX_PARALLEL = 10
+
+# A pathologically long digit string is never a real configuration value, and
+# is believed to trip CPython's int-string conversion limit inside `int()` on
+# 3.11+ (default 4300 digits, per CVE-2020-10735 — not independently probed
+# in this repo) — a limit that doesn't exist on this repo's 3.10 floor (ADR
+# 0031), so this guard is a defensive ceiling independent of which
+# interpreter runs it, not a floor-specific workaround. Kept well under
+# either boundary so a malformed override never crashes instead of falling
+# back.
+_MAX_ENV_DIGITS = 12
+
+
+def _is_valid_max_parallel(value: object) -> bool:
+    """One predicate, shared by both callers that need "a valid wave size":
+    :func:`resolve_max_parallel`'s override branch (falls back on failure)
+    and :func:`form_dispatch_waves` (raises on failure). ``bool`` is a
+    subclass of ``int`` in Python, so it's explicitly excluded — `True` would
+    otherwise pass as `1`."""
+    return isinstance(value, int) and not isinstance(value, bool) and value > 0
+
+
+def resolve_max_parallel(env: dict | None = None, override: int | None = None) -> int:
+    """Return the effective max-parallel-agents-per-wave value.
+
+    ``override`` (e.g. an explicit `--max-parallel` flag) wins whenever it is
+    given at all — valid or not. An invalid override (non-int, bool, or
+    non-positive) falls back to :data:`DEFAULT_MAX_PARALLEL` and does **not**
+    fall through to the env: an explicit flag always takes precedence over
+    config, so a malformed flag value is a request to use the safe default,
+    not a signal to consult the environment instead.
+
+    Otherwise reads ``DEV_TEAM_MAX_PARALLEL_REVIEW_AGENTS`` from ``env``
+    (defaults to ``os.environ``), tolerating incidental surrounding
+    whitespace. Unset, empty, too long to parse safely, non-integer, or
+    non-positive values all fall back to :data:`DEFAULT_MAX_PARALLEL` —
+    deterministic, never an error, and never silently degrading concurrency
+    to 1 just because an override was malformed.
+    """
+    if override is not None:
+        return override if _is_valid_max_parallel(override) else DEFAULT_MAX_PARALLEL
+
+    source = os.environ if env is None else env
+    raw = source.get(MAX_PARALLEL_ENV_VAR, "").strip()
+    if not raw:
+        return DEFAULT_MAX_PARALLEL
+    if not (raw.isascii() and raw.isdigit()) or len(raw) > _MAX_ENV_DIGITS:
+        return DEFAULT_MAX_PARALLEL
+    try:
+        value = int(raw)
+    except ValueError:
+        return DEFAULT_MAX_PARALLEL
+    return value if _is_valid_max_parallel(value) else DEFAULT_MAX_PARALLEL
+
+
+def form_dispatch_waves(agents: list[str], max_parallel: int) -> list[list[str]]:
+    """Split ``agents`` into ordered waves of at most ``max_parallel`` each.
+
+    The per-message dispatch ceiling, the wave size, and ``max_parallel``
+    below are the same number — ``maxParallel`` is its name in the CLI's JSON
+    output and in the env knob (:data:`MAX_PARALLEL_ENV_VAR`).
+
+    Roster order is preserved both within and across waves, so a fixed input
+    always yields the same waves in the same order — this function does not
+    decide that order, only where it gets cut. An empty roster yields ``[]``.
+    Raises ``ValueError`` if ``max_parallel`` is not a positive integer —
+    callers should route through :func:`resolve_max_parallel`, which never
+    returns an invalid value, but this function guards its own contract too.
+    """
+    if not _is_valid_max_parallel(max_parallel):
+        raise ValueError(f"max_parallel must be a positive integer, got {max_parallel!r}")
+    if not agents:
+        return []
+    return [
+        list(agents[start : start + max_parallel])
+        for start in range(0, len(agents), max_parallel)
+    ]
+
+
+def main(argv=None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--agents", required=True,
+        help="comma-separated agent names to dispatch, in roster order",
+    )
+    parser.add_argument(
+        "--max-parallel", type=int, default=None,
+        help=f"override the resolved max (default: env {MAX_PARALLEL_ENV_VAR} or {DEFAULT_MAX_PARALLEL})",
+    )
+    args = parser.parse_args(argv)
+
+    agents = [a.strip() for a in args.agents.split(",") if a.strip()]
+    max_parallel = resolve_max_parallel(override=args.max_parallel)
+
+    waves = form_dispatch_waves(agents, max_parallel)
+    print(json.dumps({"maxParallel": max_parallel, "waves": waves}, sort_keys=True))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main(sys.argv[1:]))
+
+
+__all__ = (
+    "DEFAULT_MAX_PARALLEL",
+    "MAX_PARALLEL_ENV_VAR",
+    "form_dispatch_waves",
+    "resolve_max_parallel",
+)