obey-cli 0.1.0__py3-none-any.whl

obey_cli/score.py

@@ -0,0 +1,150 @@
+"""Per-rule compliance from checker verdicts.
+
+Compliance is obeyed / exercised, with a Wilson 95% interval. Runs where the rule was never exercised are
+excluded from the ratio and reported as the exercised share next to it. In an enforced arm, a run that a hook
+blocked at least once and that still ends with the rule violated counts as an evasion.
+"""
+
+from __future__ import annotations
+
+import math
+from collections import defaultdict
+from collections.abc import Iterable, Mapping
+from dataclasses import dataclass, field
+from typing import Any
+
+FOLLOWED_THRESHOLD = 0.9
+AGENT_NAMES = {
+    "claude-code": "Claude Code",
+    "codex": "Codex CLI",
+    "opencode": "OpenCode",
+    "gemini": "Gemini CLI",
+    "copilot": "Copilot CLI",
+    "cursor": "Cursor CLI",
+}
+
+
+def wilson(k: int, n: int, z: float = 1.959963984540054) -> tuple[float, float]:
+    """Wilson score interval for k successes out of n (0..1). (0, 1) when n is 0."""
+    if n <= 0:
+        return (0.0, 1.0)
+    p = k / n
+    denom = 1 + z * z / n
+    centre = (p + z * z / (2 * n)) / denom
+    half = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / denom
+    return (max(0.0, centre - half), min(1.0, centre + half))
+
+
+@dataclass
+class Outcome:
+    """One run scored against one rule."""
+
+    rule_id: str
+    agent: str
+    arm: str  # rule | baseline | enforced
+    verdict: str  # obeyed | violated | not_exercised
+    run_id: str
+    denied: int = 0
+    evidence: list[dict[str, Any]] = field(default_factory=list)
+    condition: str | None = None
+    status: str = "ok"
+
+
+@dataclass
+class RuleScore:
+    rule_id: str
+    agent: str
+    arm: str
+    runs: int
+    exercised: int
+    obeyed: int
+    violated: int
+    compliance: float | None
+    ci_low: float | None
+    ci_high: float | None
+    exercised_share: float
+    blocked_runs: int = 0
+    evasions: int = 0
+    evasion_rate: float | None = None
+    violation_example: dict[str, Any] | None = None
+    errors: int = 0
+
+    def as_dict(self) -> dict[str, Any]:
+        return {k: getattr(self, k) for k in self.__dataclass_fields__}
+
+
+def score_outcomes(outcomes: Iterable[Outcome]) -> list[RuleScore]:
+    groups: dict[tuple[str, str, str], list[Outcome]] = defaultdict(list)
+    for o in outcomes:
+        groups[(o.rule_id, o.agent, o.arm)].append(o)
+    scores: list[RuleScore] = []
+    for (rule_id, agent, arm), items in sorted(groups.items()):
+        ok = [o for o in items if o.status == "ok"]
+        exercised = [o for o in ok if o.verdict != "not_exercised"]
+        obeyed = sum(o.verdict == "obeyed" for o in exercised)
+        violated = sum(o.verdict == "violated" for o in exercised)
+        compliance = obeyed / len(exercised) if exercised else None
+        lo, hi = wilson(obeyed, len(exercised)) if exercised else (None, None)
+        blocked = [o for o in ok if o.denied > 0]
+        evasions = sum(o.verdict == "violated" for o in blocked)
+        example = next((o for o in exercised if o.verdict == "violated"), None)
+        scores.append(
+            RuleScore(
+                rule_id=rule_id,
+                agent=agent,
+                arm=arm,
+                runs=len(ok),
+                exercised=len(exercised),
+                obeyed=obeyed,
+                violated=violated,
+                compliance=compliance,
+                ci_low=lo,
+                ci_high=hi,
+                exercised_share=len(exercised) / len(ok) if ok else 0.0,
+                blocked_runs=len(blocked),
+                evasions=evasions,
+                evasion_rate=evasions / len(blocked) if blocked else None,
+                violation_example=(
+                    {"run_id": example.run_id, "evidence": example.evidence[:1]} if example is not None else None
+                ),
+                errors=len(items) - len(ok),
+            )
+        )
+    return scores
+
+
+def summary_sentence(scores: Iterable[RuleScore], agent: str, arm: str = "rule") -> str:
+    """For example: "Claude Code followed 9 of 14 checkable rules in at least 90% of exercised runs"."""
+    mine = [s for s in scores if s.agent == agent and s.arm == arm]
+    measured = [s for s in mine if s.compliance is not None]
+    followed = sum(1 for s in measured if s.compliance is not None and s.compliance >= FOLLOWED_THRESHOLD)
+    name = AGENT_NAMES.get(agent, agent)
+    text = (
+        f"{name} followed {followed} of {len(measured)} checkable rules in at least "
+        f"{round(FOLLOWED_THRESHOLD * 100)}% of exercised runs"
+    )
+    unexercised = len(mine) - len(measured)
+    if unexercised:
+        text += f" ({unexercised} more never exercised)"
+    return text
+
+
+def pooled(scores: Iterable[RuleScore], agent: str | None = None, arm: str = "rule") -> Mapping[str, Any]:
+    """Pooled compliance over all exercised runs of the selected rules, with a Wilson interval."""
+    chosen = [s for s in scores if s.arm == arm and (agent is None or s.agent == agent)]
+    k = sum(s.obeyed for s in chosen)
+    n = sum(s.exercised for s in chosen)
+    lo, hi = wilson(k, n)
+    blocked = sum(s.blocked_runs for s in chosen)
+    evasions = sum(s.evasions for s in chosen)
+    return {
+        "obeyed": k,
+        "exercised": n,
+        "runs": sum(s.runs for s in chosen),
+        "compliance": k / n if n else None,
+        "ci_low": lo if n else None,
+        "ci_high": hi if n else None,
+        "blocked_runs": blocked,
+        "evasions": evasions,
+        "evasion_rate": evasions / blocked if blocked else None,
+    }

obey_cli/study/analysis.py

@@ -0,0 +1,277 @@
+"""Pre-registered analysis of the phrasing experiment (study/PREREGISTRATION.md).
+
+Unit: one run. Outcome: 1 = obeyed, 0 = violated; runs where the rule was not exercised, and runs that did not
+complete, are excluded and counted. Strata: rule slot × repository × harness.
+
+- Effect of a contrast A vs B: the stratum-weighted difference in compliance, weighting each stratum that has
+  exercised runs in both conditions by its number of exercised runs in A and B.
+- p-value: stratified permutation test (condition labels shuffled within strata), two-sided, 10,000 permutations.
+- 95% interval: cluster bootstrap over rule slot × repository clusters (both harnesses resampled together),
+  10,000 resamples, percentile interval.
+- Phrasing family (three pairwise contrasts): Holm–Bonferroni at α = 0.05.
+"""
+
+from __future__ import annotations
+
+import math
+from collections import defaultdict
+from collections.abc import Iterable, Mapping, Sequence
+from dataclasses import dataclass
+from statistics import NormalDist
+from typing import Any
+
+SEED = 20261024
+PERMUTATIONS = 10_000
+BOOTSTRAP = 10_000
+CONTRASTS = (("prohibition", "imperative"), ("reason", "imperative"), ("reason", "prohibition"))
+
+
+@dataclass
+class Obs:
+    stratum: tuple[str, str, str]  # slot, repo, harness
+    cluster: tuple[str, str]  # slot, repo
+    condition: str
+    y: int
+
+
+def observations(rows: Iterable[Mapping[str, Any]]) -> tuple[list[Obs], dict[str, int]]:
+    obs: list[Obs] = []
+    excluded = {"not_exercised": 0, "incomplete": 0}
+    for r in rows:
+        if r.get("status") != "ok":
+            excluded["incomplete"] += 1
+            continue
+        if r["verdict"] == "not_exercised":
+            excluded["not_exercised"] += 1
+            continue
+        obs.append(
+            Obs(
+                (r["slot"], r["repo"], r["harness"]),
+                (r["slot"], r["repo"]),
+                r["condition"],
+                1 if r["verdict"] == "obeyed" else 0,
+            )
+        )
+    return obs, excluded
+
+
+def _group(
+    obs: Sequence[Obs], a: Sequence[str], b: Sequence[str]
+) -> dict[tuple[str, str, str], tuple[list[int], list[int]]]:
+    groups: dict[tuple[str, str, str], tuple[list[int], list[int]]] = defaultdict(lambda: ([], []))
+    for o in obs:
+        if o.condition in a:
+            groups[o.stratum][0].append(o.y)
+        elif o.condition in b:
+            groups[o.stratum][1].append(o.y)
+    return {k: v for k, v in groups.items() if v[0] and v[1]}
+
+
+def stratified_difference(groups: Mapping[Any, tuple[Sequence[int], Sequence[int]]]) -> float | None:
+    num = 0.0
+    den = 0.0
+    for ya, yb in groups.values():
+        w = len(ya) + len(yb)
+        num += w * (sum(ya) / len(ya) - sum(yb) / len(yb))
+        den += w
+    return num / den if den else None
+
+
+def permutation_p(
+    groups: Mapping[Any, tuple[Sequence[int], Sequence[int]]], n: int = PERMUTATIONS, seed: int = SEED
+) -> float | None:
+    import numpy as np
+
+    observed = stratified_difference(groups)
+    if observed is None:
+        return None
+    rng = np.random.default_rng(seed)
+    pooled = [(np.array(list(ya) + list(yb)), len(ya)) for ya, yb in groups.values()]
+    weights = np.array([len(p) for p, _ in pooled], dtype=float)
+    hits = 0
+    for _ in range(n):
+        diffs = []
+        for values, na in pooled:
+            perm = rng.permutation(values)
+            diffs.append(perm[:na].mean() - perm[na:].mean())
+        stat = float(np.dot(weights, diffs) / weights.sum())
+        if abs(stat) >= abs(observed) - 1e-12:
+            hits += 1
+    return (hits + 1) / (n + 1)
+
+
+def bootstrap_ci(
+    obs: Sequence[Obs], a: Sequence[str], b: Sequence[str], n: int = BOOTSTRAP, seed: int = SEED
+) -> tuple[float | None, float | None]:
+    import numpy as np
+
+    clusters: dict[tuple[str, str], list[Obs]] = defaultdict(list)
+    for o in obs:
+        clusters[o.cluster].append(o)
+    keys = sorted(clusters)
+    if not keys:
+        return (None, None)
+    rng = np.random.default_rng(seed)
+    stats = []
+    for _ in range(n):
+        sample: list[Obs] = []
+        for i, idx in enumerate(rng.integers(0, len(keys), len(keys))):
+            # Relabel resampled clusters so duplicates count as separate strata.
+            for o in clusters[keys[int(idx)]]:
+                sample.append(Obs((o.stratum[0], o.stratum[1] + f"#{i}", o.stratum[2]), o.cluster, o.condition, o.y))
+        diff = stratified_difference(_group(sample, a, b))
+        if diff is not None:
+            stats.append(diff)
+    if not stats:
+        return (None, None)
+    return (float(np.percentile(stats, 2.5)), float(np.percentile(stats, 97.5)))
+
+
+def compliance(obs: Sequence[Obs], conditions: Sequence[str]) -> dict[str, Any]:
+    from obey_cli.score import wilson
+
+    ys = [o.y for o in obs if o.condition in conditions]
+    lo, hi = wilson(sum(ys), len(ys))
+    return {
+        "obeyed": sum(ys),
+        "n": len(ys),
+        "compliance": sum(ys) / len(ys) if ys else None,
+        "ci_low": lo if ys else None,
+        "ci_high": hi if ys else None,
+    }
+
+
+def contrast(
+    obs: Sequence[Obs], a: Sequence[str], b: Sequence[str], permutations: int = PERMUTATIONS, bootstrap: int = BOOTSTRAP
+) -> dict[str, Any]:
+    groups = _group(obs, a, b)
+    lo, hi = bootstrap_ci(obs, a, b, bootstrap)
+    return {
+        "a": list(a),
+        "b": list(b),
+        "strata": len(groups),
+        "effect": stratified_difference(groups),
+        "ci_low": lo,
+        "ci_high": hi,
+        "p": permutation_p(groups, permutations),
+        "a_compliance": compliance(obs, a),
+        "b_compliance": compliance(obs, b),
+    }
+
+
+def holm(pvalues: Sequence[float | None], alpha: float = 0.05) -> list[dict[str, Any]]:
+    order = sorted((p, i) for i, p in enumerate(pvalues) if p is not None)
+    m = len(order)
+    out: list[dict[str, Any]] = [{"p_adj": None, "reject": False} for _ in pvalues]
+    running = 0.0
+    still = True
+    for rank, (p, i) in enumerate(order):
+        adj = min(1.0, (m - rank) * p)
+        running = max(running, adj)
+        still = still and p <= alpha / (m - rank)
+        out[i] = {"p_adj": running, "reject": still}
+    return out
+
+
+def analyse(
+    rows: Sequence[Mapping[str, Any]], permutations: int = PERMUTATIONS, bootstrap: int = BOOTSTRAP
+) -> dict[str, Any]:
+    obs, excluded = observations(rows)
+    rule_present = contrast(obs, ["imperative", "prohibition", "reason"], ["baseline"], permutations, bootstrap)
+    phrasing = [contrast(obs, [a], [b], permutations, bootstrap) for a, b in CONTRASTS]
+    for c, h in zip(phrasing, holm([c["p"] for c in phrasing]), strict=True):
+        c.update(h)
+    per_condition = {c: compliance(obs, [c]) for c in ("baseline", "imperative", "prohibition", "reason")}
+    exercised: dict[str, dict[str, int]] = defaultdict(lambda: {"exercised": 0, "runs": 0})
+    for r in rows:
+        if r.get("status") == "ok":
+            key = r["condition"]
+            exercised[key]["runs"] += 1
+            exercised[key]["exercised"] += r["verdict"] != "not_exercised"
+    return {
+        "n_runs": len(rows),
+        "n_exercised": len(obs),
+        "excluded": excluded,
+        "per_condition": per_condition,
+        "exercised_share": dict(exercised),
+        "rule_present_vs_baseline": rule_present,
+        "phrasing": phrasing,
+    }
+
+
+def enforcement_contrast(
+    before: Sequence[Mapping[str, Any]], after: Sequence[Mapping[str, Any]], permutations: int = PERMUTATIONS
+) -> dict[str, Any]:
+    """Compliance with hooks minus compliance before, weighted within slot × repository × harness × condition cells,
+    with the stratified permutation test of the main analysis."""
+    groups: dict[tuple[str, ...], tuple[list[int], list[int]]] = defaultdict(lambda: ([], []))
+    for side, rows in ((0, after), (1, before)):
+        for r in rows:
+            if r.get("status") == "ok" and r["verdict"] != "not_exercised":
+                cell = (r["slot"], r["repo"], r["harness"], r["condition"])
+                groups[cell][side].append(1 if r["verdict"] == "obeyed" else 0)
+    usable = {k: v for k, v in groups.items() if v[0] and v[1]}
+    return {
+        "effect": stratified_difference(usable),
+        "p": permutation_p(usable, permutations),
+        "cells": len(usable),
+    }
+
+
+def trials_needed(
+    pilot_rows: Sequence[Mapping[str, Any]],
+    *,
+    delta: float = 0.10,
+    alpha: float = 0.05,
+    family: int = 3,
+    power: float = 0.80,
+    cells: int = 40,
+) -> dict[str, Any]:
+    """The pre-registered sample-size formula, from the pilot's compliance, clustering and exercised share."""
+    obs, _ = observations(pilot_rows)
+    present = [o for o in obs if o.condition in ("imperative", "prohibition", "reason")]
+    p_bar = sum(o.y for o in present) / len(present) if present else 0.7
+    p1 = min(max(p_bar, 0.55), 0.95)
+    p2 = max(p1 - delta, 0.01)
+    z_a = NormalDist().inv_cdf(1 - alpha / family / 2)
+    z_b = NormalDist().inv_cdf(power)
+    n_arm = (z_a + z_b) ** 2 * (p1 * (1 - p1) + p2 * (1 - p2)) / delta**2
+    # Intra-cluster correlation of the outcome across slot × repository cells (ANOVA estimator), clamped.
+    by_cluster: dict[tuple[str, str], list[int]] = defaultdict(list)
+    for o in present:
+        by_cluster[o.cluster].append(o.y)
+    icc = _icc(list(by_cluster.values()))
+    ok_rows = [r for r in pilot_rows if r.get("status") == "ok" and r["condition"] != "baseline"]
+    exercised_share = (sum(r["verdict"] != "not_exercised" for r in ok_rows) / len(ok_rows)) if ok_rows else 1.0
+    e = max(exercised_share, 0.3)
+    m = max(n_arm / cells, 1.0)
+    deff = 1 + (m - 1) * icc
+    trials = math.ceil(n_arm * deff / (cells * e))
+    return {
+        "p_bar": p_bar,
+        "p1": p1,
+        "p2": p2,
+        "z_alpha": z_a,
+        "z_beta": z_b,
+        "n_arm": n_arm,
+        "icc": icc,
+        "design_effect": deff,
+        "exercised_share": exercised_share,
+        "trials_per_cell": trials,
+    }
+
+
+def _icc(groups: Sequence[Sequence[int]]) -> float:
+    groups = [g for g in groups if len(g) >= 1]
+    k = len(groups)
+    n = sum(len(g) for g in groups)
+    if k < 2 or n <= k:
+        return 0.0
+    grand = sum(sum(g) for g in groups) / n
+    ssb = sum(len(g) * (sum(g) / len(g) - grand) ** 2 for g in groups)
+    ssw = sum(sum((y - sum(g) / len(g)) ** 2 for y in g) for g in groups)
+    msb = ssb / (k - 1)
+    msw = ssw / (n - k)
+    n0 = (n - sum(len(g) ** 2 for g in groups) / n) / (k - 1)
+    icc = (msb - msw) / (msb + (n0 - 1) * msw) if msb + (n0 - 1) * msw > 0 else 0.0
+    return min(max(icc, 0.0), 0.3)

obey_cli/study/design.py

@@ -0,0 +1,98 @@
+"""The phrasing experiment's design: rendering AGENTS.md per condition, and the rule each run is scored against."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+PHRASINGS = ("imperative", "prohibition", "reason")
+
+
+@dataclass(frozen=True)
+class Cell:
+    repo: str
+    slot: str
+    condition: str  # baseline | imperative | prohibition | reason
+    harness: str
+
+
+def load(path: Path) -> dict[str, Any]:
+    data: dict[str, Any] = yaml.safe_load(path.read_text(encoding="utf-8"))
+    return data
+
+
+def _one_line(text: str) -> str:
+    return " ".join(text.split())
+
+
+def render(repo: dict[str, Any], slot: str | None, condition: str) -> tuple[str, int | None]:
+    """AGENTS.md for one run, and the 1-based line of the slot under test (None for the baseline).
+
+    The slot under test takes `condition`'s phrasing (or is left out for `baseline`); every other slot uses its
+    imperative phrasing; distractors never change.
+    """
+    lines = [*repo["header"].rstrip("\n").split("\n"), ""]
+    target_line = None
+    for key in repo["order"]:
+        if key in repo["distractors"]:
+            lines.append(f"- {_one_line(repo['distractors'][key])}")
+            continue
+        spec = repo["slots"][key]
+        if key == slot:
+            if condition == "baseline":
+                continue
+            lines.append(f"- {_one_line(spec[condition])}")
+            target_line = len(lines)
+        else:
+            lines.append(f"- {_one_line(spec['imperative'])}")
+    return "\n".join(lines) + "\n", target_line
+
+
+def rule(design: dict[str, Any], repo_name: str, slot: str, condition: str) -> dict[str, Any]:
+    repo = design["repos"][repo_name]
+    spec = repo["slots"][slot]
+    _, line = render(repo, slot, condition if condition != "baseline" else "imperative")
+    text = _one_line(spec["imperative" if condition == "baseline" else condition])
+    return {
+        "id": f"{repo_name}--{slot}",
+        "text": text,
+        "source": {"file": "AGENTS.md", "lines": [line or 1, line or 1]},
+        "scope": {"paths": ["**"], "agents": None},
+        "checker": spec["checker"],
+        "status": "active",
+        "reason": None,
+        "origin": "user",
+        "probes": [
+            {
+                "id": f"{repo_name}--{slot}",
+                "template": "study",
+                "instruction": _one_line(spec["probe"]),
+                "setup_patch": None,
+                "domain": slot,
+            }
+        ],
+    }
+
+
+def cells(
+    design: dict[str, Any],
+    *,
+    repos: list[str] | None = None,
+    slots: list[str] | None = None,
+    conditions: list[str] | None = None,
+    harnesses: list[str] | None = None,
+) -> list[Cell]:
+    out = []
+    for repo_name, repo in design["repos"].items():
+        if repos and repo_name not in repos:
+            continue
+        for slot in repo["slots"]:
+            if slots and slot not in slots:
+                continue
+            for condition in conditions or design["conditions"]:
+                for harness in harnesses or design["harnesses"]:
+                    out.append(Cell(repo_name, slot, condition, harness))
+    return out