rulelab-cli 0.1.2__py3-none-any.whl

rulelab/rules.py

@@ -0,0 +1,225 @@
+"""Split an AGENTS.md / CLAUDE.md file into rules that can be removed one at a time."""
+
+from __future__ import annotations
+
+import math
+import re
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Literal
+
+Granularity = Literal["section", "bullet"]
+
+_HEADING = re.compile(r"^(#{1,6})\s+(.*?)\s*#*\s*$")
+_FENCE = re.compile(r"^\s{0,3}(```|~~~)")
+_BULLET = re.compile(r"^( {0,1})([-*+]|\d{1,3}[.)])\s+(.*)$")
+
+
+def approx_tokens(text: str) -> int:
+    """Rough token count (characters / 4). Labelled as approximate everywhere it is shown."""
+    stripped = text.strip()
+    return math.ceil(len(stripped) / 4) if stripped else 0
+
+
+@dataclass
+class Rule:
+    id: str
+    title: str
+    text: str
+    start_line: int  # 1-based, inclusive
+    end_line: int  # 1-based, inclusive
+
+    @property
+    def tokens(self) -> int:
+        return approx_tokens(self.text)
+
+    def to_dict(self) -> dict:
+        return {
+            "id": self.id,
+            "title": self.title,
+            "text": self.text,
+            "start_line": self.start_line,
+            "end_line": self.end_line,
+            "tokens": self.tokens,
+        }
+
+
+@dataclass
+class Segment:
+    lines: list[str]
+    rule_id: str | None  # None = always kept (titles, headings in bullet mode, blank lines)
+
+
+@dataclass
+class RuleFile:
+    path: str
+    text: str
+    granularity: Granularity
+    segments: list[Segment] = field(default_factory=list)
+    rules: list[Rule] = field(default_factory=list)
+
+    @property
+    def tokens(self) -> int:
+        return approx_tokens(self.text)
+
+    def render(self, without: set[str] | frozenset[str] = frozenset()) -> str:
+        """Return the file text with the given rule ids removed."""
+        unknown = set(without) - {r.id for r in self.rules}
+        if unknown:
+            raise ValueError(f"unknown rule id(s): {', '.join(sorted(unknown))}")
+        out: list[str] = []
+        for seg in self.segments:
+            if seg.rule_id is not None and seg.rule_id in without:
+                continue
+            out.extend(seg.lines)
+        text = "\n".join(out)
+        text = re.sub(r"\n{3,}", "\n\n", text).strip("\n")
+        return text + "\n" if text else ""
+
+    def rule(self, rule_id: str) -> Rule:
+        for r in self.rules:
+            if r.id == rule_id:
+                return r
+        raise KeyError(rule_id)
+
+
+def _code_mask(lines: list[str]) -> list[bool]:
+    """True for lines inside (or delimiting) fenced code blocks."""
+    mask: list[bool] = []
+    fence: str | None = None
+    for line in lines:
+        m = _FENCE.match(line)
+        if fence is None and m:
+            fence = m.group(1)
+            mask.append(True)
+        elif fence is not None:
+            mask.append(True)
+            if line.strip().startswith(fence):
+                fence = None
+        else:
+            mask.append(False)
+    return mask
+
+
+def _headings(lines: list[str], mask: list[bool]) -> list[tuple[int, int, str]]:
+    out = []
+    for i, line in enumerate(lines):
+        if mask[i]:
+            continue
+        m = _HEADING.match(line)
+        if m:
+            out.append((i, len(m.group(1)), m.group(2).strip()))
+    return out
+
+
+def _short(text: str, limit: int = 72) -> str:
+    text = re.sub(r"\s+", " ", text).strip()
+    text = re.sub(r"[*_`]", "", text)
+    return text if len(text) <= limit else text[: limit - 1].rstrip() + "…"
+
+
+def _split_sections(lines: list[str], mask: list[bool]) -> list[tuple[int, int, str | None]]:
+    """Return (start, end_exclusive, title) chunks; title None means fixed preamble."""
+    heads = _headings(lines, mask)
+    if not heads:
+        return [(0, len(lines), "Whole file")]
+    levels = [lvl for _, lvl, _ in heads]
+    split_level = min(levels)
+    # A single top-level title (e.g. "# AGENTS.md") is not a rule: split one level below it.
+    if levels.count(split_level) == 1 and heads[0][1] == split_level and len(set(levels)) > 1:
+        split_level = min(lvl for lvl in levels if lvl > split_level)
+    starts = [(i, t) for i, lvl, t in heads if lvl == split_level]
+    chunks: list[tuple[int, int, str | None]] = []
+    first = starts[0][0]
+    if first > 0:
+        pre = lines[:first]
+        body = [ln for j, ln in enumerate(pre) if ln.strip() and not _HEADING.match(ln)]
+        chunks.append((0, first, "Preamble" if body else None))
+    for k, (i, title) in enumerate(starts):
+        end = starts[k + 1][0] if k + 1 < len(starts) else len(lines)
+        chunks.append((i, end, title))
+    return chunks
+
+
+def _split_bullets(lines: list[str], mask: list[bool]) -> list[tuple[int, int, str | None]]:
+    """Each top-level bullet item (with its nested lines) and each prose paragraph is a rule.
+
+    Headings and blank lines are kept in every configuration.
+    """
+    chunks: list[tuple[int, int, str | None]] = []
+    i, n = 0, len(lines)
+    while i < n:
+        line = lines[i]
+        if not mask[i] and (not line.strip() or _HEADING.match(line)):
+            chunks.append((i, i + 1, None))
+            i += 1
+            continue
+        start = i
+        bullet = None if mask[i] else _BULLET.match(line)
+        if bullet:
+            title = _short(bullet.group(3))
+            i += 1
+            while i < n:
+                nxt = lines[i]
+                if mask[i]:
+                    i += 1
+                    continue
+                if _HEADING.match(nxt) or _BULLET.match(nxt):
+                    break
+                if not nxt.strip():
+                    # a blank line ends the item unless the next non-blank line is indented
+                    j = i
+                    while j < n and not lines[j].strip():
+                        j += 1
+                    if j < n and lines[j].startswith((" ", "\t")) and not _BULLET.match(lines[j]):
+                        i = j
+                        continue
+                    break
+                i += 1
+        else:
+            # prose paragraph or code block: runs until a blank line, heading or bullet
+            title = None
+            while i < n:
+                nxt = lines[i]
+                if not mask[i] and (not nxt.strip() or _HEADING.match(nxt) or _BULLET.match(nxt)):
+                    if i > start:
+                        break
+                if title is None and not mask[i] and nxt.strip():
+                    title = _short(nxt)
+                i += 1
+            title = title or "Code block"
+        chunks.append((start, i, title))
+    return chunks
+
+
+def parse_rules(text: str, granularity: Granularity = "section", path: str = "AGENTS.md") -> RuleFile:
+    """Split ``text`` into rules. Every line belongs to exactly one segment."""
+    if granularity not in ("section", "bullet"):
+        raise ValueError("granularity must be 'section' or 'bullet'")
+    lines = text.replace("\r\n", "\n").split("\n")
+    if lines and lines[-1] == "":
+        lines = lines[:-1]
+    mask = _code_mask(lines)
+    chunks = _split_sections(lines, mask) if granularity == "section" else _split_bullets(lines, mask)
+    rf = RuleFile(path=path, text=text, granularity=granularity)
+    k = 0
+    for start, end, title in chunks:
+        seg_lines = lines[start:end]
+        if title is None or not any(ln.strip() for ln in seg_lines):
+            rf.segments.append(Segment(seg_lines, None))
+            continue
+        k += 1
+        rid = f"R{k}"
+        body = "\n".join(seg_lines).strip("\n")
+        # trailing blank lines are layout, not content
+        last = end
+        while last > start and not lines[last - 1].strip():
+            last -= 1
+        rf.rules.append(Rule(rid, title, body, start + 1, last))
+        rf.segments.append(Segment(seg_lines, rid))
+    return rf
+
+
+def load_rules(path: str | Path, granularity: Granularity = "section") -> RuleFile:
+    p = Path(path)
+    return parse_rules(p.read_text(encoding="utf-8"), granularity, path=p.name)

rulelab/runner.py

@@ -0,0 +1,94 @@
+"""The interface rulelab needs from an agent runner, plus a deterministic fake for tests."""
+
+from __future__ import annotations
+
+import hashlib
+from dataclasses import dataclass
+from typing import Protocol
+
+
+@dataclass
+class RunOutcome:
+    success: bool | None  # None = infrastructure error, excluded from statistics
+    input_tokens: int = 0
+    output_tokens: int = 0
+    cost_usd: float = 0.0
+    turns: int = 0
+    duration_s: float = 0.0
+    error: str | None = None
+
+
+class Runner(Protocol):
+    """Runs one agent attempt on one task with a given context file (or none)."""
+
+    model: str
+    estimate_note: str
+
+    def meta(self) -> dict:
+        """Metadata stored with the results (engine, repo, hardware...)."""
+        ...
+
+    def task_ids(self) -> list[str]: ...
+
+    def estimate_run_cost(self) -> float:
+        """Expected USD for one agent run (0 for local models)."""
+        ...
+
+    def run(self, task_id: str, context: str | None, repeat: int) -> RunOutcome: ...
+
+
+class FakeRunner:
+    """Deterministic runner for tests and offline development.
+
+    ``effects`` maps a marker substring to a success-probability shift: if the marker is present
+    in the context, the task succeeds more (positive) or less (negative) often.
+    """
+
+    def __init__(
+        self,
+        tasks: list[str] | None = None,
+        base_rate: float = 0.5,
+        effects: dict[str, float] | None = None,
+        cost_per_run: float = 0.01,
+        model: str = "fake/model",
+        fail_on: set[tuple[str, int]] | None = None,
+    ) -> None:
+        self.tasks = tasks or [f"task-{i}" for i in range(1, 5)]
+        self.base_rate = base_rate
+        self.effects = effects or {}
+        self.cost_per_run = cost_per_run
+        self.model = model
+        self.fail_on = fail_on or set()
+        self.calls: list[tuple[str, str | None, int]] = []
+        self.estimate_note = "fake runner"
+
+    def meta(self) -> dict:
+        return {"engine": "fake"}
+
+    def task_ids(self) -> list[str]:
+        return list(self.tasks)
+
+    def estimate_run_cost(self) -> float:
+        return self.cost_per_run
+
+    def run(self, task_id: str, context: str | None, repeat: int) -> RunOutcome:
+        self.calls.append((task_id, context, repeat))
+        if (task_id, repeat) in self.fail_on:
+            return RunOutcome(None, error="simulated infrastructure error")
+        p = self.base_rate
+        ctx = context or ""
+        for marker, shift in self.effects.items():
+            if marker in ctx:
+                p += shift
+        digest = hashlib.sha256(f"{task_id}|{repeat}|{ctx}".encode()).digest()
+        u = int.from_bytes(digest[:8], "big") / 2**64
+        ctx_tokens = len(ctx) // 4
+        turns = 5
+        return RunOutcome(
+            success=u < p,
+            input_tokens=turns * (2000 + ctx_tokens),
+            output_tokens=400,
+            cost_usd=self.cost_per_run,
+            turns=turns,
+            duration_s=0.0,
+        )

rulelab/stats.py

@@ -0,0 +1,180 @@
+"""Effect estimates with bootstrap confidence intervals and per-rule verdicts.
+
+Effect of a rule = success(full file) - success(full file without that rule), averaged over tasks.
+Positive means the rule helps. Uncertainty comes from a two-level bootstrap:
+
+1. resample tasks with replacement (task difficulty varies a lot), then
+2. for each sampled task, draw each configuration's success rate from its
+   Beta(successes + 1/2, failures + 1/2) posterior (Jeffreys prior) instead of
+   trusting 3/3 or 0/3 as exact.
+
+Step 2 matters with small samples: plain resampling of 3 identical outcomes gives a
+zero-width interval and would turn "we saw nothing" into "no effect".
+"""
+
+from __future__ import annotations
+
+import random
+from dataclasses import asdict, dataclass
+from statistics import fmean
+from typing import Iterable, Literal
+
+Verdict = Literal["helps", "no_effect", "hurts", "inconclusive"]
+
+DEFAULT_MARGIN = 0.10  # +/- 10 percentage points counts as "no effect"
+DEFAULT_ITERATIONS = 4000
+CONFIDENCE = 0.95
+
+
+@dataclass
+class Cell:
+    successes: int = 0
+    failures: int = 0
+
+    @property
+    def n(self) -> int:
+        return self.successes + self.failures
+
+    @property
+    def rate(self) -> float:
+        return self.successes / self.n if self.n else float("nan")
+
+
+@dataclass
+class Effect:
+    estimate: float  # observed mean difference over shared tasks
+    low: float
+    high: float
+    tasks: int
+    runs_a: int
+    runs_b: int
+    verdict: Verdict
+
+    def to_dict(self) -> dict:
+        return asdict(self)
+
+
+def cells(runs: Iterable[dict], config_id: str) -> dict[str, Cell]:
+    """Success/failure counts per task for one configuration. Errored runs are skipped."""
+    out: dict[str, Cell] = {}
+    for r in runs:
+        if r["config"] != config_id or r.get("success") is None:
+            continue
+        c = out.setdefault(r["task"], Cell())
+        if r["success"]:
+            c.successes += 1
+        else:
+            c.failures += 1
+    return out
+
+
+def classify(low: float, high: float, margin: float = DEFAULT_MARGIN) -> Verdict:
+    if low > 0:
+        return "helps"
+    if high < 0:
+        return "hurts"
+    if low >= -margin and high <= margin:
+        return "no_effect"
+    return "inconclusive"
+
+
+def _percentile(sorted_vals: list[float], q: float) -> float:
+    if not sorted_vals:
+        return float("nan")
+    pos = q * (len(sorted_vals) - 1)
+    lo = int(pos)
+    hi = min(lo + 1, len(sorted_vals) - 1)
+    frac = pos - lo
+    return sorted_vals[lo] * (1 - frac) + sorted_vals[hi] * frac
+
+
+def success_effect(
+    runs: list[dict],
+    config_a: str,
+    config_b: str,
+    *,
+    iterations: int = DEFAULT_ITERATIONS,
+    margin: float = DEFAULT_MARGIN,
+    seed: int = 0,
+) -> Effect:
+    """Bootstrap CI for mean_t[p_a(t) - p_b(t)] over tasks run under both configurations."""
+    a, b = cells(runs, config_a), cells(runs, config_b)
+    tasks = sorted(set(a) & set(b))
+    runs_a = sum(a[t].n for t in tasks)
+    runs_b = sum(b[t].n for t in tasks)
+    if not tasks:
+        return Effect(float("nan"), float("nan"), float("nan"), 0, runs_a, runs_b, "inconclusive")
+    observed = fmean(a[t].rate - b[t].rate for t in tasks)
+    rng = random.Random(seed)
+    draws: list[float] = []
+    for _ in range(iterations):
+        diffs = []
+        for _ in tasks:
+            t = tasks[rng.randrange(len(tasks))]
+            pa = rng.betavariate(a[t].successes + 0.5, a[t].failures + 0.5)
+            pb = rng.betavariate(b[t].successes + 0.5, b[t].failures + 0.5)
+            diffs.append(pa - pb)
+        draws.append(fmean(diffs))
+    draws.sort()
+    alpha = (1 - CONFIDENCE) / 2
+    low, high = _percentile(draws, alpha), _percentile(draws, 1 - alpha)
+    return Effect(observed, low, high, len(tasks), runs_a, runs_b, classify(low, high, margin))
+
+
+@dataclass
+class MeanCI:
+    mean: float
+    low: float
+    high: float
+    n: int
+
+    def to_dict(self) -> dict:
+        return asdict(self)
+
+
+def metric_by_task(runs: list[dict], config_id: str, key: str) -> dict[str, list[float]]:
+    out: dict[str, list[float]] = {}
+    for r in runs:
+        if r["config"] != config_id or r.get("success") is None:
+            continue
+        out.setdefault(r["task"], []).append(float(r.get(key) or 0))
+    return out
+
+
+def metric_delta(
+    runs: list[dict],
+    config_a: str,
+    config_b: str,
+    key: str,
+    *,
+    iterations: int = DEFAULT_ITERATIONS,
+    seed: int = 0,
+) -> MeanCI:
+    """Bootstrap CI for the per-run difference of a numeric field (tokens, cost), paired by task."""
+    a, b = metric_by_task(runs, config_a, key), metric_by_task(runs, config_b, key)
+    tasks = sorted(set(a) & set(b))
+    if not tasks:
+        return MeanCI(float("nan"), float("nan"), float("nan"), 0)
+    observed = fmean(fmean(a[t]) - fmean(b[t]) for t in tasks)
+    rng = random.Random(seed)
+    draws = []
+    for _ in range(iterations):
+        diffs = []
+        for _ in tasks:
+            t = tasks[rng.randrange(len(tasks))]
+            xa = [a[t][rng.randrange(len(a[t]))] for _ in a[t]]
+            xb = [b[t][rng.randrange(len(b[t]))] for _ in b[t]]
+            diffs.append(fmean(xa) - fmean(xb))
+        draws.append(fmean(diffs))
+    draws.sort()
+    alpha = (1 - CONFIDENCE) / 2
+    n = sum(len(a[t]) + len(b[t]) for t in tasks)
+    return MeanCI(observed, _percentile(draws, alpha), _percentile(draws, 1 - alpha), n)
+
+
+def success_rate(runs: list[dict], config_id: str) -> tuple[float, int, int]:
+    """(rate, successes, valid runs) for one configuration, pooled over tasks."""
+    cs = cells(runs, config_id).values()
+    s = sum(c.successes for c in cs)
+    n = sum(c.n for c in cs)
+    return (s / n if n else float("nan"), s, n)