agentpace 0.9.2__py3-none-any.whl

agentpace/cache.py

@@ -0,0 +1,115 @@
+"""Remember parsed runs, so a second ingest reads only what changed.
+
+A cold ingest of a few thousand transcripts takes minutes, because every line has to be decoded.
+The cache maps each transcript path to the size and modification time of the transcript and of
+its companion files, such as Claude Code's `.meta.json`, and to the `RawRun` parsed from them. A
+run is parsed again when any of those stamps changes. An unfinished run can only change by
+writing to one of those files, so it needs no special case.
+
+The cache also keeps runs whose transcript has since been deleted. Claude Code removes old
+transcripts, and a session's surviving runs take their offsets from every sibling's spans, so
+losing an early sibling would shift the rest and duplicate them in the store.
+
+Privacy: the file stays in the local agentpace folder, beside the secret, and is never shared. It
+holds only `RawRun` fields, with the model already normalized, plus the transcript path. Folder
+names and agent types sit there as they sit in the transcripts it summarizes.
+"""
+
+from __future__ import annotations
+
+import dataclasses
+import json
+import os
+from pathlib import Path
+from typing import Any
+
+from agentpace.adapters.base import RawRun
+
+FORMAT = 3  # 2: keys are os.path.normcase paths. 3: each entry holds a list of runs
+
+# [size, mtime_ns] for the transcript and each companion file, or None for a missing companion.
+Stamps = list[list[int] | None]
+
+
+def _key(path: str) -> str:
+    """One key per file. Windows paths are case-insensitive, so `c:\\` and `C:\\` must match, or a
+    deleted transcript would come back twice."""
+    return os.path.normcase(path)
+
+
+def _to_json(run: RawRun) -> dict[str, Any]:
+    return dataclasses.asdict(run)
+
+
+def _from_json(data: dict[str, Any]) -> RawRun:
+    spans = tuple((float(start), float(end)) for start, end in data["spans"])
+    return RawRun(**{**data, "spans": spans})
+
+
+def _entry_runs(entry: dict[str, Any]) -> list[RawRun]:
+    return [_from_json(run) for run in entry["runs"]]
+
+
+class RunCache:
+    def __init__(self, path: Path, adapter: str, adapter_version: int, idle_cap_s: int) -> None:
+        self.path = path
+        self._header = {
+            "format": FORMAT,
+            "adapter": adapter,
+            "adapter_version": adapter_version,
+            "idle_cap_s": idle_cap_s,
+        }
+        self.entries: dict[str, dict[str, Any]] = {}
+        self.rebuilt = False
+        self._load()
+
+    def _load(self) -> None:
+        if not self.path.exists():
+            return
+        try:
+            stored = json.loads(self.path.read_text(encoding="utf-8"))
+            if {key: stored.get(key) for key in self._header} != self._header:
+                raise ValueError("cache was written by another version")
+            entries = stored["entries"]
+            for entry in entries.values():
+                if not isinstance(entry["stamps"], list) or not isinstance(entry["runs"], list):
+                    raise ValueError("cache entry has no stamps or runs")
+                _entry_runs(entry)  # reject anything that no longer fits a RawRun
+            self.entries = entries
+        except (OSError, ValueError, KeyError, TypeError, AttributeError):
+            self.entries = {}
+            self.rebuilt = True
+
+    def lookup(self, path: str, stamps: Stamps) -> tuple[bool, list[RawRun]]:
+        """Return (hit, runs). A hit needs the transcript and every companion file unchanged."""
+        entry = self.entries.get(_key(path))
+        if entry is None or entry["stamps"] != stamps:
+            return False, []
+        return True, _entry_runs(entry)
+
+    def store(self, path: str, stamps: Stamps, runs: list[RawRun]) -> None:
+        self.entries[_key(path)] = {"stamps": stamps, "runs": [_to_json(run) for run in runs]}
+
+    def vanished(self, sessions: set[tuple[str, str]], seen: set[str]) -> list[RawRun]:
+        """Cached runs of these sessions whose transcript no longer exists."""
+        runs = []
+        seen = {_key(path) for path in seen}
+        for path, entry in self.entries.items():
+            if path in seen:
+                continue
+            found = [run for run in _entry_runs(entry) if (run.project, run.session) in sessions]
+            if found and not os.path.exists(path):
+                runs.extend(found)
+        return runs
+
+    def runs(self) -> list[RawRun]:
+        """Every cached run, including runs whose transcript is gone."""
+        return [run for entry in self.entries.values() for run in _entry_runs(entry)]
+
+    def save(self) -> None:
+        """Write atomically, so a reader never sees half a file."""
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        temporary = self.path.with_name(self.path.name + ".tmp")
+        document = {**self._header, "entries": self.entries}
+        temporary.write_text(json.dumps(document, separators=(",", ":")), encoding="utf-8")
+        os.replace(temporary, self.path)

agentpace/charts.py

@@ -0,0 +1,191 @@
+"""Small static SVG charts, written by hand so the CLI stays standard-library only.
+
+Colors come from a palette checked for color-vision safety, and the charts follow the reader's
+light or dark theme. Every mark carries a `<title>` for hover, and every number also appears in a
+table in `docs/backtest.md`.
+"""
+
+from __future__ import annotations
+
+import html
+import math
+from collections.abc import Sequence
+from dataclasses import dataclass
+
+from agentpace.render import duration
+
+
+def escape(text: str) -> str:
+    """Escape &, < and >, as xml.sax.saxutils.escape does, without importing the network stack
+    that module drags in."""
+    return html.escape(text, quote=False)
+
+
+_STYLE = """
+<style>
+  svg {
+    --panel: #fcfcfb; --text: #0b0b0b; --muted: #52514e; --grid: #e4e3df;
+    --primary: #2a78d6; --second: #eb6834;
+  }
+  @media (prefers-color-scheme: dark) {
+    svg {
+      --panel: #1a1a19; --text: #f0efec; --muted: #c3c2b7; --grid: #3a3a37;
+      --primary: #3987e5; --second: #d95926;
+    }
+  }
+  .panel { fill: var(--panel); }
+  text {
+    font: 13px system-ui, -apple-system, "Segoe UI", sans-serif;
+    fill: var(--muted);
+    paint-order: stroke;
+    stroke: var(--panel);
+    stroke-width: 4px;
+    stroke-linejoin: round;
+  }
+  .title { font-size: 15px; font-weight: 600; fill: var(--text); }
+  .label { fill: var(--text); }
+  .grid { stroke: var(--grid); stroke-width: 1; }
+  .goal { stroke: var(--text); stroke-width: 1.5; stroke-dasharray: 4 3; }
+  .range { stroke: var(--primary); stroke-width: 8; stroke-linecap: round; }
+  .dot { fill: var(--primary); }
+  .second { fill: var(--second); }
+  .tick { stroke: var(--panel); stroke-width: 2; }
+</style>
+"""
+
+_TICKS = (
+    (10, "10 s"),
+    (30, "30 s"),
+    (60, "1 min"),
+    (300, "5 min"),
+    (600, "10 min"),
+    (1800, "30 min"),
+    (3600, "1 h"),
+    (10800, "3 h"),
+    (36000, "10 h"),
+)
+_MARGIN = 16
+
+
+@dataclass(frozen=True)
+class RangeBar:
+    label: str
+    n: int
+    p50: float
+    p80: float
+    p90: float
+
+
+def _open(width: int, height: int, title: str) -> list[str]:
+    return [
+        f'<svg xmlns="http://www.w3.org/2000/svg" '
+        f'viewBox="{-_MARGIN} {-_MARGIN} {width + 2 * _MARGIN} {height + 2 * _MARGIN}" '
+        f'width="{width + 2 * _MARGIN}" height="{height + 2 * _MARGIN}" role="img" '
+        f'aria-label="{escape(title)}">',
+        f"<title>{escape(title)}</title>",
+        _STYLE,
+        f'<rect class="panel" x="{-_MARGIN}" y="{-_MARGIN}" width="{width + 2 * _MARGIN}" '
+        f'height="{height + 2 * _MARGIN}" rx="8"/>',
+    ]
+
+
+def range_chart(title: str, bars: Sequence[RangeBar]) -> str:
+    """Horizontal P50 to P90 bars on a log axis: durations span several orders of magnitude."""
+    left, right, top, row = 190, 150, 62, 36
+    width = 860
+    height = top + row * len(bars) + 34
+    low = min(bar.p50 for bar in bars) / 1.5
+    high = max(max(bar.p90 for bar in bars) * 1.1, 3700.0)
+
+    def x(value: float) -> float:
+        span = math.log(high) - math.log(low)
+        return left + (math.log(value) - math.log(low)) / span * (width - left - right)
+
+    parts = _open(width, height, title)
+    parts.append(f'<text class="title" x="0" y="20">{escape(title)}</text>')
+    parts.append(
+        '<text x="0" y="42">Bar runs from P50 to P90. The gap marks P80. '
+        "Log scale, seconds to hours.</text>"
+    )
+    for tick, label in _TICKS:
+        if low <= tick <= high:
+            position = x(tick)
+            parts.append(
+                f'<line class="grid" x1="{position:.1f}" y1="{top - 6}" '
+                f'x2="{position:.1f}" y2="{height - 26}"/>'
+            )
+            parts.append(
+                f'<text x="{position:.1f}" y="{height - 8}" text-anchor="middle">{label}</text>'
+            )
+    for index, bar in enumerate(bars):
+        y = top + row * index + row / 2
+        parts.append(
+            f'<text class="label" x="0" y="{y + 4:.1f}">{escape(bar.label)} '
+            f"<tspan>n={bar.n:,}</tspan></text>"
+        )
+        parts.append(
+            f"<g><title>{escape(bar.label)}: P50 {duration(bar.p50)}, P80 {duration(bar.p80)}, "
+            f"P90 {duration(bar.p90)} (n={bar.n:,})</title>"
+            f'<line class="range" x1="{x(bar.p50):.1f}" y1="{y:.1f}" '
+            f'x2="{x(bar.p90):.1f}" y2="{y:.1f}"/>'
+            f'<line class="tick" x1="{x(bar.p80):.1f}" y1="{y - 4:.1f}" '
+            f'x2="{x(bar.p80):.1f}" y2="{y + 4:.1f}"/></g>'
+        )
+        parts.append(
+            f'<text x="{x(bar.p90) + 12:.1f}" y="{y + 4:.1f}">'
+            f"{duration(bar.p50)} to {duration(bar.p90)}</text>"
+        )
+    parts.append("</svg>")
+    return "\n".join(parts) + "\n"
+
+
+def coverage_chart(title: str, points: Sequence[tuple[str, float, bool]], goal: float) -> str:
+    """How often the actual run landed at or under the P80 forecast, for each method."""
+    left, right, top, row = 150, 70, 78, 30
+    width = 760
+    height = top + row * len(points) + 34
+
+    def x(value: float) -> float:
+        return left + value / 100 * (width - left - right)
+
+    parts = _open(width, height, title)
+    parts.append(f'<text class="title" x="0" y="20">{escape(title)}</text>')
+    parts.append(
+        '<circle class="dot" cx="6" cy="38" r="5"/><text x="18" y="42">agentpace candidate</text>'
+        '<rect class="second" x="170" y="33" width="10" height="10"/>'
+        '<text x="188" y="42">baseline</text>'
+    )
+    for tick in (0, 20, 40, 60, 80, 100):
+        position = x(tick)
+        parts.append(
+            f'<line class="grid" x1="{position:.1f}" y1="{top - 6}" '
+            f'x2="{position:.1f}" y2="{height - 26}"/>'
+            f'<text x="{position:.1f}" y="{height - 8}" text-anchor="middle">{tick}%</text>'
+        )
+    parts.append(
+        f'<line class="goal" x1="{x(goal):.1f}" y1="{top - 10}" '
+        f'x2="{x(goal):.1f}" y2="{height - 26}"/>'
+        f'<text class="label" x="{x(goal):.1f}" y="{top - 14}" text-anchor="middle">'
+        f"target {goal:.0f}%</text>"
+    )
+    for index, (name, value, is_baseline) in enumerate(points):
+        y = top + row * index + row / 2
+        position = x(value)
+        parts.append(f'<text class="label" x="0" y="{y + 4:.1f}">{escape(name)}</text>')
+        tip = f"<title>{escape(name)}: {value:.1f}%</title>"
+        if is_baseline:
+            parts.append(
+                f'<g>{tip}<rect class="second" x="{position - 5:.1f}" y="{y - 5:.1f}" '
+                f'width="10" height="10"/></g>'
+            )
+        else:
+            parts.append(
+                f'<g>{tip}<circle class="dot" cx="{position:.1f}" cy="{y:.1f}" r="5"/></g>'
+            )
+        anchor, shift = ("end", -10) if value > 88 else ("start", 10)
+        parts.append(
+            f'<text x="{position + shift:.1f}" y="{y + 4:.1f}" text-anchor="{anchor}">'
+            f"{value:.1f}%</text>"
+        )
+    parts.append("</svg>")
+    return "\n".join(parts) + "\n"

agentpace/claims.py

@@ -0,0 +1,164 @@
+"""Keep published numbers tied to the data.
+
+A number in a document sits between `<!-- claim:NAME -->` and `<!-- /claim -->`. This module works
+out what each one should say. `agentpace claims --check` fails when any of them disagrees with the
+rows and the backtest, so a figure in the README cannot outlive the data behind it.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from collections.abc import Sequence
+from pathlib import Path
+from typing import Any
+
+from agentpace import estimate as est
+from agentpace import render
+from agentpace.report import ci
+from agentpace.schema import Row
+
+_MARKER = re.compile(r"<!-- claim:([a-z0-9_]+) -->(.*?)<!-- /claim -->", re.DOTALL)
+_MIN_TABLE_ROWS = 10
+
+
+def _kind_table(rows: Sequence[Row]) -> str:
+    completed = [row for row in rows if row["outcome"] == "completed"]
+    entries = []
+    for kind in sorted({row["task_kind"] for row in completed}):
+        values = sorted(row["duration_s"] for row in completed if row["task_kind"] == kind)
+        if len(values) >= _MIN_TABLE_ROWS:
+            entries.append((kind, len(values), *(est.quantile(values, q) for q in est.QUANTILES)))
+    entries.sort(key=lambda entry: entry[2])
+    lines = ["| Task kind | Runs | P50 | P80 | P90 |", "|---|---:|---:|---:|---:|"]
+    lines += [
+        f"| {kind} | {n:,} | {render.duration(p50)} | {render.duration(p80)} | "
+        f"{render.duration(p90)} |"
+        for kind, n, p50, p80, p90 in entries
+    ]
+    return "\n" + "\n".join(lines) + "\n"
+
+
+# The timeline the README's quote example re-answers, passed as `--quoted` exactly as given.
+QUOTE_EXAMPLE = "3 months"
+
+
+def _example_output(rows: Sequence[Row], plan_path: Path | None, quoted: str | None = None) -> str:
+    """The quickstart's real output, so the README cannot show a result the tool would not give."""
+    if plan_path is None or not plan_path.exists():
+        return ""
+    tasks, cap = est.tasks_from_json(json.loads(plan_path.read_text(encoding="utf-8")))
+    return "\n" + render.plan_text(est.estimate(est.fit(rows), tasks, cap), quoted) + "\n"
+
+
+def values(
+    rows: Sequence[Row],
+    report: dict[str, Any],
+    plan_path: Path | None = None,
+    quote_plan_path: Path | None = None,
+) -> dict[str, str]:
+    data = report["dataset"]
+    selection = report["selection"]
+    winner = selection["winner"]
+    results = report["splits"]["time"]["tracks"]["pre-run"]
+    ours = results[winner]["session"]
+    ours_row = results[winner]["row"]
+    median = results["global-median"]["session"]
+    plan = report["plans"]["results"]["agentpace"]
+    naive_plan = report["plans"]["results"]["sum-of-medians"]
+    project = report["splits"]["project"]
+    contributor = report["splits"]["contributor"]["status"]
+    project_p80 = (
+        f"{project['tracks']['pre-run'][winner]['session']['coverage80']:.1f}%"
+        if project["status"] == "ok"
+        else "not available"
+    )
+    # Defined by rank, never by fold number, because ranks shift as data is added.
+    largest = project.get("without_largest")
+    project_top_harness = largest["harness"] if largest else "not available"
+    project_top_sessions = (
+        f"{largest['sessions']} of {largest['of_sessions']}" if largest else "not available"
+    )
+    completed = sorted(row["duration_s"] for row in rows if row["outcome"] == "completed")
+    found = report["splits"]["time"]["intervals"]
+    plan_found = report["plans"]["intervals"]
+    # The README describes the pooled selection-period number, so locked weeks stay out of it.
+    weeks = [w for w in report["splits"]["time"]["by_week"] if w["period"] == "selection"]
+    latest = weeks[-1]
+
+    def spread(field: str) -> str:
+        values = [week[field] for week in weeks]
+        return f"{min(values):.0f}% to {max(values):.0f}%"
+
+    def gap(name: str) -> str:
+        low, high = found["rival_minus_ours"][name]
+        return f"{low:+.0f} to {high:+.0f}"
+
+    return {
+        "runs": f"{data['rows']:,}",
+        "completed": f"{data['completed']:,}",
+        "sessions": str(data["sessions"]),
+        "projects": str(data["projects"]),
+        "contributors": str(data["contributors"]),
+        "first_week": data["weeks"][0],
+        "last_week": data["weeks"][-1],
+        "top_project": f"{data['top_project_share']:.0f}%",
+        "top_sessions": f"{data['top_three_session_share']:.0f}%",
+        "overall_p50": render.duration(est.quantile(completed, 0.5)),
+        "overall_p90": render.duration(est.quantile(completed, 0.9)),
+        "winner": winner,
+        "plan_sessions": str(report["plans"]["sessions"]),
+        "plan_p50": ci(plan["coverage50"], plan_found["coverage50"], unit="%"),
+        "plan_p80": ci(plan["coverage80"], plan_found["coverage80"], unit="%"),
+        "plan_p90": ci(plan["coverage90"], plan_found["coverage90"], unit="%"),
+        "plan_pinball": ci(plan["pinball"], plan_found["pinball"]),
+        "naive_plan_pinball": f"{naive_plan['pinball']:.0f}",
+        "plan_mape": f"{plan['mape']:.0f}%",
+        "plan_mdape": f"{plan['mdape']:.0f}%",
+        "plan_width": f"{plan['width']:.1f}x",
+        "plan_width80": f"{plan['width80']:.1f}x",
+        "naive_plan_p80": f"{naive_plan['coverage80']:.0f}%",
+        "naive_plan_mape": f"{naive_plan['mape']:.0f}%",
+        "task_p50": f"{ours['coverage50']:.1f}%",
+        "task_p80": ci(ours["coverage80"], found["coverage80"], 1, "%"),
+        "task_p90": f"{ours['coverage90']:.1f}%",
+        "task_p80_row": f"{ours_row['coverage80']:.1f}%",
+        "task_mape": f"{ours['mape']:.0f}%",
+        "task_mdape": f"{ours['mdape']:.0f}%",
+        "task_width": f"{ours['width']:.1f}x",
+        "task_pinball": ci(ours["pinball"], found["pinball"]),
+        "median_pinball": f"{median['pinball']:.0f}",
+        "median_mape": f"{median['mape']:.0f}%",
+        "human_pinball": f"{results['human-guess']['session']['pinball']:.0f}",
+        "rounds_pinball": f"{results['fixed-rounds']['session']['pinball']:.0f}",
+        "median_pinball_gap": gap("global-median"),
+        "human_pinball_gap": gap("human-guess"),
+        "rounds_pinball_gap": gap("fixed-rounds"),
+        "weeks_tested": str(len(weeks)),
+        "weekly_p80": spread("coverage80_session"),
+        "weekly_p80_row": spread("coverage80_row"),
+        "latest_week": latest["week"],
+        "latest_week_p80": f"{latest['coverage80_session']:.0f}%",
+        "latest_week_p80_row": f"{latest['coverage80_row']:.0f}%",
+        "latest_week_sessions": str(latest["sessions"]),
+        "selection_last_week": report["selection"]["last_week"],
+        "locked_status": report["locked"]["status"],
+        "project_p80": project_p80,
+        "project_top_harness": project_top_harness,
+        "project_top_sessions": project_top_sessions,
+        "contributor_split": contributor,
+        "loss_count": str(len(report["losses"])),
+        "kind_table": _kind_table(rows),
+        "example_output": _example_output(rows, plan_path),
+        "quote_example_output": _example_output(rows, quote_plan_path, QUOTE_EXAMPLE),
+    }
+
+
+def apply(text: str, known: dict[str, str]) -> str:
+    def replace(match: re.Match[str]) -> str:
+        name = match.group(1)
+        if name not in known:
+            raise ValueError(f"unknown claim: {name}")
+        return f"<!-- claim:{name} -->{known[name]}<!-- /claim -->"
+
+    return _MARKER.sub(replace, text)

agentpace/classify.py

@@ -0,0 +1,54 @@
+"""Deterministic task classification. No LLM calls and no text matching.
+
+Metadata cannot separate feature, bugfix and refactor work, so the agent type is the only signal,
+and it is known before a run starts. Tool-mix heuristics are deliberately absent: they use
+post-run information and would dress up guesses as classification. User labels always win.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+
+_READ_ONLY_REVIEW = frozenset({"architect", "performance", "reviewer", "security", "verifier"})
+_RESEARCH = frozenset({"Explore", "researcher", "web-fetch"})
+_KINDS = {
+    "db-migration": "migration",
+    "debugger": "bugfix",
+    "docs": "docs",
+    "implementer": "feature",
+    "qa": "tests",
+}
+
+
+def strip_plugin(agent_type: str | None) -> str:
+    """`some-plugin:reviewer` becomes `reviewer`."""
+    return (agent_type or "").rsplit(":", 1)[-1]
+
+
+def classify_kind(agent_type: str | None, kind_map: Mapping[str, str] | None = None) -> str:
+    name = strip_plugin(agent_type)
+    if kind_map and name in kind_map:
+        return kind_map[name]
+    if name in _READ_ONLY_REVIEW:
+        return "review"
+    return _KINDS.get(name, "other")
+
+
+def classify_risk(agent_type: str | None, kind: str) -> str:
+    name = strip_plugin(agent_type)
+    if kind == "migration":
+        return "high"
+    if name in _READ_ONLY_REVIEW or name in _RESEARCH:
+        return "low"
+    return "normal"
+
+
+def size_bucket(edit_calls: int) -> str:
+    """Measured after the run, so it is kept out of the default forecast."""
+    if edit_calls == 0:
+        return "na"
+    if edit_calls <= 3:
+        return "S"
+    if edit_calls <= 10:
+        return "M"
+    return "L" if edit_calls <= 30 else "XL"