agentpace 0.9.2__py3-none-any.whl

agentpace/share.py

@@ -0,0 +1,130 @@
+"""Prepare rows for contribution. Local only: this module writes a file and nothing else.
+
+There is deliberately no network code here or anywhere else in the package. Sending the file is a
+git push the contributor runs themselves, after reading exactly what it holds.
+"""
+
+from __future__ import annotations
+
+import os
+from collections.abc import Iterable, Sequence
+from pathlib import Path
+
+from agentpace import schema
+from agentpace.adapters import EXPERIMENTAL
+from agentpace.schema import Row
+
+NOTICE = """\
+What these rows reveal, even without any content:
+  - Rows from you link to each other through `contributor`.
+  - Rows from one session link through `batch`, and `project` groups a local project.
+    Use --drop-project to omit it.
+  - `week` and the mix of models show roughly when you worked and on what plan tier.
+  - A pull request ties this pseudonym to your GitHub identity in public history.
+  - Submitting a data pull request dedicates these rows to the public domain under CC0-1.0.
+"""
+
+
+def select(
+    rows: Iterable[Row],
+    since: str | None = None,
+    drop_project: bool = False,
+    already_published: Iterable[Row] = (),
+    drop_sessions: Iterable[str] = (),
+) -> tuple[list[Row], int, dict[str, int]]:
+    """Choose what to share. Returns the rows, how many were already published, and the rows
+    left out per dropped session.
+
+    A dropped session is withheld whole, before anything else is decided, so the file name and
+    the published count see only the rows actually shared. Every dropped session must match a
+    local row: a mistyped one would otherwise share the session it meant to withhold.
+    """
+    known = {schema.dedupe_key(row) for row in already_published}
+    dropped = dict.fromkeys(drop_sessions, 0)
+    chosen: list[Row] = []
+    skipped = 0
+    for row in rows:
+        if row["batch"] in dropped:
+            dropped[row["batch"]] += 1
+            continue
+        if since and row["week"] < since:
+            continue
+        if schema.dedupe_key(row) in known:
+            skipped += 1
+            continue
+        kept = {key: value for key, value in row.items() if key != "project" or not drop_project}
+        problems = schema.check_row(kept)
+        if problems:
+            raise ValueError(f"refusing to share an invalid row: {problems}")
+        chosen.append(kept)
+    unmatched = sorted(batch for batch, count in dropped.items() if not count)
+    if unmatched:
+        raise ValueError(
+            f"--drop-session names {len(unmatched)} sessions with no local run "
+            f"({', '.join(unmatched)}); nothing was written. See `agentpace runs`"
+        )
+    return sorted(chosen, key=schema.dedupe_key), skipped, dropped
+
+
+def file_name(rows: Sequence[Row]) -> str:
+    weeks = sorted(row["week"] for row in rows)
+    return f"{rows[0]['contributor']}-{weeks[0]}-{weeks[-1]}.jsonl"
+
+
+def held_rows(path: Path, rows: Sequence[Row]) -> int | None:
+    """How many rows `path` already holds, or None when it does not exist yet.
+
+    Refuses a file that holds anyone else's rows: a contribution file belongs to one contributor.
+    """
+    if not path.exists():
+        return None
+    contributor = rows[0]["contributor"]
+    old = schema.read_rows(path)
+    if any(row.get("contributor") != contributor for row in old):
+        raise ValueError(
+            f"refusing to add rows to {path.name}: it holds another contributor's rows"
+        )
+    return len(old)
+
+
+def write_contribution(path: Path, rows: Sequence[Row]) -> tuple[int, int]:
+    """Create `path`, or extend it. Returns (rows added, rows it already held).
+
+    An existing file keeps its bytes exactly; the new rows go after them. `select` has already
+    left out every published run, so an appended row is never a duplicate.
+    """
+    if not rows:
+        raise ValueError("nothing to write")
+    existing = held_rows(path, rows) or 0
+    old = path.read_bytes() if path.exists() else b""
+    if old and not old.endswith(b"\n"):
+        old += b"\n"
+    new = "".join(schema.canonical(row) + "\n" for row in rows).encode("utf-8")
+    path.parent.mkdir(parents=True, exist_ok=True)
+    temporary = path.with_name(path.name + ".tmp")
+    temporary.write_bytes(old + new)
+    os.replace(temporary, path)
+    return len(rows), existing
+
+
+def experimental_note(rows: Iterable[Row]) -> str | None:
+    """A warning when rows from unverified adapters are in the share. They stay in it."""
+    harnesses = [row["harness"] for row in rows if row["harness"] in EXPERIMENTAL]
+    if not harnesses:
+        return None
+    names = ", ".join(sorted(set(harnesses)))
+    return (
+        f"{len(harnesses)} rows come from experimental adapters ({names}). validate rejects them"
+        " until the adapter is verified, so a pull request with them stays red until a maintainer"
+        " reconciles them by hand; see docs/verify-adapter.md."
+    )
+
+
+def git_commands(path: Path) -> list[str]:
+    return [
+        f'agentpace validate "{path.parent}"',
+        f'git add "{path}"',
+        'git commit -m "data: add runs"',
+        "git push origin HEAD",
+        "Then open a pull request.",
+    ]

agentpace/site.py

@@ -0,0 +1,133 @@
+"""Build the data block inside `docs/index.html`.
+
+The page is one static file with no build step. Its data sits inline, so it works from `file://`
+and makes no requests of any kind. Fetching a JSON file would add a network call.
+"""
+
+from __future__ import annotations
+
+import json
+import math
+import re
+from collections import defaultdict
+from collections.abc import Sequence
+from html import escape
+from typing import Any
+
+from agentpace import estimate as est
+from agentpace import render
+from agentpace.schema import Row
+
+START = '<script id="leaderboard-data" type="application/json">'
+END = "</script>"
+BINS_PER_DECADE = 10
+
+
+def _summary(values: Sequence[int]) -> dict[str, int]:
+    ordered = sorted(values)
+    return {
+        "n": len(ordered),
+        "p50": round(est.quantile(ordered, 0.5)),
+        "p75": round(est.quantile(ordered, 0.75)),
+        "p90": round(est.quantile(ordered, 0.9)),
+    }
+
+
+def _histogram(values: Sequence[int]) -> list[dict[str, int]]:
+    """Counts on a log axis. Aggregates only: individual runs are not embedded in the page."""
+    counts: dict[int, int] = defaultdict(int)
+    for value in values:
+        counts[math.floor(math.log10(value) * BINS_PER_DECADE)] += 1
+    return [
+        {
+            "low": round(10 ** (index / BINS_PER_DECADE)),
+            "high": round(10 ** ((index + 1) / BINS_PER_DECADE)),
+            "n": counts[index],
+        }
+        for index in range(max(counts) + 1)
+    ]
+
+
+def payload(rows: Sequence[Row], backtest_report: dict[str, Any]) -> dict[str, Any]:
+    completed = [row for row in rows if row["outcome"] == "completed"]
+    cells: dict[tuple[str, str, str], list[int]] = defaultdict(list)
+    for row in completed:
+        cells[(row["model"], row["task_kind"], row["risk"])].append(row["duration_s"])
+    durations = [row["duration_s"] for row in completed]
+    weeks = sorted({row["week"] for row in rows})
+    time_split = backtest_report["splits"]["time"]["tracks"]["pre-run"]
+    winner = backtest_report["selection"]["winner"]
+    plans = backtest_report["plans"]
+    return {
+        "runs": len(rows),
+        "completed": len(completed),
+        "sessions": len({row["batch"] for row in rows}),
+        "contributors": len({row["contributor"] for row in rows}),
+        "first_week": weeks[0],
+        "last_week": weeks[-1],
+        "overall": _summary(durations),
+        "histogram": _histogram(durations),
+        "cells": [
+            {"model": key[0], "kind": key[1], "risk": key[2], **_summary(values)}
+            for key, values in sorted(cells.items())
+        ],
+        "backtest": {
+            "sessions": plans["sessions"],
+            "plan_p80_coverage": round(plans["results"]["agentpace"]["coverage80"]),
+            "plan_p90_coverage": round(plans["results"]["agentpace"]["coverage90"]),
+            "task_p80_coverage": round(time_split[winner]["session"]["coverage80"]),
+            "task_mdape": round(time_split[winner]["session"]["mdape"]),
+        },
+    }
+
+
+def texts(data: dict[str, Any]) -> dict[str, str]:
+    """Sentences the page shows before any script runs, so link previews, readers without
+    JavaScript and crawlers all see the same numbers as everyone else."""
+    overall, scores = data["overall"], data["backtest"]
+    people = "contributor" if data["contributors"] == 1 else "contributors"
+    return {
+        "lede": (
+            f"Measured from {data['completed']:,} completed agent runs, not from anyone's "
+            f"opinion. Half finished within {render.duration(overall['p50'])}. One in ten took "
+            f"more than {render.duration(overall['p90'])}."
+        ),
+        "trace-note": (
+            f"Every completed run from {data['first_week']} to {data['last_week']}. "
+            "Time runs left to right on a log scale."
+        ),
+        "reading-scope": (
+            f"{data['runs']:,} runs in {data['sessions']} sessions from {data['contributors']} "
+            f"{people}, with heavy automated fan-out. It will not match your setup until more "
+            "people contribute."
+        ),
+        "reading-backtest": (
+            f"On {scores['sessions']} held-out sessions, the P80 for the whole session covered "
+            f"{scores['plan_p80_coverage']}% and the P90 covered {scores['plan_p90_coverage']}%. "
+            f"For a single task the median miss is {scores['task_mdape']}%, so read a range, "
+            "never one number."
+        ),
+    }
+
+
+def _fill(html: str, element_id: str, text: str) -> str:
+    pattern = re.compile(rf'(id="{element_id}"[^>]*>)(.*?)(</(?:p|figcaption)>)', re.DOTALL)
+    filled, count = pattern.subn(lambda m: m[1] + escape(text, quote=False) + m[3], html, count=1)
+    if count != 1:
+        raise ValueError(f"the page has no element with id {element_id}")
+    return filled
+
+
+def embed(html: str, data: dict[str, Any]) -> str:
+    start = html.index(START) + len(START)
+    end = html.index(END, start)
+    block = json.dumps(data, separators=(",", ":")).replace("</", "<\\/")
+    return f"{html[:start]}{block}{html[end:]}"
+
+
+def build(html: str, data: dict[str, Any]) -> str:
+    """The data block plus the sentences drawn from it."""
+    page = embed(html, data)
+    for element_id, text in texts(data).items():
+        page = _fill(page, element_id, text)
+    return page

agentpace/turns.py

@@ -0,0 +1,282 @@
+"""Waiting on the human, measured from main-session timestamps. Local only.
+
+A main Claude Code session alternates between agent turns and human prompts. The time from the
+agent's last record to the next prompt a person typed is a wait on the human: review, testing,
+decisions and everything else between turns, breaks included. It is elapsed time, not effort.
+
+Main sessions are never a source of run rows. This module reads them for this one quantity, and
+copies four things out of each parsed line: the record type, its timestamp, its `turnOrigin`
+(only whether it says a human typed the prompt) and whether it carries a tool result. Nothing it
+writes reaches `share`, the dataset or a run row.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import statistics
+from collections.abc import Iterator, Sequence
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from agentpace import estimate as est
+from agentpace import ingest, paths, pseudonym, schema
+from agentpace.adapters.claude_code import _extended, _parse_time
+
+Turn = dict[str, Any]
+
+HOUR_S = 3600
+_ASSISTANT, _HUMAN, _OTHER = "assistant", "human", "other"
+
+
+@dataclass(frozen=True)
+class _Mark:
+    time: float
+    kind: str
+
+
+@dataclass(frozen=True)
+class Gap:
+    time: float
+    gap_s: int
+    agent_turn_s: int
+
+
+@dataclass
+class TurnsReport:
+    files: int = 0
+    sessions: int = 0
+    gaps: int = 0
+    no_origin: int = 0
+    too_short: int = 0
+    unreadable: int = 0
+    kept_from_earlier: int = 0
+
+    def lines(self) -> list[str]:
+        lines = [
+            f"read {self.files} main-session transcripts: {self.gaps} gaps in "
+            f"{self.sessions} sessions"
+        ]
+        if self.no_origin:
+            lines.append(
+                f"skipped {self.no_origin} sessions with no turnOrigin, so human prompts cannot "
+                "be told apart from other records"
+            )
+        if self.too_short:
+            lines.append(f"skipped {self.too_short} sessions with fewer than 2 human prompts")
+        if self.unreadable:
+            lines.append(f"could not read {self.unreadable} files")
+        if self.kept_from_earlier:
+            lines.append(
+                f"kept {self.kept_from_earlier} gaps from sessions whose transcripts are gone"
+            )
+        return lines
+
+
+def _schema() -> dict[str, Any]:
+    return schema.load_schema_file(paths.turn_schema_file())
+
+
+def check_turn(turn: Any) -> list[str]:
+    return schema.check_against(_schema(), turn)
+
+
+def _project(record: Any) -> tuple[str, float, bool | None] | None:
+    """Copy the allowed values out of a parsed line: the only code that touches one.
+
+    Returns the mark kind, its time, and whether the line carried a `turnOrigin` key at all
+    (None for assistant records, where it does not apply)."""
+    if not isinstance(record, dict):
+        return None
+    role = record.get("type")
+    time = _parse_time(record.get("timestamp"))
+    if time is None:
+        return None
+    if role == "assistant":
+        return (_ASSISTANT, time, None)
+    if role != "user":
+        return None
+    message = record.get("message")
+    content = message.get("content") if isinstance(message, dict) else None
+    if isinstance(content, list) and any(
+        isinstance(block, dict) and block.get("type") == "tool_result" for block in content
+    ):
+        return (_OTHER, time, None)
+    origin = record.get("turnOrigin")
+    return (_HUMAN if origin == "human" else _OTHER, time, "turnOrigin" in record)
+
+
+def gaps(marks: Sequence[_Mark]) -> list[Gap]:
+    """Each human prompt after an agent record opens a gap, paired with the agent turn before
+    it."""
+    found: list[Gap] = []
+    last_agent: float | None = None
+    previous_prompt: float | None = None
+    for mark in sorted(marks, key=lambda m: m.time):
+        if mark.kind == _ASSISTANT:
+            last_agent = mark.time
+        elif mark.kind == _HUMAN:
+            if last_agent is not None:
+                turn = 0.0
+                if previous_prompt is not None and last_agent >= previous_prompt:
+                    turn = last_agent - previous_prompt
+                found.append(Gap(mark.time, round(mark.time - last_agent), round(turn)))
+            previous_prompt = mark.time
+    return found
+
+
+@dataclass
+class _Session:
+    marks: list[_Mark] = field(default_factory=list)
+    has_origin: bool = False
+
+
+def _read(path: str) -> _Session:
+    session = _Session()
+    with open(path, encoding="utf-8") as handle:
+        for line in handle:
+            try:
+                record = json.loads(line)
+            except ValueError:
+                continue
+            projected = _project(record)
+            if projected is None:
+                continue
+            kind, time, carries_origin = projected
+            session.marks.append(_Mark(time, kind))
+            session.has_origin = session.has_origin or bool(carries_origin)
+    return session
+
+
+def discover(root: Path) -> Iterator[tuple[str, str]]:
+    """Main-session transcripts sit directly in each project folder as `<session>.jsonl`."""
+    base = _extended(root)
+    try:
+        projects = sorted(os.scandir(base), key=lambda entry: entry.name)
+    except OSError:
+        return
+    for project in projects:
+        if not project.is_dir():
+            continue
+        try:
+            files = sorted(os.scandir(project.path), key=lambda entry: entry.name)
+        except OSError:
+            continue
+        for entry in files:
+            if entry.is_file() and entry.name.endswith(".jsonl"):
+                yield entry.path, entry.name[: -len(".jsonl")]
+
+
+def collect(root: Path, secret: bytes) -> tuple[list[Turn], set[str], TurnsReport]:
+    report = TurnsReport()
+    turns: list[Turn] = []
+    seen: set[str] = set()
+    for path, session_id in discover(root):
+        report.files += 1
+        try:
+            session = _read(path)
+        except (OSError, UnicodeDecodeError):
+            report.unreadable += 1
+            continue
+        if not session.has_origin:
+            report.no_origin += 1
+            continue
+        if sum(mark.kind == _HUMAN for mark in session.marks) < 2:
+            report.too_short += 1
+            continue
+        pseudonym_id = pseudonym.turn_session(secret, session_id)
+        seen.add(pseudonym_id)
+        found = gaps(session.marks)
+        if found:
+            report.sessions += 1
+        for gap in found:
+            turns.append(
+                {
+                    "session": pseudonym_id,
+                    "week": ingest.iso_week(gap.time),
+                    "gap_s": gap.gap_s,
+                    "agent_turn_s": gap.agent_turn_s,
+                }
+            )
+    report.gaps = len(turns)
+    return turns, seen, report
+
+
+def read_turns(path: Path) -> list[Turn]:
+    if not path.exists():
+        return []
+    turns = []
+    with path.open(encoding="utf-8") as handle:
+        for number, line in enumerate(handle, 1):
+            if not line.strip():
+                continue
+            try:
+                turn = json.loads(line)
+            except ValueError:
+                turn = None
+            if turn is None or check_turn(turn):
+                raise ValueError(
+                    f"line {number} of {path.name} is not a valid turn; run agentpace turns"
+                )
+            turns.append(turn)
+    return turns
+
+
+def write_turns(path: Path, turns: Sequence[Turn]) -> None:
+    """Atomic, like the run store, so an interrupted write cannot leave half a file."""
+    order = ("session", "week", "gap_s", "agent_turn_s")
+    path.parent.mkdir(parents=True, exist_ok=True)
+    temporary = path.with_name(path.name + ".tmp")
+    with temporary.open("w", encoding="utf-8", newline="\n") as handle:
+        for turn in turns:
+            handle.write(json.dumps({key: turn[key] for key in order}, separators=(",", ":")))
+            handle.write("\n")
+    os.replace(temporary, path)
+
+
+def update(root: Path, store: Path, secret: bytes) -> TurnsReport:
+    """Replace the gaps of every session read in this pass and keep the rest, because Claude
+    Code deletes old transcripts."""
+    fresh, seen, report = collect(root, secret)
+    kept = [turn for turn in read_turns(store) if turn["session"] not in seen]
+    report.kept_from_earlier = len(kept)
+    for turn in fresh:
+        problems = check_turn(turn)
+        if problems:
+            raise ValueError(f"a turn failed its schema: {problems}")
+    write_turns(store, kept + fresh)
+    return report
+
+
+@dataclass(frozen=True)
+class Summary:
+    gaps: int
+    sessions: int
+    p50: float
+    p80: float
+    p90: float
+    over_hour: float
+    waiting_share: float
+
+
+def summarize(turns: Sequence[Turn]) -> Summary | None:
+    """Uncapped quantiles of the wait per human turn, with no constant applied."""
+    if not turns:
+        return None
+    values = sorted(float(turn["gap_s"]) for turn in turns)
+    totals: dict[str, list[float]] = {}
+    for turn in turns:
+        waiting, working = totals.setdefault(turn["session"], [0.0, 0.0])
+        totals[turn["session"]] = [waiting + turn["gap_s"], working + turn["agent_turn_s"]]
+    shares = [w / (w + a) for w, a in totals.values() if w + a > 0]
+    p50, p80, p90 = (est.quantile(values, q) for q in est.QUANTILES)
+    return Summary(
+        gaps=len(values),
+        sessions=len(totals),
+        p50=p50,
+        p80=p80,
+        p90=p90,
+        over_hour=sum(value > HOUR_S for value in values) / len(values),
+        waiting_share=statistics.median(shares) if shares else 0.0,
+    )

agentpace/validate.py

@@ -0,0 +1,115 @@
+"""Checks every data file must pass, locally and in CI.
+
+Contributed files are parsed, never executed. Findings name a file, a line and a rule, and never
+quote the offending value, because a CI log is public and the value may be the very thing that
+should not be published.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from collections.abc import Sequence
+from dataclasses import dataclass
+from pathlib import Path
+
+from agentpace import ingest, schema
+from agentpace.adapters import EXPERIMENTAL
+
+MAX_DURATION_S = 86_400
+MAX_CALLS_PER_SECOND = 5
+MAX_FILE_BYTES = 20_000_000
+
+_PII = {
+    "email address": re.compile(r"[\w.+-]+@[\w-]+\.[\w.-]+"),
+    "url": re.compile(r"[a-z][a-z0-9+.-]*://", re.IGNORECASE),
+    "windows path": re.compile(r"[A-Za-z]:[\\/]"),
+    "unix path": re.compile(r"/(?:home|Users|root|var|tmp|etc|opt|mnt|workspace)/"),
+    "ip address": re.compile(r"(?<![\d.])\d{1,3}(?:\.\d{1,3}){3}(?![\d.])"),
+    "api key": re.compile(r"\b(?:sk-[A-Za-z0-9_-]{8,}|gh[pousr]_[A-Za-z0-9]{8,}|AKIA[0-9A-Z]{8,})"),
+    "jwt": re.compile(r"\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}"),
+}
+
+
+@dataclass(frozen=True)
+class Finding:
+    file: str
+    line: int
+    rule: str
+
+    def __str__(self) -> str:
+        return f"{self.file}:{self.line}: {self.rule}"
+
+
+def scan_for_pii(text: str) -> list[str]:
+    """Names of the PII shapes present. Runs over raw text, before any parsing."""
+    return [name for name, pattern in _PII.items() if pattern.search(text)]
+
+
+def check_values(row: schema.Row) -> list[str]:
+    """Rules a schema cannot express."""
+    problems: list[str] = []
+    if row["harness"] in EXPERIMENTAL:
+        problems.append(
+            "harness adapter is experimental: rows are not published until it is verified"
+        )
+    if ingest.normalize_model(row["model"]) != row["model"]:
+        problems.append("model is not a known public family or 'other'")
+    if row["duration_s"] > MAX_DURATION_S:
+        problems.append("outlier: duration over 24 hours")
+    if row["tool_calls"] > row["duration_s"] * MAX_CALLS_PER_SECOND:
+        problems.append("outlier: implausible tool calls per second")
+    if row["outcome"] == "completed" and row["tool_calls"] == 0 and not row.get("tokens"):
+        problems.append("outlier: completed with no activity")
+    return problems
+
+
+def _check_file(path: Path, seen: dict[tuple[str, str, int, int], str]) -> list[Finding]:
+    name = path.name
+    if path.suffix != ".jsonl":
+        return [Finding(name, 0, "data files must end in .jsonl")]
+    if path.stat().st_size > MAX_FILE_BYTES:
+        return [Finding(name, 0, "file is too large")]
+    try:
+        lines = path.read_text(encoding="utf-8").splitlines()
+    except UnicodeDecodeError:
+        return [Finding(name, 0, "file is not UTF-8")]
+    findings: list[Finding] = []
+    contributors: set[str] = set()
+    for number, line in enumerate(lines, start=1):
+        if not line.strip():
+            continue
+        for shape in scan_for_pii(line):
+            findings.append(Finding(name, number, f"looks like PII: {shape}"))
+        try:
+            row = json.loads(line)
+        except ValueError:
+            findings.append(Finding(name, number, "not valid JSON"))
+            continue
+        problems = schema.check_row(row)
+        if problems:
+            findings += [Finding(name, number, problem) for problem in problems]
+            continue
+        findings += [Finding(name, number, problem) for problem in check_values(row)]
+        key = schema.dedupe_key(row)
+        if key in seen:
+            findings.append(Finding(name, number, f"duplicate of a run in {seen[key]}"))
+        seen[key] = name
+        contributors.add(row["contributor"])
+    if len(contributors) > 1:
+        findings.append(Finding(name, 0, "a file must hold one contributor"))
+    elif contributors and not name.startswith(next(iter(contributors))):
+        findings.append(Finding(name, 0, "file name must start with its contributor"))
+    return findings
+
+
+def validate(sources: Sequence[Path]) -> list[Finding]:
+    files: list[Path] = []
+    for source in sources:
+        files += sorted(p for p in source.iterdir() if p.is_file()) if source.is_dir() else [source]
+    seen: dict[tuple[str, str, int, int], str] = {}
+    findings: list[Finding] = []
+    for path in files:
+        if path.name != ".gitkeep":
+            findings += _check_file(path, seen)
+    return findings