stackvitals 0.1.0__py3-none-any.whl

stackvitals/ci/api.py

@@ -0,0 +1,218 @@
+import os
+import threading
+import time
+from collections.abc import Callable, Iterator
+from urllib.parse import quote
+
+import httpx
+
+# GitHub Enterprise Server: Actions sets GITHUB_API_URL and GITHUB_GRAPHQL_URL for every job there.
+API = (os.environ.get("GITHUB_API_URL") or "https://api.github.com").rstrip("/")
+GRAPHQL = os.environ.get("GITHUB_GRAPHQL_URL") or f"{API}/graphql"
+
+
+class OutOfBudget(RuntimeError):
+    pass
+
+
+class APIError(RuntimeError):
+    pass
+
+
+class NotFound(APIError):
+    pass
+
+
+class ActionsAPI:
+    # REST reads of Actions runs, attempts and jobs; stops cleanly before the token's limit is used up.
+    def __init__(self, token: str | None, repo: str, client: httpx.Client | None = None,
+                 sleep: Callable[[float], None] = time.sleep, max_requests: int | None = None, reserve: int = 50,
+                 etags: dict | None = None, cacheable: Callable[[str], bool] | None = None):
+        self.repo, self.sleep, self.max_requests, self.reserve = repo, sleep, max_requests, reserve
+        # Conditional reads: a 304 answer reuses the kept body and doesn't count against GitHub's hourly limit.
+        self.etags, self.cacheable, self.unchanged = etags, cacheable, 0
+        self.client = client or httpx.Client(timeout=httpx.Timeout(60.0, connect=15.0), follow_redirects=True)
+        self.headers = {"Accept": "application/vnd.github+json", "X-GitHub-Api-Version": "2022-11-28",
+                        "User-Agent": "stackvitals"}
+        if token and token.strip():
+            self.headers["Authorization"] = f"Bearer {token.strip()}"
+        self.requests = 0
+        self._lock = threading.Lock()
+
+    def _get(self, path: str, params: dict | None = None, raw: bool = False):
+        for attempt in range(4):
+            # Requests run in parallel threads; the count and the budget check must not race.
+            with self._lock:
+                if self.max_requests is not None and self.requests - self.unchanged >= self.max_requests:
+                    raise OutOfBudget(f"stopped after {self.requests} requests (budget)")
+                self.requests += 1
+            key = path + "?" + "&".join(f"{k}={v}" for k, v in sorted((params or {}).items()))
+            kept = self.etags.get(key) if self.etags is not None and not raw and self.cacheable and self.cacheable(path) else None
+            headers = {**self.headers, "If-None-Match": kept[0]} if kept else self.headers
+            try:
+                resp = self.client.get(f"{API}{path}", headers=headers, params=params)
+            except httpx.TransportError:
+                # Dropped connections happen on long collections; try again after a short pause.
+                self.sleep(2 * (attempt + 1))
+                continue
+            if resp.status_code in (403, 429) and ("retry-after" in resp.headers or "secondary" in resp.text.lower()):
+                # Secondary limits ask for a pause; waiting is the only allowed answer.
+                self.sleep(min(float(resp.headers.get("retry-after", 60)), 300))
+                continue
+            if resp.status_code in (403, 429) and resp.headers.get("x-ratelimit-remaining") == "0":
+                raise OutOfBudget("GitHub's hourly limit for this token is used up")
+            if resp.status_code >= 500 and attempt < 3:
+                # GitHub answers some heavy job lists with 502 now and then; a short pause usually fixes it.
+                self.sleep(3 * (attempt + 1))
+                continue
+            if resp.status_code == 304 and kept:
+                with self._lock:
+                    self.unchanged += 1
+                return kept[1]
+            if resp.status_code >= 400:
+                raise APIError(f"GET {path}: HTTP {resp.status_code} {resp.text[:200]}")
+            small = len(resp.content) < 20000
+            if self.etags is not None and not raw and self.cacheable and self.cacheable(path) and resp.headers.get("etag") and small:
+                self.etags[key] = [resp.headers["etag"], resp.json()]
+            left = resp.headers.get("x-ratelimit-remaining")
+            if left is not None and int(left) <= self.reserve:
+                raise OutOfBudget(f"only {left} requests left this hour; keeping a reserve")
+            return resp.content if raw else resp.json()
+        raise OutOfBudget("GitHub kept asking to slow down")
+
+    def get_if_changed(self, path: str, params: dict | None, etag: str | None) -> tuple[dict | None, str | None]:
+        """(body, etag), or (None, etag) when GitHub says nothing changed since `etag` (which costs no hourly limit)."""
+        with self._lock:
+            if self.max_requests is not None and self.requests - self.unchanged >= self.max_requests:
+                raise OutOfBudget(f"stopped after {self.requests} requests (budget)")
+            self.requests += 1
+        headers = {**self.headers, "If-None-Match": etag} if etag else self.headers
+        try:
+            resp = self.client.get(f"{API}{path}", headers=headers, params=params)
+        except httpx.TransportError as exc:
+            raise APIError(f"GET {path}: {type(exc).__name__}") from None
+        if resp.status_code == 304:
+            with self._lock:
+                self.unchanged += 1
+            return None, etag
+        # A limit answer stops the caller (it keeps what it had), never reads as "not available".
+        if resp.status_code in (403, 429) and ("retry-after" in resp.headers or "secondary" in resp.text.lower()
+                                               or resp.headers.get("x-ratelimit-remaining") == "0"):
+            raise OutOfBudget("GitHub asked to slow down or the hourly limit is used up")
+        if resp.status_code >= 400:
+            raise APIError(f"GET {path}: HTTP {resp.status_code} {resp.text[:200]}")
+        left = resp.headers.get("x-ratelimit-remaining")
+        if left is not None and int(left) <= self.reserve:
+            raise OutOfBudget(f"only {left} requests left this hour; keeping a reserve")
+        return resp.json(), resp.headers.get("etag")
+
+    def fetch(self, path: str) -> tuple[int, bytes]:
+        """(status, body) for endpoints whose status says something, like 202 while a report is being made. Redirects are
+        followed; httpx drops the token when one leaves api.github.com."""
+        with self._lock:
+            if self.max_requests is not None and self.requests - self.unchanged >= self.max_requests:
+                raise OutOfBudget(f"stopped after {self.requests} requests (budget)")
+            self.requests += 1
+        try:
+            resp = self.client.get(f"{API}{path}", headers=self.headers)
+        except httpx.TransportError as exc:
+            raise APIError(f"GET {path}: {type(exc).__name__}") from None
+        if resp.status_code >= 400:
+            raise APIError(f"GET {path}: HTTP {resp.status_code} {resp.text[:200]}")
+        return resp.status_code, resp.content
+
+    def send(self, method: str, path: str, body: dict | None = None) -> dict:
+        # Writes (PR comments, re-runs) are counted against the same budget and never retried blindly.
+        if os.environ.get("STACKVITALS_READ_ONLY"):
+            # A run on someone's machine (`stackvitals ui --collect`) reads GitHub and never writes to it.
+            raise APIError(f"{method} {path}: read-only local run, nothing is written to GitHub")
+        with self._lock:
+            self.requests += 1
+        resp = self.client.request(method, f"{API}{path}", headers=self.headers, json=body)
+        if resp.status_code >= 400:
+            raise APIError(f"{method} {path}: HTTP {resp.status_code} {resp.text[:200]}")
+        return resp.json() if resp.content else {}
+
+    def runs_on_day(self, day: str) -> Iterator[dict]:
+        # Filtered run lists stop at 1,000 results, so callers ask one day at a time.
+        for page in range(1, 11):
+            body = self._get(f"/repos/{self.repo}/actions/runs",
+                             {"created": day, "status": "completed", "per_page": 100, "page": page})
+            runs = body.get("workflow_runs") or []
+            yield from runs
+            if len(runs) < 100:
+                return
+
+    def file(self, path: str, ref: str | None, repo: str | None = None) -> bytes:
+        # One file from a branch, raw (used to read a repo's stackvitals-data findings in one request); Portal reads other repos.
+        with self._lock:
+            if self.max_requests is not None and self.requests - self.unchanged >= self.max_requests:
+                raise OutOfBudget(f"stopped after {self.requests} requests (budget)")
+            self.requests += 1
+        # Paths come from PRs: "#" or "?" in a folder name would otherwise cut the URL short.
+        for attempt in range(3):
+            try:
+                resp = self.client.get(f"{API}/repos/{repo or self.repo}/contents/{quote(path, safe='/')}",
+                                       params={"ref": ref} if ref else None,
+                                       headers={**self.headers, "Accept": "application/vnd.github.raw"})
+                break
+            except httpx.TransportError:
+                # A dropped connection is retried, then reported like any refusal: it must never stop a run.
+                if attempt == 2:
+                    raise APIError(f"GET contents/{path}@{ref}: network error")
+                self.sleep(2 * (attempt + 1))
+        if resp.status_code == 404:
+            raise NotFound(f"GET contents/{path}@{ref}: HTTP 404")
+        if resp.status_code >= 400:
+            raise APIError(f"GET contents/{path}@{ref}: HTTP {resp.status_code}")
+        return resp.content
+
+    def file_if_changed(self, path: str, ref: str | None = None, etag: str | None = None) -> tuple[bytes | None, str | None]:
+        """(content, etag) of a file, or (None, etag) when it still matches `etag`: GitHub doesn't count a 304 against the
+        hourly limit, so a file read on every run costs nothing while it stays the same. No ref means the default branch."""
+        with self._lock:
+            if self.max_requests is not None and self.requests - self.unchanged >= self.max_requests:
+                raise OutOfBudget(f"stopped after {self.requests} requests (budget)")
+            self.requests += 1
+        headers = {**self.headers, "Accept": "application/vnd.github.raw", **({"If-None-Match": etag} if etag else {})}
+        for attempt in range(3):
+            try:
+                resp = self.client.get(f"{API}/repos/{self.repo}/contents/{quote(path, safe='/')}",
+                                       params={"ref": ref} if ref else None, headers=headers)
+            except httpx.TransportError:
+                if attempt == 2:
+                    raise APIError(f"GET contents/{path}: network error")
+                self.sleep(2 * (attempt + 1))
+                continue
+            if resp.status_code >= 500 and attempt < 2:
+                self.sleep(2 * (attempt + 1))
+                continue
+            break
+        if resp.status_code == 304:
+            return None, resp.headers.get("etag") or etag
+        if resp.status_code == 404:
+            raise NotFound(f"GET contents/{path}: HTTP 404")
+        if resp.status_code >= 400:
+            raise APIError(f"GET contents/{path}: HTTP {resp.status_code}")
+        return resp.content, resp.headers.get("etag")
+
+    def artifacts(self, run_id: int) -> list[dict]:
+        body = self._get(f"/repos/{self.repo}/actions/runs/{run_id}/artifacts", {"per_page": 100})
+        return body.get("artifacts") or []
+
+    def download(self, artifact_id: int) -> bytes:
+        # GitHub answers with a redirect to storage; httpx drops the token when it leaves github.com.
+        return self._get(f"/repos/{self.repo}/actions/artifacts/{artifact_id}/zip", raw=True)
+
+    def attempt(self, run_id: int, n: int) -> dict:
+        return self._get(f"/repos/{self.repo}/actions/runs/{run_id}/attempts/{n}")
+
+    def jobs(self, run_id: int, n: int) -> list[dict]:
+        out, page = [], 1
+        while True:
+            body = self._get(f"/repos/{self.repo}/actions/runs/{run_id}/attempts/{n}/jobs", {"per_page": 100, "page": page})
+            batch = body.get("jobs") or []
+            out.extend(batch)
+            if len(batch) < 100:
+                return out
+            page += 1

stackvitals/ci/automation.py

@@ -0,0 +1,212 @@
+import glob
+import subprocess
+from datetime import timedelta
+from urllib.parse import urlencode
+
+import httpx
+
+from stackvitals.ci.api import ActionsAPI, APIError
+from stackvitals.ci.feedback import SAFE_FLAKES
+from stackvitals.ci.owners import owners_line
+from stackvitals.ci.report import began_text
+from stackvitals.ci.testreports import merge, parse_report
+
+LABEL = "flaky-test"
+
+
+def should_rerun(items: list[dict], run_attempt: int) -> bool:
+    # Re-run once, and only when every failure is a known flake or an outside service failing.
+    return run_attempt == 1 and bool(items) and all(i["verdict"] in ("known flaky", "outside failure") for i in items)
+
+
+def rerun_failed_jobs(api: ActionsAPI, run_id: int) -> None:
+    api.send("POST", f"/repos/{api.repo}/actions/runs/{run_id}/rerun-failed-jobs")
+
+
+def expired(result: dict, settings: dict | None, now) -> list[str]:
+    """Flaky tests quarantined on their own for longer than [ci.quarantine] max_days: they fail builds again until fixed."""
+    days = ((((settings or {}).get("ci") or {}).get("quarantine") or {}).get("max_days")) or 0
+    if not days or now is None:
+        return []
+    cut = (now - timedelta(days=days)).strftime("%Y-%m-%dT%H:%M:%SZ")
+    return sorted(t["id"] for t in result.get("tests", []) if t["state"] == "flaky" and (t.get("since") or "~") < cut)
+
+
+def quarantine_list(result: dict, settings: dict | None = None, decisions: dict | None = None, now=None) -> list[str]:
+    """Flaky tests, plus the repo's "always" list and tests someone quarantined, minus its "never" list, tests someone
+    took out of quarantine (.github/stackvitals.toml and the decisions log) and flaky tests past max_days."""
+    q = ((settings or {}).get("ci") or {}).get("quarantine") or {}
+    # Only tests that flaked as often as a safe re-run needs are let through on their own; one flake isn't enough.
+    flaky = {t["id"] for t in result.get("tests", []) if t["state"] == "flaky" and t.get("flake_events", 0) >= SAFE_FLAKES}
+    flaky -= set(expired(result, settings, now))
+    picked = lambda verb: {d["id"] for d in (decisions or {}).values() if d.get("kind") == "test" and d.get("verb") == verb}
+    return sorted((flaky | set(q.get("always") or []) | picked("quarantine")) - set(q.get("never") or []) - picked("unquarantine"))
+
+
+def job_key(j: dict) -> str:
+    # How people name a job in decisions and issue titles: the workflow file, then the job.
+    return f"{(j.get('workflow') or '?').rsplit('/', 1)[-1]} / {j['job']}"
+
+
+def apply_decisions(result: dict, decisions: dict) -> None:
+    """Each test and job carries the decision made about it; muted or acknowledged ones raise no alert and get no new
+    issue (acknowledged has no end)."""
+    for kind, items, key in (("test", result.get("tests") or [], lambda t: t["id"]), ("job", result.get("jobs") or [], job_key)):
+        for item in items:
+            d = decisions.get(f"{kind}:{key(item)}")
+            if not d:
+                continue
+            item["decision"] = {k: d.get(k) for k in ("verb", "by", "at", "until", "reason")}
+            if d.get("verb") in ("mute", "acknowledge"):
+                item["muted_until"] = d.get("until") or "9999-12-31"
+
+
+def quarantine_exit(code: int, failed: list[str], quarantined: set | None) -> int:
+    # Pass only when the run failed solely on quarantined tests; an unreadable list quarantines nothing.
+    if code == 0 or quarantined is None or not failed:
+        return code
+    return 0 if set(failed) <= quarantined else code
+
+
+def run_quarantined(command: list[str], report_globs: list[str], quarantined: set | None) -> int:
+    code = subprocess.run(command).returncode
+    if code == 0:
+        return 0
+    parts = []
+    for pattern in report_globs:
+        for path in glob.glob(pattern, recursive=True):
+            with open(path, "rb") as f:
+                parsed = parse_report(path, f.read())
+            if parsed:
+                parts.append(parsed)
+    return quarantine_exit(code, [f[0] for f in merge(parts)["failed"]], quarantined)
+
+
+def _flaky_names(result: dict | None) -> dict[str, str]:
+    out = {}
+    for t in (result or {}).get("tests", []):
+        # A muted test is still measured, but nobody is alerted or given an issue until the mute ends.
+        if t["state"] == "flaky" and not t.get("muted_until"):
+            out[f"test {t['id']}"] = f"Flaky test: {t['id']}"
+    for j in (result or {}).get("jobs", []):
+        if j["state"] == "flaky" and not j.get("muted_until"):
+            out[f"job {job_key(j)}"] = f"Flaky job: {job_key(j)}"
+    return out
+
+
+def new_flakes(old: dict | None, new: dict) -> list[str]:
+    # The first run has nothing to compare with, so it never alerts.
+    if old is None:
+        return []
+    before = _flaky_names(old)
+    return [k for k in _flaky_names(new) if k not in before]
+
+
+def _owners_by_key(result: dict | None) -> dict[str, list[str]]:
+    out = {f"test {t['id']}": t.get("owners") for t in (result or {}).get("tests", [])}
+    out.update({f"job {(j.get('workflow') or '?').rsplit('/', 1)[-1]} / {j['job']}": j.get("owners")
+                for j in (result or {}).get("jobs", [])})
+    return out
+
+
+def alert_message(names: list[str], repo: str, result: dict | None = None, links=()) -> dict:
+    from stackvitals.core import chat
+    owners = _owners_by_key(result)
+    lines = [f"- `{chat.safe(n)}`" + (f" (owners {' '.join(owners[n])})" if owners.get(n) else "") for n in names]
+    return chat.message(f"Newly flaky in {repo}", lines, links,
+                        {"repo": repo, "alerts": [{"kind": "newly_flaky", "name": n} for n in names]})
+
+
+def owners_of(names: list[str], result: dict | None) -> dict[str, list[str]]:
+    owners = _owners_by_key(result)
+    return {n: owners.get(n) or [] for n in names}
+
+
+def send_alert(url: str, msg: dict, client: httpx.Client | None = None) -> str | None:
+    """Every chat (Teams included) gets its own shape; returns what went wrong, if anything."""
+    from stackvitals.core import chat
+    return chat.post(url, msg, client, "the flaky-test webhook")
+
+
+def _evidence(entry: dict) -> str:
+    last = (entry.get("last_flake") or "")[:10]
+    began = began_text(entry.get("began"))
+    return (f"StackVitals saw this fail and then pass with no code change {entry.get('flake_events', 0)} times "
+            f"in {entry.get('runs', 0)} runs over the last 30 days" + (f" (last on {last})" if last else "") + "."
+            + (f" It {began}." if began else ""))
+
+
+def _issue_body(entry: dict, mention: bool = False, linear: bool = False) -> str:
+    owners = owners_line(entry, mention)
+    name = entry.get("id") or f"{(entry.get('workflow') or '?').rsplit('/', 1)[-1]} / {entry.get('job')}"
+    # A team that tracks work in Linear files it there in one click, prefilled; no token is needed.
+    file_it = (f"\n\n[File it in Linear](https://linear.new?{urlencode({'title': f'Flaky: {name}', 'description': _evidence(entry)})})"
+               if linear else "")
+    return (_evidence(entry) + (f"\n\n{owners}" if owners else "") + file_it
+            + "\n\nThis issue is kept up to date automatically and closed once it stays quiet for two weeks. "
+              "Close it as *not planned* if the team accepts this flake: StackVitals keeps it closed.")
+
+
+def _first_person(entry: dict) -> str | None:
+    # CODEOWNERS entries are @user, @org/team or e-mail addresses; only a user can be assigned.
+    for owner in entry.get("owners") or []:
+        if owner.startswith("@") and "/" not in owner:
+            return owner[1:]
+    return None
+
+
+def _all_issues(api: ActionsAPI) -> dict[str, dict]:
+    """The newest issue for each title, open or closed, so a decision someone made on it is respected."""
+    found = {}
+    for page in range(1, 4):
+        batch = api._get(f"/repos/{api.repo}/issues", {"labels": LABEL, "state": "all", "per_page": 100, "page": page}) or []
+        for issue in batch:
+            if "pull_request" not in issue and issue["number"] > found.get(issue["title"], {}).get("number", 0):
+                found[issue["title"]] = issue
+        if len(batch) < 100:
+            break
+    return found
+
+
+def sync_issues(api: ActionsAPI, result: dict, mention_owners: bool = False, assign_owners: bool = False,
+                linear: bool = False) -> dict:
+    """One issue per flaky test or job for its whole life: updated while flaky, closed once fixed, reopened with fresh
+    evidence when it flakes again after being closed as done. Closed as "not planned" (or a duplicate) is the team's
+    decision to live with it: it stays closed and nothing new is opened. Owners are named in code format (no
+    notification) unless mention_owners is on; assign_owners assigns the first owner who is a person."""
+    issues = _all_issues(api)
+    # Muted tests get no new issue; one already open stays as it is until the mute ends.
+    entries = {f"Flaky test: {t['id']}": t for t in result.get("tests", []) if not t.get("muted_until")}
+    entries.update({f"Flaky job: {job_key(j)}": j for j in result.get("jobs", []) if not j.get("muted_until")})
+    counts = {"created": 0, "updated": 0, "closed": 0, "reopened": 0, "kept_closed": 0}
+    for title, entry in entries.items():
+        issue = issues.get(title)
+        if entry["state"] == "flaky":
+            if issue and issue.get("state") == "open":
+                api.send("PATCH", f"/repos/{api.repo}/issues/{issue['number']}", {"body": _issue_body(entry, mention_owners, linear)})
+                counts["updated"] += 1
+            elif issue and issue.get("state_reason") not in (None, "completed", "reopened"):
+                counts["kept_closed"] += 1
+            elif issue and issue.get("closed_at") and (entry.get("last_flake") or "")[:19] <= issue["closed_at"][:19]:
+                pass  # closed as done after its last flake: it stays closed unless it flakes again
+            elif issue:
+                api.send("PATCH", f"/repos/{api.repo}/issues/{issue['number']}",
+                         {"state": "open", "body": _issue_body(entry, mention_owners, linear)})
+                api.send("POST", f"/repos/{api.repo}/issues/{issue['number']}/comments",
+                         {"body": "Flaky again. " + _evidence(entry)})
+                counts["reopened"] += 1
+            else:
+                body = {"title": title, "body": _issue_body(entry, mention_owners, linear), "labels": [LABEL]}
+                person = _first_person(entry) if assign_owners else None
+                try:
+                    api.send("POST", f"/repos/{api.repo}/issues", {**body, **({"assignees": [person]} if person else {})})
+                except APIError:
+                    if not person:
+                        raise
+                    # Someone without access to the repo can't be assigned; the issue still matters.
+                    api.send("POST", f"/repos/{api.repo}/issues", body)
+                counts["created"] += 1
+        elif issue and issue.get("state") == "open" and entry["state"] == "fixed":
+            api.send("PATCH", f"/repos/{api.repo}/issues/{issue['number']}", {"state": "closed", "state_reason": "completed"})
+            counts["closed"] += 1
+    return counts

stackvitals/ci/bench.py

@@ -0,0 +1,195 @@
+import json
+import math
+import os
+import time
+from datetime import datetime, timedelta
+from pathlib import Path
+
+import httpx
+
+from mergescout.bench.ci import repo_stats, tier
+from mergescout.github.releases import ReleaseError, Releases
+from stackvitals.core.stats import median_range, shrink, wilson
+
+# The public repo whose bench-latest release holds copies of the CI and shared benchmark files ("owner/repo").
+PUBLIC_HOME: str | None = "stackvitals/stackvitals"
+# Built daily by the crawler from about 90k active public repos; the private store keeps every file, read with a token.
+BENCH_REPO, BENCH_TAG, BENCH_FILE = "kaushik-yadav/merge-scout", "bench-latest", "ci-benchmarks.json"
+
+
+def _asset_url(name: str) -> str | None:
+    return f"https://github.com/{PUBLIC_HOME}/releases/download/{BENCH_TAG}/{name}" if PUBLIC_HOME else None
+
+
+# A missing or unreachable public file (before launch, say) just leaves the comparison out.
+DEFAULT_URL = _asset_url(BENCH_FILE)
+RANKS = (0.10, 0.25, 0.50, 0.75, 0.90)
+EVIDENCE_MAX = 300  # pushes listed behind the first-try share, newest first
+
+
+def _push_runs(data: dict) -> list[dict]:
+    final = {}
+    for r in data.get("runs", []):
+        if r.get("ev") == "push" and r["att"] >= final.get(r["id"], {}).get("att", 0):
+            final[r["id"]] = r
+    return [{"id": r["id"], "head_sha": r.get("sha"), "path": r.get("path"), "conclusion": r.get("c"),
+              "head_branch": r.get("br"), "run_attempt": r["att"], "status": "completed", "created_at": r.get("cr"),
+              "run_started_at": r.get("st"), "updated_at": r.get("up")} for r in final.values()]
+
+
+def push_stats(data: dict, now: datetime, days: int = 30) -> dict | None:
+    """This repo's pushes measured exactly like the benchmark sampler measures every other repo, with each number's
+    likely range."""
+    out = repo_stats(_push_runs(data), since=now - timedelta(days=days), durations=True, evidence=EVIDENCE_MAX)
+    if not out:
+        return out
+    durations = out.pop("durations")
+    green = round(out["first_try_share"] * out["pushes"])
+    out["first_try_range"] = [round(x, 4) for x in wilson(green, out["pushes"])]
+    rng = median_range(durations)
+    out["median_range_s"] = [round(x) for x in rng] if rng else None
+    return out
+
+
+def push_window(data: dict, start: str, end: str) -> dict | None:
+    """The push measure over runs created from `start` up to `end`, with each push's wait kept for change tests."""
+    return repo_stats([r for r in _push_runs(data) if r["created_at"] and start <= r["created_at"] < end], durations=True)
+
+
+def push_stats_by(data: dict, period) -> dict[str, dict]:
+    """The same push measure for each period (a week, say); periods with too few pushes are left out."""
+    groups = {}
+    for r in _push_runs(data):
+        if r["created_at"]:
+            groups.setdefault(period(r["created_at"]), []).append(r)
+    return {k: s for k, g in groups.items() if (s := repo_stats(g))}
+
+
+def load(token: str | None, cache_dir: Path, client: httpx.Client | None = None, name: str = BENCH_FILE,
+         url_env: str = "STACKVITALS_BENCH_URL", validate=None, default_url: str | None = DEFAULT_URL) -> dict | None:
+    """The latest benchmarks, or None: comparisons are extra and must never break a run."""
+    validate = validate or _valid
+    source = os.environ.get(url_env) or default_url
+    if not source and not token:
+        return None
+    cache = Path(cache_dir) / name
+    try:
+        if source and not source.startswith(("http://", "https://")):
+            return validate(json.loads(Path(source).read_text(encoding="utf-8")))
+        if cache.exists() and time.time() - cache.stat().st_mtime < 86400:
+            return validate(json.loads(cache.read_text(encoding="utf-8")))
+        cache.parent.mkdir(parents=True, exist_ok=True)
+        if source and _fetch(source, cache, client):
+            return validate(json.loads(cache.read_text(encoding="utf-8")))
+        # A token falls back to the private store, so a missing public copy changes nothing for those who can read it.
+        if not token or (source and os.environ.get(url_env)):
+            return None
+        if not Releases(token, BENCH_REPO, client=client).download(BENCH_TAG, name, cache):
+            return None
+        return validate(json.loads(cache.read_text(encoding="utf-8")))
+    except (httpx.HTTPError, ReleaseError, OSError, ValueError):
+        return None
+
+
+def _fetch(url: str, cache: Path, client: httpx.Client | None) -> bool:
+    try:
+        # Release downloads redirect to a file host; the caller's client may not follow redirects by default.
+        resp = (client or httpx.Client(timeout=30)).get(url, follow_redirects=True)
+        resp.raise_for_status()
+    except httpx.HTTPError:
+        return False
+    cache.write_bytes(resp.content)
+    return True
+
+
+def _group_ok(g) -> bool:
+    nums = lambda q: isinstance(q, dict) and all(isinstance(q.get(f"p{int(r * 100)}"), (int, float)) for r in RANKS)  # noqa: E731
+    return (isinstance(g, dict) and isinstance(g.get("repos"), int) and not isinstance(g.get("repos"), bool)
+            and nums(g.get("median_s")) and nums(g.get("first_try_share")))
+
+
+def _valid(bench) -> dict | None:
+    # Only well-formed groups are kept, so a bad file can neither break a run nor reach a page.
+    if not isinstance(bench, dict) or not isinstance(bench.get("groups"), dict):
+        return None
+    groups = {k: g for k, g in bench["groups"].items() if isinstance(k, str) and k.count("|") == 1 and _group_ok(g)}
+    return {**bench, "groups": groups} if groups else None
+
+
+# Who every comparison is with: the crawl's sample, which may not resemble a private team's work.
+PEERS_ARE = "active public open-source repos (100+ stars, pushed in the last 90 days)"
+# Opt-in figures that repos filed themselves ([benchmarks] share), public and private; published at launch.
+SHARED_FILE = "shared-benchmarks.json"
+SHARED_URL = _asset_url(SHARED_FILE)
+SHARED_ARE = "repos that chose to share their rounded CI figures, public and private, latest month each"
+
+
+def fit(key: str, language: str | None, size: str) -> str:
+    # The closest group is a loose fit when it had to drop the language or the size; the pages say so.
+    return "" if key == f"{language}|{size}" else "a loose fit: too few repos share this language and size, so a wider group"
+
+
+def _label(key: str, everyone: str = "public repos we track") -> str:
+    lang, size = key.split("|")
+    stars = f"with {size} stars" if size != "all" else ""
+    if lang == "all":
+        return f"repos {stars}" if stars else everyone
+    return f"{lang} repos {stars}".strip()
+
+
+def _rank(value: float, q: dict) -> float:
+    # Rough place in the group from its published percentiles, kept between 10% and 90%.
+    points = [(r, q[f"p{int(r * 100)}"]) for r in RANKS]
+    if value <= points[0][1]:
+        return RANKS[0]
+    if value >= points[-1][1]:
+        return RANKS[-1]
+    for (r1, v1), (r2, v2) in zip(points, points[1:]):
+        if v1 <= value <= v2 and v2 > v1:
+            return r1 + (value - v1) / (v2 - v1) * (r2 - r1)
+    return 0.5
+
+
+def _about(share: float) -> int:
+    return int(math.floor(share * 100 / 5) * 5)
+
+
+def _pulled(stats: dict, group: dict) -> dict:
+    # The share pulled toward the group's prior: few pushes move little; many keep the repo's own share.
+    prior = group.get("first_try_prior")
+    if not (isinstance(prior, list) and len(prior) == 2 and all(isinstance(x, (int, float)) and x > 0 for x in prior)):
+        return {}
+    green = round(stats["first_try_share"] * stats["pushes"])
+    return {"pulled": shrink(green, stats["pushes"], tuple(prior))}
+
+
+def compare(stats: dict | None, bench: dict | None, language: str | None, stars: int | None,
+            everyone: str = "public repos we track") -> dict | None:
+    """Where this repo stands among similar public repos: the closest group with enough repos."""
+    bench = _valid(bench) if bench else None
+    if not stats or not bench:
+        return None
+    groups, size = bench["groups"], tier(stars)
+    key = next((k for k in (f"{language}|{size}", f"{language}|all", f"all|{size}", "all|all") if k in groups), None)
+    if key is None:
+        return None
+    g = groups[key]
+    r = _rank(stats["median_s"], g["median_s"])
+    time_place = f"slower than about {_about(r)}%" if r >= 0.5 else f"faster than about {_about(1 - r)}%"
+    r2 = _rank(stats["first_try_share"], g["first_try_share"])
+    green_place = f"above about {_about(r2)}%" if r2 >= 0.5 else f"below about {_about(1 - r2)}%"
+    return {"group": _label(key, everyone), "repos": g["repos"], "built_at": bench.get("built_at"), "fit": fit(key, language, size),
+            "metrics": [{"name": "Time from push to all checks done", "kind": "time", "you": stats["median_s"],
+                         "typical": g["median_s"]["p50"], "place": time_place},
+                        {"name": "Pushes green on the first try", "kind": "share", "you": stats["first_try_share"],
+                         "typical": g["first_try_share"]["p50"], "place": green_place, **_pulled(stats, g)}]}
+
+
+def badge(stats: dict | None) -> dict:
+    """A shields.io endpoint: the share of pushes whose checks all passed on the first try."""
+    if not stats:
+        return {"schemaVersion": 1, "label": "CI first try", "message": "no data", "color": "lightgrey"}
+    share = stats["first_try_share"]
+    color = next(c for floor, c in ((0.9, "brightgreen"), (0.8, "green"), (0.7, "yellow"), (0.5, "orange"), (0, "red"))
+                 if share >= floor)
+    return {"schemaVersion": 1, "label": "CI first try", "message": f"{share:.0%}", "color": color}