stackvitals 0.1.0__py3-none-any.whl

stackvitals/flow/gitlab_mr.py

@@ -0,0 +1,75 @@
+"""GitLab merge requests and deployments in the shapes Delivery reads for GitHub, so DORA and the review phases come
+out the same way (Middleware's GitLab support, without a server):
+
+    stackvitals delivery group/project --gitlab [--days 90]
+
+Each merged merge request's notes tell when someone else first answered and when it was approved; its merge (or
+squash) commit links it to the production deployment that shipped it. Public projects need no token."""
+from datetime import datetime, timedelta
+
+from stackvitals.ci.api import APIError
+from stackvitals.ci.gitlab import GitLab
+
+APPROVED = ("approved this merge request",)
+READY = ("marked this merge request as ready", "marked as ready")
+DRAFT = ("marked this merge request as draft", "marked as a draft", "marked this merge request as a draft")
+
+
+def _iso(s: str | None) -> str | None:
+    if not s:
+        return None
+    return datetime.fromisoformat(s.replace("Z", "+00:00")).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+
+def mr_record(mr: dict, notes: list[dict]) -> dict:
+    """One merge request as Team Flow's pull request record."""
+    author = (mr.get("author") or {}).get("username")
+    ev, talk = [], []
+    for n in sorted(notes or [], key=lambda n: n.get("created_at") or ""):
+        who, at, body = (n.get("author") or {}).get("username"), _iso(n.get("created_at")), (n.get("body") or "").lower()
+        if n.get("system"):
+            if any(body.startswith(x) for x in APPROVED):
+                ev.append(["review", at, who, "APPROVED", "MEMBER", "User", False])
+            elif any(body.startswith(x) for x in READY):
+                ev.append(["ready", at])
+            elif any(body.startswith(x) for x in DRAFT):
+                ev.append(["draft", at])
+        elif who and who != author:
+            talk.append([at, who, "MEMBER", "User", False])
+    if mr.get("merged_at"):
+        ev.append(["merged", _iso(mr["merged_at"]), (mr.get("merged_by") or {}).get("username")])
+    return {"t": "pr", "n": mr["iid"], "ti": (mr.get("title") or "")[:200], "draft": bool(mr.get("draft")),
+            "st": "MERGED" if mr.get("merged_at") else (mr.get("state") or "").upper(), "cr": _iso(mr.get("created_at")),
+            "up": _iso(mr.get("updated_at")), "mg": _iso(mr.get("merged_at")), "cl": _iso(mr.get("closed_at") or mr.get("merged_at")),
+            "au": author, "at": "Bot" if (mr.get("author") or {}).get("bot") else "User", "as": "MEMBER",
+            "base": mr.get("target_branch"), "head": mr.get("source_branch"),
+            "mc": mr.get("squash_commit_sha") or mr.get("merge_commit_sha") or mr.get("sha"),
+            "lb": list(mr.get("labels") or []), "paths": [], "rq": [], "ev": sorted(ev, key=lambda e: e[1] or ""),
+            "cm": talk, "fa": None, "nc": None}
+
+
+def deployment_record(d: dict) -> dict:
+    """One GitLab deployment as the repo query's deployment."""
+    state = {"success": "SUCCESS", "failed": "FAILURE", "canceled": None}.get(d.get("status"))
+    return {"env": (d.get("environment") or {}).get("name"), "at": _iso(d.get("created_at")), "sha": d.get("sha"),
+            "state": state, "done": _iso(d.get("finished_at") or d.get("updated_at")) if state == "SUCCESS" else None}
+
+
+def collect(lab: GitLab, now: datetime, days: int = 90, max_mrs: int = 400) -> tuple[dict, dict]:
+    """(flow, repo) for Delivery: merged merge requests of the window with their notes, and the deployments."""
+    since = (now - timedelta(days=days)).strftime("%Y-%m-%dT%H:%M:%SZ")
+    info = lab.get("") or {}
+    closed = []
+    for mr in lab.pages("/merge_requests", {"state": "merged", "updated_after": since, "order_by": "updated_at"}):
+        if len(closed) >= max_mrs:
+            break
+        try:
+            notes = list(lab.pages(f"/merge_requests/{mr['iid']}/notes", {"sort": "asc"}, limit=3))
+        except APIError:
+            # gitlab.com shows notes only with a token, even on public projects: phases then end at the merge.
+            notes = []
+        closed.append(mr_record(mr, notes))
+    deps = [deployment_record(d) for d in lab.pages("/deployments", {"updated_after": since, "order_by": "updated_at"}, limit=20)]
+    repo = {"default_branch": info.get("default_branch"), "private": info.get("visibility") != "public",
+            "deployments": [d for d in deps if d["state"]], "releases": []}
+    return {"closed": closed, "open": []}, repo

stackvitals/flow/knowledge.py

@@ -0,0 +1,202 @@
+from collections import defaultdict
+from functools import lru_cache
+from datetime import datetime, timedelta
+
+from stackvitals import codeowners
+from stackvitals.core.actors import People
+from stackvitals.flow.analyze import _human_reviews, _iso, _median, _secs, _t, is_bot
+
+HALF_LIFE_DAYS = 180
+SINGLE_OWNER = 0.8
+LOAD_SCALE = 4
+# CODEOWNERS only nudges: on home-assistant a full boost cut hits from 46% to 41%, a quarter cost nothing.
+OWNER_BOOST = 0.25
+# Who reviewed this author's earlier PRs counts too: with it, a real reviewer was in the top 3 for 79.3% of 555 held-out
+# PRs, against 77.5% without and 73.0% for the busiest reviewers (bench/accuracy/team).
+PAIR_WEIGHT = 1.0
+REVERT_WINDOW = timedelta(days=7)
+# Lock files change with every dependency bump; they say nothing about who knows the code.
+LOCK_FILES = {"Cargo.lock", "package-lock.json", "yarn.lock", "pnpm-lock.yaml", "poetry.lock", "uv.lock", "go.sum",
+              "Gemfile.lock", "composer.lock", "Pipfile.lock", "bun.lockb", "flake.lock", "mix.lock", "pubspec.lock"}
+
+
+def _areas(paths: list[str] | None) -> set[str]:
+    return {area(path) for path in paths or [] if path.rsplit("/", 1)[-1] not in LOCK_FILES}
+
+
+def area(path: str) -> str:
+    """A path's area: its first two folders ("src/db"), its folder, or "(root)"."""
+    parts = path.split("/")
+    if len(parts) == 1:
+        return "(root)"
+    return parts[0] if len(parts) == 2 else "/".join(parts[:2])
+
+
+class Expertise(dict):
+    """Area -> {login: weight}, plus how many PRs each person wrote and reviewed there."""
+
+    def __init__(self):
+        super().__init__()
+        self.authored = defaultdict(lambda: defaultdict(int))
+        self.reviewed = defaultdict(lambda: defaultdict(int))
+        # People who can review here: seen as owner, member or collaborator.
+        self.insiders = set()
+        # Author -> {reviewer: merged PRs of theirs that person reviewed}.
+        self.pairs = defaultdict(lambda: defaultdict(int))
+
+
+def _merged_humans(prs: list[dict], now: datetime, days: int = 365, before: str | None = None) -> list[dict]:
+    since = _iso(now - timedelta(days=days))
+    return [p for p in prs if p.get("mg") and p["mg"] >= since and not is_bot(p) and (before is None or p["mg"] < before)]
+
+
+def expertise(prs: list[dict], now: datetime, before: str | None = None, people: People | None = None) -> Expertise:
+    """Who knows each area: authors and maintainer reviewers of merged PRs, recent work counting more."""
+    ex = Expertise()
+    if people is None:
+        people = People(prs)
+    for p in _merged_humans(prs, now, before=before):
+        age = (now - _t(p["mg"])).total_seconds() / 86400
+        w = 0.5 ** (age / HALF_LIFE_DAYS)
+        reviewers = {e[2] for e in _human_reviews(p) if people.is_insider(e[2])}
+        ex.insiders |= reviewers | ({p["au"]} if people.is_insider(p.get("au")) else set())
+        for r in reviewers:
+            ex.pairs[p["au"]][r] += 1
+        for a in _areas(p.get("paths")):
+            known = ex.setdefault(a, {})
+            known[p["au"]] = known.get(p["au"], 0.0) + w
+            ex.authored[a][p["au"]] += 1
+            for r in reviewers:
+                known[r] = known.get(r, 0.0) + 0.5 * w
+                ex.reviewed[a][r] += 1
+    return ex
+
+
+@lru_cache(maxsize=16)
+def _rules(text: str | None) -> list:
+    return codeowners.parse(text)
+
+
+def codeowners_for(path: str, text: str | None) -> list[str]:
+    """Owners of a path from CODEOWNERS text, matched like GitHub (shared matcher), without the "@"."""
+    return [w.lstrip("@") for w in codeowners.owners_for(path, _rules(text))]
+
+
+def suggest_reviewers(pr: dict, ex: Expertise, load: dict[str, int], owners: dict[str, list[str]],
+                      speed: dict[str, float] | None = None, k: int = 3) -> list[dict]:
+    """Up to k people for a PR: who knows its areas, plus code owners, eased off by how busy they are.
+    Answer speed only counts in private views (speed given)."""
+    areas = _areas(pr.get("paths"))
+    score = defaultdict(float)
+    for a in areas:
+        for who, w in ex.get(a, {}).items():
+            score[who] += w
+    top = max(score.values(), default=1.0)
+    owned = {o for path in pr.get("paths") or [] for o in owners.get(path, [])}
+    for o in owned:
+        score[o] += OWNER_BOOST * top
+    paired = {r: n for r, n in ex.pairs.get(pr.get("au"), {}).items() if r != pr.get("au")}
+    for r, n in paired.items():
+        score[r] += PAIR_WEIGHT * top * n / sum(paired.values())
+    out = []
+    for who, s in score.items():
+        if who == pr.get("au") or (who or "").endswith("[bot]") or (who not in ex.insiders and who not in owned):
+            continue
+        busy = load.get(who, 0)
+        s = s / (1 + busy / LOAD_SCALE)
+        if speed and who in speed and speed[who] is not None:
+            s = s / (1 + speed[who] / (2 * 86400))
+        wrote = sum(ex.authored[a].get(who, 0) for a in areas)
+        reviewed = sum(ex.reviewed[a].get(who, 0) for a in areas)
+        why = []
+        if wrote or reviewed:
+            why.append(f"wrote {wrote} and reviewed {reviewed} merged PRs in {', '.join(sorted(areas))}")
+        if who in owned:
+            why.append("code owner")
+        if paired.get(who):
+            why.append(f"reviewed {paired[who]} of this author's merged PRs")
+        if busy:
+            why.append(f"{busy} open review requests")
+        if speed and speed.get(who) is not None:
+            hours = speed[who] / 3600
+            why.append(f"usually answers in {hours:.0f}h" if hours >= 1 else "usually answers in under an hour")
+        out.append({"login": who, "score": round(s, 3), "why": "; ".join(why)})
+    return sorted(out, key=lambda x: -x["score"])[:k]
+
+
+def knowledge_map(prs: list[dict], now: datetime) -> dict:
+    """Per area: its main author and their share, one-person areas, owners gone quiet, and recent hotspots."""
+    merged = _merged_humans(prs, now)
+    ex = expertise(prs, now)
+    last_seen = {}
+    for p in merged:
+        last_seen[p["au"]] = max(last_seen.get(p["au"], ""), p["mg"])
+    recent = _iso(now - timedelta(days=90))
+    # Busy = how many merged PRs touched the area lately (line counts per file aren't collected).
+    busy, counts = defaultdict(int), defaultdict(int)
+    for p in merged:
+        for a in _areas(p.get("paths")):
+            counts[a] += 1
+            if p["mg"] >= recent:
+                busy[a] += 1
+    areas = []
+    for a, people in ex.items():
+        authors = {who: w for who, w in people.items() if ex.authored[a].get(who)}
+        if not authors:
+            continue
+        owner = max(authors, key=authors.get)
+        share = authors[owner] / sum(authors.values())
+        quiet = (now - _t(last_seen[owner])).days if owner in last_seen else None
+        areas.append({"area": a, "prs": counts[a], "owner": owner, "share": round(share, 3),
+                      "single_owner": share >= SINGLE_OWNER and counts[a] >= 3, "owner_quiet_days": quiet})
+    areas.sort(key=lambda x: -x["prs"])
+    hotspots = sorted(({"area": a, "prs_90d": c, "prs": counts[a]} for a, c in busy.items()), key=lambda x: -x["prs_90d"])
+    return {"areas": areas, "hotspots": hotspots[:10]}
+
+
+def _is_revert(p: dict) -> bool:
+    title, head = (p.get("ti") or "").lower(), (p.get("head") or "").lower()
+    return title.startswith('revert "') or head.startswith("revert-") or "hotfix" in title or "hotfix" in head
+
+
+def delivery(prs: list[dict], releases: list[dict], now: datetime, window_days: int = 90) -> dict:
+    """Release frequency, merge-to-release time and a revert/hotfix-based change failure share (a labelled proxy)."""
+    since = _iso(now - timedelta(days=window_days))
+    rel = sorted((r for r in releases if not r.get("pre") and r.get("at") and r["at"] >= since), key=lambda r: r["at"])
+    merged = [p for p in _merged_humans(prs, now, days=window_days)]
+    if not rel:
+        return {"source": "merges", "releases": 0, "per_week": round(len(merged) / (window_days / 7), 2),
+                "lead_time_median_s": None, "change_fail": None, "recovery_median_s": None}
+    leads = []
+    for p in merged:
+        after = [r["at"] for r in rel if r["at"] >= p["mg"]]
+        if after:
+            leads.append(_secs(p["mg"], after[0]))
+    reverts = sorted(p["mg"] for p in merged if _is_revert(p))
+    failed, recovery = 0, []
+    for i, r in enumerate(rel):
+        nxt = rel[i + 1]["at"] if i + 1 < len(rel) else None
+        limit = min(filter(None, [nxt, _iso(_t(r["at"]) + REVERT_WINDOW)]))
+        if any(r["at"] <= x < limit for x in reverts):
+            failed += 1
+            if nxt:
+                recovery.append(_secs(r["at"], nxt))
+    return {"source": "releases", "releases": len(rel), "per_week": round(len(rel) / (window_days / 7), 2),
+            "lead_time_median_s": _median([x for x in leads if x is not None]),
+            "change_fail": {"failed": failed, "of": len(rel), "rate": round(failed / len(rel), 4), "proxy": True},
+            "recovery_median_s": _median([x for x in recovery if x is not None])}
+
+
+def backtest(prs: list[dict], now: datetime, last: int = 50) -> dict:
+    """For recent reviewed PRs, using only what was known before each opened: was a real reviewer in our top 3?"""
+    people = People(prs)
+    candidates = sorted((p for p in _merged_humans(prs, now)
+                         if p.get("paths") and {e[2] for e in _human_reviews(p) if people.is_insider(e[2])}),
+                        key=lambda p: p["mg"])[-last:]
+    hits = 0
+    for p in candidates:
+        ex = expertise(prs, now, before=p["cr"], people=people)
+        top = {s["login"] for s in suggest_reviewers(p, ex, load={}, owners={})}
+        if top & {e[2] for e in _human_reviews(p) if people.is_insider(e[2])}:
+            hits += 1
+    return {"checked": len(candidates), "hits": hits, "rate": round(hits / len(candidates), 3) if candidates else None}

stackvitals/flow/peers.py

@@ -0,0 +1,90 @@
+from datetime import datetime, timedelta
+from pathlib import Path
+
+from mergescout.bench.ci import tier
+from stackvitals.ci import bench as ci_bench
+from stackvitals.ci.bench import RANKS, _about, _label, _rank, fit
+from stackvitals.core.actors import People
+from stackvitals.flow.analyze import _human_reviews, _iso, _median, _secs, is_bot
+
+FLOW_FILE = "flow-benchmarks.json"
+MIN_PRS = 10
+# Lower is better for waits; a higher merge rate is better; size is neither, so it gets plain words.
+METRICS = [("first_review_hours", "First maintainer review after", "hours", -1),
+           ("merge_hours", "Opened to merged", "hours", -1),
+           ("outsider_merge_rate", "Outside contributors' PRs merged", "share", 1),
+           ("size_lines", "Typical PR size", "lines", 0)]
+
+
+def own_metrics(prs: list[dict], now: datetime, window_days: int = 90, people: People | None = None) -> dict | None:
+    """This repo's numbers with exactly the crawl benchmarks' definitions (PRs opened in the window, people only,
+    maintainers found by what they do)."""
+    since = _iso(now - timedelta(days=window_days))
+    if people is None:
+        people = People(prs)
+    mine = [p for p in prs if p["cr"] >= since and not is_bot(p)]
+    if len(mine) < MIN_PRS:
+        return None
+    reviews, merges, sizes = [], [], []
+    for p in mine:
+        # Like the crawl: the first human review, counted only when a maintainer gave it.
+        humans = _human_reviews(p)
+        first = [humans[0][1]] if humans and people.is_insider(humans[0][2]) else []
+        # Times out of order (clock skew, imported history) are skipped, never divided.
+        if first and (wait := _secs(p["cr"], first[0])) is not None:
+            reviews.append(wait / 3600)
+        if p.get("mg") and (wait := _secs(p["cr"], p["mg"])) is not None:
+            merges.append(wait / 3600)
+        sizes.append((p.get("add") or 0) + (p.get("del") or 0))
+    outsiders = [p for p in mine if people.is_outside(p) and p.get("st") in ("MERGED", "CLOSED")]
+    return {"prs": len(mine), "first_review_hours": _median(reviews), "merge_hours": _median(merges),
+            "outsider_merge_rate": round(sum(p["st"] == "MERGED" for p in outsiders) / len(outsiders), 4) if outsiders else None,
+            "size_lines": _median(sizes)}
+
+
+def _ok(q) -> bool:
+    return isinstance(q, dict) and all(isinstance(q.get(f"p{int(r * 100)}"), (int, float)) for r in RANKS)
+
+
+def valid(data) -> dict | None:
+    # A bad file is ignored, metric by metric, so it can neither break a run nor reach a page.
+    if not isinstance(data, dict) or not isinstance(data.get("groups"), dict):
+        return None
+    groups = {}
+    for key, g in data["groups"].items():
+        if isinstance(key, str) and key.count("|") == 1 and isinstance(g, dict) and isinstance(g.get("repos"), int):
+            kept = {m: g[m] for m, *_ in METRICS if _ok(g.get(m))}
+            if kept:
+                groups[key] = {"repos": g["repos"], **kept}
+    return {**data, "groups": groups} if groups else None
+
+
+def load(token: str | None, cache_dir: Path, client=None) -> dict | None:
+    return ci_bench.load(token, cache_dir, client=client, name=FLOW_FILE, url_env="STACKVITALS_FLOW_BENCH_URL",
+                         validate=valid, default_url=None)
+
+
+def compare(own: dict | None, bench: dict | None, language: str | None, stars: int | None) -> dict | None:
+    """Where this repo's review flow stands among similar public repos: the closest group with enough repos."""
+    bench = valid(bench) if bench else None
+    if not own or not bench:
+        return None
+    groups, size = bench["groups"], tier(stars)
+    key = next((k for k in (f"{language}|{size}", f"{language}|all", f"all|{size}", "all|all") if k in groups), None)
+    if key is None:
+        return None
+    g, metrics = groups[key], []
+    for name, label, kind, better in METRICS:
+        if own.get(name) is None or name not in g:
+            continue
+        r = _rank(own[name], g[name])
+        if better < 0:
+            place = f"slower than about {_about(r)}%" if r >= 0.5 else f"faster than about {_about(1 - r)}%"
+        elif better > 0:
+            place = f"above about {_about(r)}%" if r >= 0.5 else f"below about {_about(1 - r)}%"
+        else:
+            place = f"bigger than about {_about(r)}%" if r >= 0.5 else f"smaller than about {_about(1 - r)}%"
+        metrics.append({"key": name, "name": label, "kind": kind, "you": own[name], "typical": g[name]["p50"],
+                        "place": place, "repos": g[name]["repos"]})
+    return {"group": _label(key), "repos": g["repos"], "built_at": bench.get("built_at"), "metrics": metrics,
+            "fit": fit(key, language, size)} if metrics else None

stackvitals/flow/preflight.py

@@ -0,0 +1,210 @@
+import json
+import re
+from collections import Counter, defaultdict
+from datetime import datetime, timedelta
+
+from mergescout.github.graphql import GraphQLError, RateLimited
+from stackvitals.flow.analyze import INSIDERS, _iso, _pct
+from stackvitals.flow.knowledge import _areas, area
+from stackvitals.flow.store import FlowStore
+
+# What maintainers bring up in reviews, in plain categories. Only the categories are kept, never the text.
+CATEGORIES = {
+    "tests": r"\btests?\b|\btest cases?\b|\bcoverage\b",
+    "docs": r"\bdocs?\b|\bdocumentation\b|\breadme\b|\bdocstrings?\b",
+    "changelog": r"\bchange ?log\b|\bnews (entry|fragment)\b|\brelease notes?\b|\bchangesets?\b|\btowncrier\b",
+    "style": r"\bnit\b|\blint\w*\b|\bformat\w*\b|\btypos?\b|\brename\b|\bnaming\b|\bwhitespace\b|\bstyle\b",
+    "types": r"\btype (hint|annotation)s?\b|\btyping\b|\bmypy\b|\bpyright\b|\btype-?check\w*\b",
+    "compatibility": r"\bbreaking\b|\bbackwards?[- ]compat\w*\b|\bdeprecat\w*\b",
+    "performance": r"\bperformance\b|\bperf\b|\bslow(er)?\b|\ballocat\w*\b|\bbenchmark\w*\b",
+    "scope": r"\bseparate pr\b|\bsplit (this|the|it)\b|\bout of scope\b|\bunrelated\b|\bsmaller prs?\b|\btoo (big|large)\b",
+    "rebase": r"\brebase\b|\bmerge conflicts?\b|\bconflicts?\b",
+    "description": r"\bpr description\b|\bdescription\b|\blink(ed)? (the |an |to the )?issue\b|\bmotivation\b",
+    "security": r"\bsecurity\b|\bvulnerab\w*\b|\binjection\b|\bsanitiz\w*\b",
+}
+PATTERNS = {name: re.compile(rx, re.I) for name, rx in CATEGORIES.items()}
+MIN_SCOPE_PRS = 10
+
+
+def categories(text: str | None) -> set[str]:
+    return {name for name, rx in PATTERNS.items() if text and rx.search(text)}
+
+
+def _by_maintainer(item: dict, author: str | None) -> bool:
+    who = item.get("author") or {}
+    return item.get("authorAssociation") in INSIDERS and who.get("__typename") == "User" and who.get("login") != author
+
+
+def asks_from(node: dict, author: str | None, paths: list[str]) -> list[list[str]]:
+    """[category, area] pairs a PR's maintainers raised: review bodies apply to all its areas, line comments to their file's."""
+    found = set()
+    areas = _areas(paths) or {"(root)"}
+    for review in (node.get("reviews") or {}).get("nodes") or []:
+        if review and _by_maintainer(review, author):
+            found |= {(c, a) for c in categories(review.get("body")) for a in areas}
+    for thread in (node.get("reviewThreads") or {}).get("nodes") or []:
+        where = area(thread.get("path") or "") if thread and thread.get("path") else None
+        for comment in ((thread or {}).get("comments") or {}).get("nodes") or []:
+            if comment and _by_maintainer(comment, author):
+                found |= {(c, a) for c in categories(comment.get("body")) for a in ([where] if where else areas)}
+    return [list(x) for x in sorted(found)]
+
+
+def _query(numbers: list[int]) -> str:
+    parts = "\n".join(
+        f"  p{n}: pullRequest(number:{n}) {{ number\n"
+        "    reviews(first:20) { nodes { body authorAssociation author { __typename login } } }\n"
+        "    reviewThreads(first:30) { nodes { path comments(first:3) { nodes { body authorAssociation author { __typename login } } } } } }"
+        for n in numbers)
+    return f"query($owner:String!,$name:String!) {{ rateLimit {{ cost remaining resetAt }} repository(owner:$owner,name:$name) {{\n{parts}\n}} }}"
+
+
+def _reviewed(p: dict) -> bool:
+    return any(e[0] == "review" and e[4] in INSIDERS and e[2] != p.get("au") for e in p["ev"]) or \
+        any(c[2] in INSIDERS and c[1] != p.get("au") for c in p["cm"])
+
+
+def collect_asks(gql, store: FlowStore, owner: str, name: str, prs: list[dict], batch: int = 20, cap: int = 1000) -> dict:
+    """Read review comments of closed PRs that maintainers commented on, once each, newest first, at most `cap` per run."""
+    done = store.asks()
+    todo = sorted((p for p in prs if p.get("st") != "OPEN" and p["n"] not in done and _reviewed(p)),
+                  key=lambda p: p.get("cl") or "", reverse=True)
+    take, read = todo[:cap], 0
+    for i in range(0, len(take), batch):
+        chunk = take[i:i + batch]
+        try:
+            repo = gql.query(_query([p["n"] for p in chunk]), {"owner": owner, "name": name}).data["repository"]
+        except (GraphQLError, RateLimited):
+            # The hourly limit or a slow page: keep what was read, the next run continues.
+            break
+        store.put_asks({p["n"]: asks_from(repo.get(f"p{p['n']}") or {}, p.get("au"), p.get("paths") or []) for p in chunk})
+        read += len(chunk)
+    return {"read": read, "left": len(todo) - read}
+
+
+def preflight(prs: list[dict], asks: dict[int, list], paths: list[str], now: datetime, months: int = 12) -> dict:
+    """What maintainers usually ask for on PRs touching these paths, with counts; the whole repo when history is thin."""
+    since = _iso(now - timedelta(days=30 * months))
+    read = [p for p in prs if p["n"] in asks and (p.get("cl") or "") >= since]
+    targets = _areas(paths)
+    local = [p for p in read if _areas(p.get("paths")) & targets]
+    scoped = len(local) >= MIN_SCOPE_PRS
+    pool = local if scoped else read
+    counts = Counter()
+    for p in pool:
+        counts.update({cat for cat, a in asks[p["n"]] if not scoped or a in targets})
+    n = len(pool)
+    return {"scope": ", ".join(sorted(targets)) if scoped else "whole repo", "reviewed_prs": n,
+            "asks": [{"ask": cat, "share": round(c / n, 4), "prs": c} for cat, c in counts.most_common()] if n else []}
+
+
+TEST_FILE = re.compile(r"(^|/)(tests?|__tests__|spec|testing)(/|$)|(_test|\.test|\.spec|_spec)\.[a-z]+$|(^|/)test_[^/]+$", re.I)
+CHANGELOG_FILE = re.compile(r"(^|/)(CHANGELOG|CHANGES|HISTORY|NEWS)(\.[a-z]+)?$|(^|/)(changelog\.d|changes|news|\.changeset|changelog)/", re.I)
+DOCS_FILE = re.compile(r"(^|/)docs?/|\.(md|rst|adoc)$", re.I)
+HABIT = 0.6
+
+
+def _kinds(paths: list[str]) -> set[str]:
+    out = set()
+    for path in paths or []:
+        if TEST_FILE.search(path):
+            out.add("tests")
+        if CHANGELOG_FILE.search(path):
+            out.add("changelog")
+        elif DOCS_FILE.search(path):
+            out.add("docs")
+    return out
+
+
+class Habits:
+    """Merged PRs indexed by folder once, so per-PR habits are cheap even with thousands of PRs."""
+
+    def __init__(self, prs: list[dict], now: datetime, months: int = 12):
+        since = _iso(now - timedelta(days=30 * months))
+        merged = [p for p in prs if p.get("mg") and p["mg"] >= since and p.get("paths") and p.get("at") != "Bot"]
+        self.kinds = [_kinds(p["paths"]) for p in merged]
+        self.lines = [(p.get("add") or 0) + (p.get("del") or 0) for p in merged]
+        self.by_area = defaultdict(list)
+        for i, p in enumerate(merged):
+            for a in _areas(p["paths"]):
+                self.by_area[a].append(i)
+        self.cache = {}
+
+    def norms(self, paths: list[str]) -> dict:
+        # Only folders with merged history are named (a PR's own folder names are never echoed back).
+        targets = frozenset(a for a in _areas(paths) if a in self.by_area)
+        if targets not in self.cache:
+            local = sorted({i for a in targets for i in self.by_area[a]})
+            scoped = len(local) >= MIN_SCOPE_PRS
+            pool = local if scoped else range(len(self.lines))
+            n = len(pool)
+            kinds = Counter(k for i in pool for k in self.kinds[i])
+            lines = [self.lines[i] for i in pool]
+            self.cache[targets] = {
+                "scope": ", ".join(sorted(targets)) if scoped else "whole repo", "merged_prs": n,
+                "include": {k: round(kinds[k] / n, 4) if n else 0.0 for k in ("tests", "changelog", "docs")},
+                "size": {"median_lines": _pct(lines, 0.5), "p75_lines": _pct(lines, 0.75), "p90_lines": _pct(lines, 0.9)}}
+        return self.cache[targets]
+
+
+def norms(prs: list[dict], paths: list[str], now: datetime, months: int = 12) -> dict:
+    """What merged PRs touching these paths usually include (tests, changelog, docs) and how big they are."""
+    return Habits(prs, now, months).norms(paths)
+
+
+MAX_FOLDERS = 30
+
+
+def folder_habits(prs: list[dict], asks: dict[int, list], now: datetime, index: Habits | None = None, months: int = 12,
+                  cap: int = MAX_FOLDERS) -> list[dict]:
+    """Per folder with enough merged history (busiest first): the norms and review asks the CLI's preflight gives for it."""
+    index = index or Habits(prs, now, months)
+    out = []
+    for name, idx in sorted(index.by_area.items(), key=lambda kv: (-len(kv[1]), kv[0])):
+        if len(idx) < MIN_SCOPE_PRS or len(out) >= cap:
+            break
+        # A file inside the folder: the same lookup as a planned PR touching only that folder.
+        probe = "_" if name == "(root)" else f"{name}/_"
+        h = index.norms([probe])
+        asked = preflight(prs, asks, [probe], now, months)
+        scoped = asked["scope"] != "whole repo"
+        out.append({"area": name, "merged_prs": h["merged_prs"], "include": h["include"], "size": h["size"],
+                    "reviewed_prs": asked["reviewed_prs"] if scoped else 0,
+                    "asks": [a for a in asked["asks"] if a["share"] >= 0.05 and a["prs"] >= 3][:3] if scoped else []})
+    return out
+
+
+def gaps(pr: dict, habits: dict) -> list[dict]:
+    """Where a PR differs from what merged PRs here usually do: missing tests/changelog/docs, or unusually big."""
+    if not habits["merged_prs"]:
+        return []
+    have, out = _kinds(pr.get("paths")), []
+    # Only the first 50 file names are read; a bigger PR may well include what we couldn't see.
+    complete = not pr.get("files") or pr["files"] <= len(pr.get("paths") or [])
+    labels = {"tests": "test files", "changelog": "a changelog entry", "docs": "docs"}
+    for kind in ("tests", "changelog", "docs") if complete else ():
+        share = habits["include"][kind]
+        if share >= HABIT and kind not in have:
+            out.append({"what": kind, "why": f"{share:.0%} of merged PRs in {habits['scope']} include {labels[kind]}"})
+    lines = (pr.get("add") or 0) + (pr.get("del") or 0)
+    p90 = habits["size"]["p90_lines"]
+    if p90 is not None and lines > p90:
+        out.append({"what": "size", "why": f"{lines} lines; 90% of merged PRs in {habits['scope']} have at most {p90}"})
+    return out
+
+
+def load_asks(path) -> dict[int, list]:
+    out = {}
+    try:
+        lines = path.read_text(encoding="utf-8").splitlines()
+    except OSError:
+        return out
+    for line in lines:
+        # A line cut short by an interrupted write is skipped, never fatal.
+        try:
+            rec = json.loads(line) if line.strip() else None
+        except ValueError:
+            continue
+        if isinstance(rec, dict) and "n" in rec:
+            out[rec["n"]] = rec.get("a") or []
+    return out