stackvitals 0.1.0__py3-none-any.whl

stackvitals/flow/report.py

@@ -0,0 +1,253 @@
+import html
+
+from stackvitals import shell
+from stackvitals.ci.bench import PEERS_ARE
+from stackvitals.ci.report import _md as _md_ci
+
+QUEUE_ROWS = 15
+STAGE_NAMES = [("pickup", "Waiting for first review"), ("review", "First to final review"),
+               ("merge", "Approved to merged"), ("cycle", "Ready to merged")]
+
+
+def _span(secs) -> str:
+    if secs is None:
+        return "-"
+    secs = int(secs)
+    if secs >= 86400:
+        return f"{secs // 86400}d {secs % 86400 // 3600}h"
+    if secs >= 3600:
+        return f"{secs // 3600}h {secs % 3600 // 60}m"
+    return f"{secs // 60}m"
+
+
+def _md(text) -> str:
+    # PR titles and logins come from anyone who opens a PR: HTML and Markdown are both neutralised.
+    return _md_ci(html.escape(str(text), quote=False))
+
+
+def _pct(x) -> str:
+    return "-" if x is None else f"{x:.0%}"
+
+
+def _waiting_on(q: dict) -> str:
+    if q["actor"] == "reviewers":
+        return ", ".join(q["who"])
+    if q["actor"] == "author":
+        return f"the author ({q['author']})"
+    return "maintainers"
+
+
+def _suggested(q: dict) -> str:
+    return "; ".join(f"{x['login']} ({x['why']})" for x in q.get("suggested") or [])
+
+
+def _knowledge_lines(r: dict) -> list[str]:
+    k = r.get("knowledge")
+    if not k:
+        return []
+    one = [a for a in k["areas"] if a["single_owner"]]
+    quiet = [a for a in k["areas"] if (a.get("owner_quiet_days") or 0) > 90]
+    out = []
+    if one:
+        out.append("Known mostly by one person: " + ", ".join(
+            f"{a['area']} ({a['owner'] + ', ' if a.get('owner') else ''}{a['share']:.0%} of merged PRs)" for a in one[:8]))
+    if quiet:
+        out.append("Main author gone quiet: " + ", ".join(
+            f"{a['area']} ({a['owner'] + ', ' if a.get('owner') else ''}no merged PR for {a['owner_quiet_days']} days)" for a in quiet[:8]))
+    if k["hotspots"]:
+        out.append("Busiest areas, last 90 days: " + ", ".join(
+            f"{h['area']} ({h['prs_90d']} merged PRs)" for h in k["hotspots"][:5]))
+    return out
+
+
+def stage_note(pages) -> str:
+    """How first review here differs from Delivery's pickup, which also counts a first comment (LinearB's definition)."""
+    where = "<a href='delivery.html'>Delivery's pickup</a>" if "delivery.html" in pages else "Delivery's pickup"
+    return (f"<p class=mut>First review counts reviews only. {where} also counts a first comment by someone else "
+            "(LinearB's definition), so it is usually shorter.</p>")
+
+
+def _delivery_lines(r: dict) -> list[str]:
+    d = r.get("delivery")
+    if not d:
+        return []
+    if d["source"] == "merges":
+        return [f"No releases in the last {r['window_days']} days; {d['per_week']} merged PRs a week."]
+    cf = d["change_fail"]
+    later = f"; the next release came {_span(d['recovery_median_s'])} later (median)" if d["recovery_median_s"] else ""
+    return [f"{d['releases']} releases in {r['window_days']} days ({d['per_week']} a week); merge to release takes "
+            f"{_span(d['lead_time_median_s'])} (median).",
+            f"{cf['failed']} of {cf['of']} releases were followed by a revert or hotfix within 7 days "
+            f"(a proxy; the Delivery page counts change failure the way DevLake does, from incidents){later}."]
+
+
+def _gaps(q: dict) -> str:
+    return "; ".join(g["why"] for g in q.get("gaps") or [])
+
+
+def _habit_lines(r: dict) -> list[str]:
+    h = r.get("habits")
+    if not h or not h["merged_prs"]:
+        return []
+    inc, size = h["include"], h["size"]
+    out = [f"Merged PRs ({h['scope']}, n={h['merged_prs']}) include tests {inc['tests']:.0%}, a changelog entry "
+           f"{inc['changelog']:.0%}, docs {inc['docs']:.0%}; size {size['median_lines']} lines median, "
+           f"90% at most {size['p90_lines']} lines."]
+    if h.get("asks"):
+        out.append("Maintainers most often ask for: " + ", ".join(f"{a['ask']} ({a['share']:.0%} of reviewed PRs)" for a in h["asks"][:5]))
+    return out
+
+
+def _peer_lines(r: dict) -> tuple[str, list[str]] | None:
+    c = r.get("peers")
+    if not c:
+        return None
+    fmt = {"hours": lambda v: _span(v * 3600), "share": lambda v: f"{v:.0%}", "lines": lambda v: f"{v:.0f} lines"}
+    lines = [f"{m['name']}: {fmt[m['kind']](m['you'])} (typical {fmt[m['kind']](m['typical'])}), {m['place']}"
+             for m in c["metrics"]]
+    loose = f", {c['fit']}" if c.get("fit") else ""
+    return (f"Compared with {int(c['repos'])} {c['group']} (PRs opened in the last 90 days{loose}); the group is "
+            f"{PEERS_ARE}, so it may not resemble a private team's work:"), lines
+
+
+def _head(r: dict, repo: str) -> str:
+    return (f"{repo}: review flow over the last {r['window_days']} days ({r['merged']} merged, {r['closed_unmerged']} closed "
+            f"without merging, {r['open']} open of which {r['drafts']} drafts; bot PRs left out: {r['bots']['open']} open, "
+            f"{r['bots']['closed']} closed)")
+
+
+def _groups(r: dict) -> list[str]:
+    out = []
+    for name, label in (("insider", "Insiders"), ("outsider", "Outside contributors")):
+        g = r["groups"][name]
+        out.append(f"{label}: {_pct(g['merge_rate'])} merged, first review after {_span(g['pickup']['median_s'])} "
+                   f"(median, n={g['pickup']['n']})")
+    return out
+
+
+def render_text(r: dict, repo: str) -> str:
+    out = [_head(r, repo), ""]
+    if peers := _peer_lines(r):
+        out += [peers[0], *[f"  {line}" for line in peers[1]], ""]
+    out += ["Median time (90th percentile), GitHub's review stages:"]
+    out += [f"  {label}: {_span(r['stages'][k]['median_s'])} ({_span(r['stages'][k]['p90_s'])}), n={r['stages'][k]['n']}"
+            for k, label in STAGE_NAMES]
+    out += [f"  Merged without any human review: {r['merged_without_review']}", "", *_groups(r), "",
+            "Who acts next (overdue first; overdue = longer than usual for this repo, and over a day):"]
+    live = [q for q in r["queue"] if not q["stale"]]
+    for q in live[:QUEUE_ROWS]:
+        flag = "  [overdue]" if q["slow"] else ""
+        out.append(f"  #{q['n']} \"{q['title']}\" by {q['author']}{' (outsider)' if q['outsider'] else ''}: waiting on "
+                   f"{_waiting_on(q)} for about {_span(q['waiting_s'])} (usual {_span(q['usual_s'])}) - {q['why']}{flag}")
+        if q.get("suggested"):
+            out.append(f"      suggested reviewers: {_suggested(q)}")
+        if q.get("gaps"):
+            out.append(f"      unlike merged PRs here: {_gaps(q)}")
+    if not live:
+        out.append("  nothing waiting")
+    if r["stale"]:
+        out.append(f"  ... and {r['stale']} stale PRs waiting over 30 days")
+    if r["load"]:
+        out += ["", "Open review requests: " + ", ".join(f"{x['reviewer']} {x['open_requests']}" for x in r["load"][:10])]
+    if habits := _habit_lines(r):
+        out += ["", "What merged PRs here usually include:", *[f"  {line}" for line in habits]]
+    if knowledge := _knowledge_lines(r):
+        out += ["", "Knowledge map:", *[f"  {line}" for line in knowledge]]
+    if ship := _delivery_lines(r):
+        out += ["", "Delivery:", *[f"  {line}" for line in ship]]
+    if r.get("responders"):
+        out += ["", "Answer times (private: shown only in this local view, never on the dashboard):"]
+        out += [f"  {x['reviewer']}: {_span(x['median_s'])} median (n={x['n']})" for x in r["responders"][:10]]
+    return "\n".join(out)
+
+
+def render_markdown(r: dict, repo: str) -> str:
+    out = [f"### StackVitals Team Flow: {_md(repo)}", _md(_head(r, repo).split(": ", 1)[1]), ""]
+    if peers := _peer_lines(r):
+        out += [_md(peers[0]), *[f"- {_md(line)}" for line in peers[1]], ""]
+    out += ["| Stage | Median | 90th percentile | PRs |", "|---|---|---|---|"]
+    out += [f"| {label} | {_span(r['stages'][k]['median_s'])} | {_span(r['stages'][k]['p90_s'])} | {r['stages'][k]['n']} |"
+            for k, label in STAGE_NAMES]
+    out += ["", *[_md(line) for line in _groups(r)], "", "| PR | Waiting on | For | Why |", "|---|---|---|---|"]
+    live = [q for q in r["queue"] if not q["stale"]]
+    out += [f"| #{q['n']} {_md(q['title'])} | {_md(_waiting_on(q))} | about {_span(q['waiting_s'])}{' (overdue)' if q['slow'] else ''} "
+            f"| {_md(q['why'])}{(' - suggested: ' + _md(', '.join(x['login'] for x in q['suggested']))) if q.get('suggested') else ''}"
+            f"{(' - ' + _md(_gaps(q))) if q.get('gaps') else ''} |"
+            for q in live[:QUEUE_ROWS]] or ["| - | nothing waiting | | |"]
+    if r["stale"]:
+        out.append(f"\n{r['stale']} stale PRs are waiting over 30 days.")
+    for title, lines in (("What merged PRs here usually include", _habit_lines(r)), ("Knowledge map", _knowledge_lines(r)),
+                         ("Delivery", _delivery_lines(r))):
+        if lines:
+            out += ["", f"**{title}**", *[f"- {_md(line)}" for line in lines]]
+    return "\n".join(out)
+
+
+def _hours(secs) -> float | None:
+    return None if secs is None else round(secs / 3600, 1)
+
+
+def _trends(r: dict) -> str:
+    months = r.get("monthly") or []
+    if len(months) < 2:
+        return ""
+    labels = [m["month"] for m in months]
+    counts = shell.chart("PRs opened and merged per month", "Month", labels,
+                         [("Opened", [m["opened"] for m in months]), ("Merged", [m["merged"] for m in months])], y="PRs", kind="bar")
+    times = shell.chart("Review times per month (median)", "Month", labels,
+                        [("First review", [_hours(m.get("pickup_median_s")) for m in months]),
+                         ("Ready to merged", [_hours(m.get("cycle_median_s")) for m in months])], y="hours", fmt=lambda v: f"{v:g} h")
+    return ("<h2>Trends</h2><p class=mut>The last 12 months; review times are for PRs merged that month, bots left out.</p>"
+            + shell.charts([counts, times]))
+
+
+def render_html(r: dict, repo: str, pages=()) -> str:
+    e = html.escape
+    link = lambda n: f"<a href='https://github.com/{e(repo)}/pull/{int(n)}'>#{int(n)}</a>"  # noqa: E731
+    body = []
+    cards = [("Open PRs", r["open"]), ("Merged", r["merged"]), ("First review (median)", _span(r["stages"]["pickup"]["median_s"])),
+             ("Ready to merged (median)", _span(r["stages"]["cycle"]["median_s"])), ("Stale backlog", f"{r['stale']} stale"),
+             ("Merged without review", r["merged_without_review"])]
+    body.append("<div class=cards>" + "".join(f"<div class=card><span class=mut>{e(k)}</span><b>{e(str(v))}</b></div>"
+                                              for k, v in cards) + "</div>")
+    if peers := _peer_lines(r):
+        body.append(f"<h2>Compared with similar repos</h2><p class=mut>{e(peers[0])}</p><ul>"
+                    + "".join(f"<li>{e(line)}</li>" for line in peers[1]) + "</ul>")
+    body.append("<h2>Who acts next</h2><p class=mut>Overdue = waiting longer than usual for this repo, and over a day.</p>")
+    live = [q for q in r["queue"] if not q["stale"]]
+    if live:
+        body.append("<div class=tbl><table class=find><tr><th>PR</th><th>Waiting on</th><th>For</th><th>Usual</th><th>Why</th></tr>")
+        for q in live[:QUEUE_ROWS * 2]:
+            tag = " <span class='tag flaky'>overdue</span>" if q["slow"] else ""
+            body.append(f"<tr><td>{link(q['n'])} {e(q['title'] or '')}<br><span class=mut>by {e(q['author'] or '?')}"
+                        f"{' (outsider)' if q['outsider'] else ''}</span></td><td>{e(_waiting_on(q))}</td>"
+                        f"<td>about {_span(q['waiting_s'])}{tag}</td><td>{_span(q['usual_s'])}</td><td>{e(q['why'])}"
+                        + (f"<br><span class=mut>Suggested: {e(_suggested(q))}</span>" if q.get("suggested") else "")
+                        + (f"<br><span class=mut>Unlike merged PRs here: {e(_gaps(q))}</span>" if q.get("gaps") else "")
+                        + "</td></tr>")
+        body.append("</table></div>")
+    else:
+        body.append("<p>Nothing waiting.</p>")
+    if r["stale"]:
+        body.append(f"<p class=mut>{r['stale']} stale PRs are waiting over 30 days.</p>")
+    if r.get("excluded"):
+        n = r["excluded"]
+        body.append(f"<p class=mut>{n} pull request{'' if n == 1 else 's'} left out by the team (the "
+                    "<code>stackvitals: exclude</code> label or <code>/stackvitals exclude</code>).</p>")
+    body.append(_trends(r))
+    body.append("<h2>Review stages</h2><div class=tbl><table><tr><th>Stage</th><th>Median</th><th>90th percentile</th><th>PRs</th></tr>"
+                + "".join(f"<tr><td>{e(label)}</td><td>{_span(r['stages'][k]['median_s'])}</td><td>{_span(r['stages'][k]['p90_s'])}</td>"
+                          f"<td>{r['stages'][k]['n']}</td></tr>" for k, label in STAGE_NAMES) + "</table></div>")
+    body.append(stage_note(pages))
+    body.append("<h2>Insiders and outside contributors</h2><ul>" + "".join(f"<li>{e(line)}</li>" for line in _groups(r)) + "</ul>")
+    for title, lines in (("What merged PRs here usually include", _habit_lines(r)), ("Knowledge map", _knowledge_lines(r)),
+                         ("Delivery", _delivery_lines(r))):
+        if lines:
+            body.append(f"<h2>{title}</h2><ul>" + "".join(f"<li>{e(line)}</li>" for line in lines) + "</ul>")
+    if r["load"]:
+        body.append("<h2>Open review requests</h2><ul>" + "".join(
+            f"<li>{e(x['reviewer'])}: {x['open_requests']} open</li>" for x in r["load"][:20]) + "</ul>")
+    body.append("<footer class=mut>Stages follow GitHub's own boundaries; bots, authors and Copilot reviews are left out. "
+                "History is kept in this repo's <code>stackvitals-data</code> branch. Made by StackVitals.</footer>")
+    return shell.page("Team Flow", repo, "flow.html", f"Last {r['window_days']} days, updated {e(r['generated'])}",
+                      "".join(body), pages)

stackvitals/flow/rules.py

@@ -0,0 +1,135 @@
+"""PR rules as code: each [[rules]] entry in the settings file is a `when` (author kind, paths, size, change risk, labels,
+a first-time author) and a `then` (a label, reviewers, one comment). Matching open pull requests get each action once,
+planned before the data branch's commit and sent after it like the rest of team automation; a dry-run rule only says
+what it matched, and every rule can be replayed on past pull requests to see what it would have done."""
+import re
+
+from stackvitals.codeowners import matches as path_matches
+from stackvitals.core.ai_outcomes import mode
+from stackvitals.flow.analyze import is_bot
+
+AUTHORS = ("agent", "person", "bot")
+RISKS = ("medium", "high")
+WHEN = {"author", "paths", "lines_over", "risk", "labels", "first_time"}
+THEN = {"label", "reviewers", "comment"}
+LOGIN = re.compile(r"@?[A-Za-z0-9][A-Za-z0-9-]{0,38}(/[A-Za-z0-9._-]{1,100})?")
+MAX_COMMENT = 1000
+MARK = "<!-- stackvitals-rule -->"
+
+
+def check_rule(rule) -> str | None:
+    """What is wrong with one [[rules]] entry, in words, or None when it can be used."""
+    if not isinstance(rule, dict) or not isinstance(rule.get("name"), str) or not rule["name"].strip():
+        return "each rule needs a name"
+    when, then = rule.get("when") or {}, rule.get("then")
+    if not isinstance(when, dict) or set(when) - WHEN:
+        return f"when takes {', '.join(sorted(WHEN))}"
+    if "author" in when and when["author"] not in AUTHORS:
+        return f"author is one of {', '.join(AUTHORS)}"
+    if "risk" in when and when["risk"] not in RISKS:
+        return "risk is medium or high"
+    if "paths" in when and not (isinstance(when["paths"], list) and all(isinstance(x, str) and x for x in when["paths"])):
+        return "paths is a list of CODEOWNERS-style patterns"
+    if "labels" in when and not (isinstance(when["labels"], list) and all(isinstance(x, str) for x in when["labels"])):
+        return "labels is a list of label names"
+    if "lines_over" in when and not (isinstance(when["lines_over"], int) and when["lines_over"] >= 0):
+        return "lines_over is a number of lines"
+    if "first_time" in when and not isinstance(when["first_time"], bool):
+        return "first_time is true or false"
+    if not isinstance(then, dict) or not then or set(then) - THEN:
+        return f"then needs one or more of {', '.join(sorted(THEN))}"
+    if "label" in then and not (isinstance(then["label"], str) and 0 < len(then["label"]) <= 50):
+        return "label is a label name"
+    if "reviewers" in then and not (isinstance(then["reviewers"], list) and then["reviewers"]
+                                    and all(isinstance(x, str) and LOGIN.fullmatch(x) for x in then["reviewers"])):
+        return "reviewers is a list of logins or @org/team names"
+    if "comment" in then and not (isinstance(then["comment"], str) and 0 < len(then["comment"]) <= MAX_COMMENT):
+        return f"comment is text up to {MAX_COMMENT} characters"
+    if "dry_run" in rule and not isinstance(rule["dry_run"], bool):
+        return "dry_run is true or false"
+    return None
+
+
+def _kind(p: dict) -> str:
+    if mode(p) == "agent":
+        return "agent"
+    return "bot" if is_bot(p) else "person"
+
+
+def matches(p: dict, when: dict, ctx: dict) -> bool:
+    if "author" in when and _kind(p) != when["author"]:
+        return False
+    if "paths" in when and not any(path_matches(pat, path) for path in p.get("paths") or [] for pat in when["paths"]):
+        return False
+    if "lines_over" in when and (p.get("add") or 0) + (p.get("del") or 0) <= when["lines_over"]:
+        return False
+    if "risk" in when:
+        # "medium" matches medium and high; a PR without a risk score never matches a risk condition.
+        level = (ctx.get("risk") or {}).get(p.get("n"))
+        if level not in RISKS or RISKS.index(level) < RISKS.index(when["risk"]):
+            return False
+    if "labels" in when and not set(when["labels"]) & set(p.get("lb") or []):
+        return False
+    if "first_time" in when and ((p.get("au") or "").lower() in {x.lower() for x in ctx.get("first_time") or ()}) != when["first_time"]:
+        return False
+    return True
+
+
+def plan(open_prs: list[dict], rules: list[dict], ctx: dict, state: dict) -> tuple[list[dict], dict, dict]:
+    """(writes to make, the state to keep, {rule name: matched PR numbers}); each rule's action on a PR happens once."""
+    done = list(state.get("rules_done") or [])
+    seen, acts, report = set(done), [], {}
+    for rule in rules:
+        hit = [p for p in open_prs if not p.get("draft") and matches(p, rule.get("when") or {}, ctx)]
+        report[rule["name"]] = [p["n"] for p in hit]
+        if rule.get("dry_run"):
+            continue
+        then = rule["then"]
+        for p in hit:
+            # Each action once: a label someone removed afterwards stays removed.
+            label = {"labels": [then["label"]]} if "label" in then and then["label"] not in (p.get("lb") or []) else None
+            for kind, body in (("label", label), ("review", _reviewers(then.get("reviewers"), p)),
+                               ("comment", _comment(then.get("comment"), rule))):
+                key = f"{rule['name']}:{p['n']}:{kind}"
+                if body and key not in seen:
+                    acts.append({"kind": kind, "n": p["n"], "body": body})
+                    seen.add(key)
+                    done.append(key)
+    return acts, {"rules_done": done[-2000:]}, report
+
+
+def _reviewers(names, p: dict) -> dict | None:
+    if not names:
+        return None
+    people = [x.lstrip("@") for x in names if "/" not in x and x.lstrip("@").lower() != (p.get("au") or "").lower()]
+    teams = [x.split("/", 1)[1] for x in names if "/" in x]
+    return {"reviewers": people, "team_reviewers": teams} if people or teams else None
+
+
+def _comment(text, rule: dict) -> dict | None:
+    return {"body": f"{text}\n\n_From the StackVitals rule \"{rule['name']}\"._\n{MARK}"} if text else None
+
+
+def replay(rules: list[dict], past: list[dict], ctx: dict, examples: int = 10) -> dict:
+    """What each rule would have matched among past pull requests (newest last), so a rule is checked before it acts."""
+    out = {}
+    for rule in rules:
+        hit = [p["n"] for p in past if matches(p, rule.get("when") or {}, ctx)]
+        out[rule["name"]] = {"matched": len(hit), "of": len(past), "examples": hit[:examples]}
+    return out
+
+
+def context(flow: dict, rules: list[dict]) -> dict:
+    """The signals rules can ask about, worked out only when a rule needs them: change risk (shown only where the
+    repo's own backtest says it works) and authors with no merged pull request yet."""
+    ctx = {}
+    wants = {k for r in rules for k in (r.get("when") or {})}
+    closed = flow.get("closed") or []
+    if "risk" in wants:
+        from stackvitals.core import change_risk
+        found = change_risk.assess(closed, change_risk.follow_up_fixes(closed), flow.get("open") or [])
+        ctx["risk"] = {s["pr"]: s["level"] for s in found.get("open") or []}
+    if "first_time" in wants:
+        merged = {(p.get("au") or "").lower() for p in closed if p.get("mg")}
+        ctx["first_time"] = {p.get("au") for p in flow.get("open") or [] if (p.get("au") or "").lower() not in merged}
+    return ctx

stackvitals/flow/store.py

@@ -0,0 +1,108 @@
+import gzip
+import json
+import os
+from pathlib import Path
+
+SCHEMA = "1"
+
+
+class FlowStore:
+    # Closed PRs never change, so they are appended once by close day; open PRs live in one small snapshot.
+    def __init__(self, root: Path, folder: str = "flow"):
+        # Community keeps its issues the same way, in its own folder.
+        self.dir = Path(root) / folder
+        self.root = Path(root)
+        self.closed = self.dir / "closed"
+        self.closed.mkdir(parents=True, exist_ok=True)
+        version = self.dir / "VERSION"
+        if not version.exists():
+            version.write_text(SCHEMA + "\n", encoding="utf-8")
+
+    def put(self, records: list[dict], day: str) -> None:
+        if not records:
+            return
+        with (self.closed / f"{day}.ndjson").open("a", encoding="utf-8", newline="\n") as f:
+            for r in records:
+                f.write(json.dumps(r, separators=(",", ":"), ensure_ascii=False) + "\n")
+
+    def open(self) -> dict[int, dict]:
+        try:
+            return {r["n"]: r for r in json.loads((self.dir / "open.json").read_text(encoding="utf-8"))}
+        except (OSError, ValueError):
+            return {}
+
+    def save_open(self, prs: dict[int, dict]) -> None:
+        self._atomic(self.dir / "open.json", json.dumps(sorted(prs.values(), key=lambda r: r["n"]),
+                                                        separators=(",", ":"), ensure_ascii=False))
+
+    def _lines(self):
+        for path in sorted(self.closed.iterdir()):
+            if path.name.endswith(".ndjson.gz"):
+                text = gzip.decompress(path.read_bytes()).decode("utf-8")
+            elif path.name.endswith(".ndjson"):
+                text = path.read_text(encoding="utf-8")
+            else:
+                continue
+            for line in text.splitlines():
+                if line.strip():
+                    yield json.loads(line)
+
+    def load(self) -> dict[str, list[dict]]:
+        # The newest copy of a closed PR wins; a PR that was reopened counts as open only.
+        opened = self.open()
+        closed = {}
+        for r in self._lines():
+            if r["n"] not in closed or r.get("f", "") >= closed[r["n"]].get("f", ""):
+                closed[r["n"]] = r
+        return {"open": list(opened.values()), "closed": [r for n, r in closed.items() if n not in opened]}
+
+    def asks(self) -> dict[int, list]:
+        # What maintainers asked for per closed PR, as categories only (never the review text).
+        from stackvitals.flow.preflight import load_asks
+        return load_asks(self.dir / "asks.ndjson")
+
+    def put_asks(self, asks: dict[int, list]) -> None:
+        with (self.dir / "asks.ndjson").open("a", encoding="utf-8", newline="\n") as f:
+            for n, a in asks.items():
+                f.write(json.dumps({"n": n, "a": a}, separators=(",", ":")) + "\n")
+
+    def repo(self) -> dict:
+        """What the repo query saved, plus deployments and incidents recorded from other tools (repository_dispatch)."""
+        from stackvitals.core import recorded
+        try:
+            info = json.loads((self.dir / "repo.json").read_text(encoding="utf-8"))
+        except (OSError, ValueError):
+            info = {}
+        got = recorded.load(self.root)
+        return {**info, "recorded": got} if got["deployments"] or got["incidents"] else info
+
+    def save_repo(self, info: dict) -> None:
+        self._atomic(self.dir / "repo.json", json.dumps(info, indent=1, sort_keys=True))
+
+    def state(self) -> dict:
+        try:
+            return json.loads((self.dir / "state.json").read_text(encoding="utf-8"))
+        except (OSError, ValueError):
+            return {}
+
+    def save_state(self, state: dict) -> None:
+        self._atomic(self.dir / "state.json", json.dumps(state, indent=1, sort_keys=True))
+
+    def compress(self, today: str) -> None:
+        # Past days are gzipped right away, merged with any earlier gz of the same day.
+        for path in sorted(self.closed.glob("*.ndjson")):
+            day = path.name[:10]
+            if day >= today:
+                continue
+            packed = self.closed / f"{day}.ndjson.gz"
+            old = gzip.decompress(packed.read_bytes()) if packed.exists() else b""
+            tmp = packed.with_suffix(".tmp")
+            tmp.write_bytes(gzip.compress(old + path.read_bytes(), mtime=0))
+            os.replace(tmp, packed)
+            path.unlink()
+
+    @staticmethod
+    def _atomic(path: Path, text: str) -> None:
+        tmp = path.with_suffix(".tmp")
+        tmp.write_text(text, encoding="utf-8", newline="\n")
+        os.replace(tmp, path)

stackvitals/flow/survival.py

@@ -0,0 +1,84 @@
+"""Code survival: of the lines a squash-merged pull request added, how many are still in the default branch 30 and 90
+days later (git blame, read through GitHub's API), split by people's and agents' PRs. DevLake shows line age per file;
+this asks the question teams adopting agents have: does agent code last?"""
+import json
+import re
+from datetime import datetime, timedelta
+from pathlib import Path
+
+from mergescout.github.graphql import GraphQLError
+
+AGES = (30, 90)
+PER_RUN = 12
+FILES = 8
+FILE = "survival.json"
+HEX = re.compile(r"[0-9a-f]{40}")
+
+
+def _t(s: str) -> datetime:
+    return datetime.fromisoformat(s.replace("Z", "+00:00"))
+
+
+def due(prs: list[dict], saved: dict, now: datetime, per_run: int = PER_RUN) -> list[tuple[dict, int]]:
+    """(PR, age) still to measure: merged at least `age` days ago (but within 30 days past it), newest first."""
+    out = []
+    for p in sorted((p for p in prs if p.get("mg") and HEX.fullmatch(p.get("mc") or "") and p.get("add")), key=lambda p: p["mg"], reverse=True):
+        days = (now - _t(p["mg"])).days
+        for age in AGES:
+            if age <= days < age + 30 and str(age) not in (saved.get(str(p["n"])) or {}):
+                out.append((p, age))
+    return out[:per_run]
+
+
+def _query(mc: str, paths: list[str]) -> str:
+    parts = " ".join(f'f{i}: blame(path: {json.dumps(path)}) {{ ranges {{ startingLine endingLine commit {{ oid }} }} }}'
+                     for i, path in enumerate(paths))
+    return ("query($owner:String!,$name:String!) { rateLimit { cost remaining resetAt } repository(owner:$owner,name:$name) { "
+            f'm: object(oid:"{mc}") {{ ... on Commit {{ parents {{ totalCount }} }} }} '
+            f'h: defaultBranchRef {{ target {{ ... on Commit {{ {parts} }} }} }} }} }}')
+
+
+def measure_one(gql, owner: str, name: str, p: dict) -> dict | None:
+    """{"kept": lines still blamed on its merge commit, "added": lines it added}; None when it wasn't squash-merged
+    (its lines belong to other commits then) or the API refused."""
+    paths = [x for x in (p.get("paths") or [])][:FILES]
+    if not paths:
+        return None
+    try:
+        repo = gql.query(_query(p["mc"], paths), {"owner": owner, "name": name}).data["repository"]
+    except GraphQLError:
+        return None
+    if ((repo.get("m") or {}).get("parents") or {}).get("totalCount") != 1:
+        return None
+    target = ((repo.get("h") or {}).get("target")) or {}
+    kept = sum(r["endingLine"] - r["startingLine"] + 1 for i in range(len(paths))
+               for r in ((target.get(f"f{i}") or {}).get("ranges") or []) if (r.get("commit") or {}).get("oid") == p["mc"])
+    return {"kept": kept, "added": p["add"], "files": len(paths), "all_files": len(p.get("paths") or []) <= FILES}
+
+
+def collect(gql, store_dir: Path, owner: str, name: str, prs: list[dict], now: datetime) -> dict:
+    """Measures the PRs due this run and keeps every answer: {pr: {"30": {...}, "90": {...}}}."""
+    path = Path(store_dir) / FILE
+    try:
+        saved = json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, ValueError):
+        saved = {}
+    for p, age in due(prs, saved, now):
+        got = measure_one(gql, owner, name, p)
+        saved.setdefault(str(p["n"]), {})[str(age)] = got or {"skipped": True}
+    path.write_text(json.dumps(saved, sort_keys=True, separators=(",", ":")), encoding="utf-8")
+    return saved
+
+
+def summary(saved: dict, prs: list[dict], kind_of) -> dict | None:
+    """Share of added lines still there at each age, for people's PRs and agents' PRs (kind_of(pr) -> "agent" or other)."""
+    by_n = {str(p["n"]): p for p in prs}
+    out = {}
+    for age in AGES:
+        for who in ("people", "agents"):
+            rows = [v[str(age)] for n, v in saved.items() if str(age) in v and not v[str(age)].get("skipped") and n in by_n
+                    and v[str(age)].get("all_files") and (kind_of(by_n[n]) == "agent") == (who == "agents")]
+            added = sum(r["added"] for r in rows)
+            if rows and added:
+                out.setdefault(str(age), {})[who] = {"prs": len(rows), "share": round(min(1.0, sum(r["kept"] for r in rows) / added), 4)}
+    return out or None

stackvitals/flow/triage.py

@@ -0,0 +1,62 @@
+"""Private triage of open outside PRs for maintainers: the chance of a merge within 30 days and why.
+Computed on demand with the maintainer's own token; never stored, never on a page, never on a PR (user decision D1)."""
+from datetime import datetime, timedelta
+
+from mergescout.signals.triage_math import FEATURES, buckets, explain, predict
+from stackvitals.core.actors import People
+from stackvitals.flow.analyze import _iso
+
+MODEL_FILE = "triage-model.json"
+NOTE = ("Private: computed on demand for maintainers with your own token; never shown publicly, stored or posted. "
+        "It scores the PR's chances, not the person.")
+MIN_REPO_PRS = 10
+MIN_MODEL_PRS = 1000
+
+
+def _repo_rate(prs: list[dict], opened: str, skip: int | None = None, people: People | None = None) -> float | None:
+    # Same as training: outside PRs opened 30 to 395 days before this one, merged within 30 days of opening.
+    at = datetime.fromisoformat(opened.replace("Z", "+00:00"))
+    lo, hi = _iso(at - timedelta(days=395)), _iso(at - timedelta(days=30))
+    if people is None:
+        people = People(prs)
+    pool = [p for p in prs if p["n"] != skip and people.is_outside(p) and lo <= p["cr"] <= hi]
+    if len(pool) < MIN_REPO_PRS:
+        return None
+    merged = sum(1 for p in pool if p.get("mg") and p["mg"] <= _iso(datetime.fromisoformat(p["cr"].replace("Z", "+00:00"))
+                                                                    + timedelta(days=30)))
+    return merged / len(pool)
+
+
+def features(target: dict, prs: list[dict], now: datetime, people: People | None = None) -> dict:
+    """Only what this repo knew when the PR was opened, and the PR itself (the same rules as training)."""
+    au = target.get("au")
+    earlier = [p for p in prs if au and p.get("au") == au and p["cr"] < target["cr"]] if au else []
+    here = sum(1 for p in earlier if p.get("mg") and p["mg"] < target["cr"])
+    # A first contribution is judged from history: GitHub's label is computed when asked, so it isn't reliable.
+    return buckets(here, (target.get("add") or 0) + (target.get("del") or 0), bool(target.get("iss")), not earlier,
+                   _repo_rate(prs, target["cr"], skip=target["n"], people=people))
+
+
+def valid(model) -> dict | None:
+    # Refused: a model fitted on too few PRs (every PR gets about the same chance) or built with other inputs.
+    ok = (isinstance(model, dict) and isinstance(model.get("intercept"), (int, float)) and isinstance(model.get("coef"), dict)
+          and all(isinstance(v, (int, float)) for v in model["coef"].values()) and (model.get("prs") or 0) >= MIN_MODEL_PRS
+          and model.get("features") == FEATURES)
+    return model if ok else None
+
+
+def triage(data: dict, model: dict, now: datetime) -> dict:
+    """Score every open, non-draft PR from an outside contributor."""
+    prs = data["open"] + data["closed"]
+    people = People(prs)
+    out = []
+    for p in data["open"]:
+        if not people.is_outside(p) or p.get("draft"):
+            continue
+        feats = features(p, prs, now, people)
+        out.append({"pr": p["n"], "title": p.get("ti"), "author": p.get("au"), "chance": round(predict(model, feats), 3),
+                    "reasons": explain(model, feats), "features": feats})
+    ev = model.get("evaluation") or {}
+    return {"note": NOTE, "prs": sorted(out, key=lambda x: -x["chance"]),
+            "model": {"checked_on_prs": ev.get("test_prs"), "brier": ev.get("brier"), "baseline_brier": ev.get("baseline_brier"),
+                      "built_at": model.get("built_at"), "features": list(FEATURES)}}