stackvitals 0.1.0__py3-none-any.whl

mergescout/signals/triage_math.py

@@ -0,0 +1,120 @@
+"""The outside-PR triage model: plain buckets, a small logistic fit and plain reasons. No database or numpy needed,
+so the suite can score PRs with the model file the pipeline publishes."""
+import math
+
+# The first value of each feature is the reference: its weight is 0 and it is never given as a reason.
+FEATURES = {
+    # The author's past here: a first PR, earlier PRs none merged, or merged PRs. One input, so the parts can't overlap.
+    "history": ["new", "tried", "1-2", "3+"],
+    "size": ["<50", "50-199", "200-999", "1000+"],
+    "issue": ["no", "yes"],
+    "repo_rate": ["unknown", "<30%", "30-60%", "60-80%", "80%+"],
+}
+WORDS = {
+    "history=tried": "has opened PRs here before, none merged yet",
+    "history=1-2": "has 1-2 merged PRs in this repo already", "history=3+": "has 3+ merged PRs in this repo already",
+    "size=50-199": "changes 50-199 lines", "size=200-999": "changes 200-999 lines", "size=1000+": "changes 1,000+ lines",
+    "issue=yes": "links an issue",
+    "repo_rate=<30%": "this repo merges under 30% of outside PRs", "repo_rate=30-60%": "this repo merges 30-60% of outside PRs",
+    "repo_rate=60-80%": "this repo merges 60-80% of outside PRs", "repo_rate=80%+": "this repo merges 80%+ of outside PRs",
+}
+
+
+def _bucket(value, edges: list[float], names: list[str]) -> str:
+    for edge, name in zip(edges, names):
+        if value < edge:
+            return name
+    return names[len(edges)]
+
+
+def buckets(here_merged: int, lines: int, linked_issue: bool, first_timer: bool, repo_rate: float | None) -> dict:
+    # An author's record in other repos was tried and dropped: it added nothing on unseen repos, and the live count
+    # (all of GitHub) can't match the training count (crawled repos only).
+    history = ("new" if first_timer else "tried") if here_merged < 1 else _bucket(here_merged, [3], ["1-2", "3+"])
+    return {"history": history, "size": _bucket(lines, [50, 200, 1000], FEATURES["size"]),
+            "issue": "yes" if linked_issue else "no",
+            "repo_rate": "unknown" if repo_rate is None else _bucket(repo_rate, [0.3, 0.6, 0.8], FEATURES["repo_rate"][1:])}
+
+
+COLUMNS = [f"{name}={v}" for name, values in FEATURES.items() for v in values[1:]]
+
+
+def _x(feats: dict) -> list[float]:
+    return [1.0] + [1.0 if feats.get(c.split("=")[0]) == c.split("=", 1)[1] else 0.0 for c in COLUMNS]
+
+
+def _sigmoid(z: float) -> float:
+    return 1 / (1 + math.exp(-max(-30.0, min(30.0, z))))
+
+
+def _solve(a: list[list[float]], b: list[float]) -> list[float]:
+    n = len(b)
+    m = [row[:] + [b[i]] for i, row in enumerate(a)]
+    for col in range(n):
+        pivot = max(range(col, n), key=lambda r: abs(m[r][col]))
+        m[col], m[pivot] = m[pivot], m[col]
+        for r in range(n):
+            if r != col and m[col][col]:
+                f = m[r][col] / m[col][col]
+                m[r] = [x - f * y for x, y in zip(m[r], m[col])]
+    return [m[i][n] / m[i][i] if m[i][i] else 0.0 for i in range(n)]
+
+
+def fit(rows: list[dict], ridge: float = 1.0, iters: int = 25) -> dict:
+    """Logistic regression on aggregated cells (rows sharing every bucket fit as one), Newton steps with a light ridge."""
+    cells = {}
+    for r in rows:
+        key = tuple(r[name] for name in FEATURES)
+        n, k = cells.get(key, (0, 0))
+        cells[key] = (n + 1, k + bool(r["merged"]))
+    xs = [(_x(dict(zip(FEATURES, key))), n, k) for key, (n, k) in cells.items()]
+    w = [0.0] * (len(COLUMNS) + 1)
+    for _ in range(iters):
+        grad = [0.0] * len(w)
+        hess = [[0.0] * len(w) for _ in w]
+        for x, n, k in xs:
+            p = _sigmoid(sum(a * b for a, b in zip(w, x)))
+            for i, xi in enumerate(x):
+                if xi:
+                    grad[i] += xi * (k - n * p)
+                    for j, xj in enumerate(x):
+                        if xj:
+                            hess[i][j] += n * p * (1 - p) * xi * xj
+        for i in range(1, len(w)):
+            grad[i] -= ridge * w[i]
+            hess[i][i] += ridge
+        step = _solve(hess, grad)
+        w = [a + b for a, b in zip(w, step)]
+        if max(abs(s) for s in step) < 1e-6:
+            break
+    return {"version": 1, "features": FEATURES, "intercept": w[0], "coef": dict(zip(COLUMNS, w[1:]))}
+
+
+def predict(model: dict, feats: dict) -> float:
+    return _sigmoid(model["intercept"] + sum(model["coef"].get(f"{k}={v}", 0.0) for k, v in feats.items()))
+
+
+def calibration(model: dict, rows: list[dict]) -> dict:
+    """Predicted against actual merge share in ten equal bins, and Brier scores against "the repo's usual rate" alone."""
+    scored = sorted(((predict(model, r), bool(r["merged"]), r["repo_rate"]) for r in rows), key=lambda t: t[0])
+    deciles = []
+    for i in range(10):
+        part = scored[i * len(scored) // 10:(i + 1) * len(scored) // 10]
+        if part:
+            deciles.append({"predicted": round(sum(p for p, _, _ in part) / len(part), 4),
+                            "actual": round(sum(y for _, y, _ in part) / len(part), 4), "n": len(part)})
+    by_rate = {}
+    for _, y, rate in scored:
+        n, k = by_rate.get(rate, (0, 0))
+        by_rate[rate] = (n + 1, k + y)
+    brier = sum((p - y) ** 2 for p, y, _ in scored) / len(scored)
+    baseline = sum((by_rate[rate][1] / by_rate[rate][0] - y) ** 2 for _, y, rate in scored) / len(scored)
+    return {"deciles": deciles, "brier": round(brier, 5), "baseline_brier": round(baseline, 5), "n": len(scored)}
+
+
+def explain(model: dict, feats: dict, k: int = 3) -> list[dict]:
+    """The biggest reasons behind a score, in plain words; reference buckets are never reasons."""
+    parts = [(f"{name}={value}", model["coef"].get(f"{name}={value}", 0.0)) for name, value in feats.items()]
+    parts = [(key, w) for key, w in parts if key in WORDS and abs(w) >= 0.1]
+    parts.sort(key=lambda kw: -abs(kw[1]))
+    return [{"why": WORDS[key], "effect": "raises" if w > 0 else "lowers", "weight": round(w, 3)} for key, w in parts[:k]]

mergescout/signals/walls.py

@@ -0,0 +1,91 @@
+import re
+
+import duckdb
+import pyarrow as pa
+
+AI = r"\b(ai|llm|chatgpt|copilot)\b"
+WALLS = {
+    "cla": r"contributor license agreement|\bcla\b|cla-assistant",
+    "dco": r"developer certificate of origin|signed-off-by|\bdco\b",
+    "issue_first": r"(open|file|create) an issue (first|before)"
+                   r"|discuss(ed)? .{0,40}(first|before (opening|sending|submitting))",
+    "ai_ban": rf"(do not|don't|will not|won't|cannot|can't|not) (accept|allow|merge|review|take)[^.]{{0,40}}{AI}"
+              rf"|{AI}[- ](generated|written|assisted)[^.]{{0,60}}"
+              r"(not (be )?(accepted|allowed|merged|permitted|welcome)|prohibited|forbidden|banned|will be (closed|rejected))"
+              rf"|\b(prohibit(s|ed)?|bans?|banned|forbid(s|den)?)\b[^.]{{0,30}}{AI}",
+    # "...contributions that use other licenses" is a licence rule, not a ban on outside PRs.
+    "no_outside_prs": r"(not|no longer) accept(ing)? (external |outside |unsolicited )?(pull requests|contributions|prs)"
+                      r"(?! (that|which|who|with|without|unless|under|using|licensed|from (?:bots|ai)))",
+}
+# AI rules and bans come from CONTRIBUTING only: a README describes the product, which may itself be about AI.
+CONTRIBUTING_ONLY = {"ai_ban"}
+AI_RULES = (r"\b(ai|llm)[- ](generated|assisted|assistance|guidelines|policy|usage|tools?|agents?|powered)\b"
+            r"|use of (ai|llm|generative)|generative ai|coding with ai")
+# The Apache license header ("under one or more contributor license agreements") is not a CLA to sign.
+LICENSE_HEADER = r"one or more\W+contributor license agreements?"
+BLOCKING = set(WALLS) | {"issues_disabled"}
+# "A lighter alternative, should the CLA prove heavy: the DCO" names a rule the repo does not have.
+# Only words before the match count: "this is not optional" and "instead of sending a PR" follow real rules.
+HYPOTHETICAL = r"\b(alternative|optionally|if you prefer|should (the|it|this|we) \S+ prove|we (might|may) (switch|adopt|move))\b"
+# A list item, heading or blank line starts a new sentence even without a full stop.
+BLOCK_BREAK = r"\n\s*\n|\n\s*(?=[-*+#>|]|\d+[.)]\s)"
+
+
+def _clean(text: str | None) -> str:
+    # One line of single spaces, so phrases broken across lines still match.
+    text = re.sub(BLOCK_BREAK, ". ", text or "")
+    return re.sub(LICENSE_HEADER, " ", " ".join(text.split()), flags=re.I)
+
+
+def _snippet(text: str, m: re.Match) -> str:
+    start = max(0, m.start() - 60)
+    return text[start:start + 160].strip()
+
+
+def _sentence_before(text: str, m: re.Match) -> str:
+    starts = [b.end() for b in re.finditer(r"[.!?]\s", text[:m.start()])]
+    return text[starts[-1] if starts else 0:m.start()]
+
+
+def _first(wall: str, pattern: str, text: str) -> re.Match | None:
+    for m in re.finditer(pattern, text, re.I):
+        if re.search(HYPOTHETICAL, _sentence_before(text, m), re.I):
+            continue
+        # "For major changes, please open an issue first" (a common README template) does not stop small fixes.
+        if wall == "issue_first" and re.search(r"major change", text[max(0, m.start() - 40):m.start()], re.I):
+            continue
+        return m
+    return None
+
+
+def find_walls(meta: dict) -> list[tuple[str, str]]:
+    found = []
+    if meta.get("has_issues") is False:
+        found.append(("issues_disabled", "issues are turned off for this repo"))
+    contributing = _clean(meta.get("contributing_excerpt"))
+    text = f"{contributing} {_clean(meta.get('readme_excerpt'))}"
+    for wall, pattern in WALLS.items():
+        source = contributing if wall in CONTRIBUTING_ONLY else text
+        m = _first(wall, pattern, source)
+        if m:
+            found.append((wall, _snippet(source, m)))
+    if "ai_ban" not in {w for w, _ in found}:
+        m = re.search(AI_RULES, contributing, re.I)
+        if m:
+            found.append(("ai_policy", _snippet(contributing, m)))
+        elif meta.get("ai_policy_sha"):
+            found.append(("ai_policy", "AI_POLICY.md in the repo root"))
+    return found
+
+
+def build_walls(con: duckdb.DuckDBPyConnection) -> None:
+    rows = []
+    for repo_id, has_issues, ai_sha, contributing, readme in con.execute(
+            "SELECT repo_id, has_issues, ai_policy_sha, contributing_excerpt, readme_excerpt FROM repo_meta").fetchall():
+        for wall, evidence in find_walls({"has_issues": has_issues, "ai_policy_sha": ai_sha,
+                                          "contributing_excerpt": contributing, "readme_excerpt": readme}):
+            rows.append({"repo_id": repo_id, "wall": wall, "blocking": wall in BLOCKING, "evidence": evidence})
+    schema = pa.schema([("repo_id", pa.int64()), ("wall", pa.string()), ("blocking", pa.bool_()),
+                        ("evidence", pa.string())])
+    con.register("walls_rows", pa.Table.from_pylist(rows, schema=schema))
+    con.execute("CREATE OR REPLACE TABLE repo_walls AS SELECT * FROM walls_rows")

mergescout/snapshot/__main__.py

@@ -0,0 +1,215 @@
+import argparse
+import json
+import os
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+
+from mergescout.config import load_config
+
+
+def write_extras(con, out: Path, now: datetime, log=lambda msg: print(msg, flush=True)) -> None:
+    """Review-flow and community benchmarks, the triage model and the dependency evidence for the suite; a failure here
+    never stops the snapshot."""
+    import mergescout.signals.bump_ledger as bump_ledger
+    import mergescout.signals.flow_bench as flow_bench
+    import mergescout.signals.triage as triage
+    try:
+        bench = flow_bench.flow_benchmarks(con, now)
+        # Outside merge odds checked against what happened next, and coding-agent PRs per group, in the same file;
+        # their failure never costs the peer comparisons.
+        try:
+            bench["odds_backtest"] = flow_bench.odds_backtest(con, now)
+            bench["agent_prs"] = flow_bench.agent_prs(con, now)
+        except Exception as exc:  # noqa: BLE001
+            log(f"odds backtest skipped: {exc}")
+        # Agents' PRs against people's of the same repo, month and size; older crawls without branch names skip it.
+        try:
+            bench["agent_matched"] = flow_bench.agent_matched(con, now)
+        except Exception as exc:  # noqa: BLE001
+            log(f"matched agent gaps skipped: {exc}")
+        # Each coding agent on its own across the crawl: the public leaderboard repos compare their own agents with.
+        try:
+            bench["agent_leaderboard"] = flow_bench.agent_leaderboard(con, now)
+        except Exception as exc:  # noqa: BLE001
+            log(f"agent leaderboard skipped: {exc}")
+        (out / "flow-benchmarks.json").write_text(json.dumps(bench, indent=1, sort_keys=True), encoding="utf-8")
+        log(f"flow benchmarks: {bench['repos']} repos, {len(bench['groups'])} groups")
+        bt = bench.get("odds_backtest") or {}
+        if bt.get("mae"):
+            log(f"outside merge odds backtest: {bt['repos']} repos, {bt['prs']} PRs; mean error raw {bt['mae']['raw']:.1%}, "
+                f"shrunk {bt['mae']['shrunk']:.1%}, typical only {bt['mae']['typical_only']:.1%}")
+    except Exception as exc:  # noqa: BLE001
+        log(f"flow benchmarks skipped: {exc}")
+    try:
+        import mergescout.signals.delivery_bench as delivery_bench
+        bench = delivery_bench.delivery_benchmarks(con, now)
+        (out / "delivery-benchmarks.json").write_text(json.dumps(bench, indent=1, sort_keys=True), encoding="utf-8")
+        mid = (bench["groups"].get("all|all") or {}).get("pickup_p75_hours") or {}
+        log(f"delivery benchmarks: {bench['repos']} repos, {len(bench['groups'])} groups; pickup P75 median of all repos "
+            f"{mid.get('p50')} h ({mid.get('p50_low')}-{mid.get('p50_high')}), inside LinearB's good band "
+            f"{(mid.get('linearb') or {}).get('good')}")
+    except Exception as exc:  # noqa: BLE001
+        log(f"delivery benchmarks skipped: {exc}")
+    try:
+        import mergescout.signals.community_bench as community_bench
+        bench = community_bench.community_benchmarks(con, now)
+        (out / "community-benchmarks.json").write_text(json.dumps(bench, indent=1, sort_keys=True), encoding="utf-8")
+        log(f"community benchmarks: {bench['repos']} repos, {len(bench['groups'])} groups")
+        # Logged daily so the newcomer lever is checked on the whole crawl, like the upgrade backtest.
+        log("newcomers returning by first answer: " + "; ".join(
+            f"{b['bucket']} {b['returned']:.1%} (n {b['newcomers']})" for b in bench["newcomers"]["by_answer"]))
+    except Exception as exc:  # noqa: BLE001
+        log(f"community benchmarks skipped: {exc}")
+    try:
+        model = triage.build_triage_model(con, now)
+        (out / "triage-model.json").write_text(json.dumps(model, indent=1, sort_keys=True), encoding="utf-8")
+        log(f"triage model: {model['prs']} PRs, checked on {model['evaluation'].get('test_prs')}")
+    except Exception as exc:  # noqa: BLE001
+        log(f"triage model skipped: {exc}")
+    try:
+        prs = bump_ledger.build_bump_prs(con)
+        cells = bump_ledger.build_bump_ledger(con, now)
+        (out / "deps-ledger.json").write_text(json.dumps(bump_ledger.ledger_json(con, now), sort_keys=True), encoding="utf-8")
+        log(f"upgrade ledger: {prs} bot PRs, {cells} cells")
+    except Exception as exc:  # noqa: BLE001
+        log(f"upgrade ledger skipped: {exc}")
+    try:
+        # Whether flags on older upgrades predicted later trouble, re-checked daily as titles and CI results accumulate.
+        log(f"upgrade backtest: {bump_ledger.bump_backtest(con, now)}")
+    except Exception as exc:  # noqa: BLE001
+        log(f"upgrade backtest skipped: {exc}")
+    try:
+        write_package_map(con, out, now, log)
+    except Exception as exc:  # noqa: BLE001
+        log(f"package map skipped: {exc}")
+    try:
+        write_trends(con, out, now, log)
+    except Exception as exc:  # noqa: BLE001
+        log(f"trend lines skipped: {exc}")
+    try:
+        write_vulns(out, now, log)
+    except Exception as exc:  # noqa: BLE001
+        log(f"vulnerability history skipped: {exc}")
+    try:
+        write_owners(out, now, log)
+    except Exception as exc:  # noqa: BLE001
+        log(f"ownership watch skipped: {exc}")
+
+
+def write_owners(out: Path, now: datetime, log) -> None:
+    """Today's publishers and owners of the mapped packages, with the changes since yesterday's file (fetched by the workflow)."""
+    import mergescout.signals.owners_watch as ow
+    mapped = json.loads((out / "deps-map.json").read_text(encoding="utf-8")).get("packages", {})
+    packages = sorted((*key.split(":", 1), info.get("version")) for key, info in mapped.items())
+    path = out / "deps-owners.json"
+    previous = json.loads(path.read_text(encoding="utf-8")) if path.exists() else {}
+    state = ow.refresh_owners(packages, previous, now)
+    path.write_text(json.dumps(state, sort_keys=True), encoding="utf-8")
+    changed = sum(1 for p in state["packages"].values() if p.get("events"))
+    log(f"ownership watch: {len(state['packages'])} packages, {changed} with changes since watching began")
+
+
+def write_vulns(out: Path, now: datetime, log) -> None:
+    """OSV history of the mapped packages; yesterday's file (fetched by the workflow) caches unchanged advisories."""
+    import mergescout.signals.osv as osv
+    from mergescout.signals.exploits import lookup
+    mapped = json.loads((out / "deps-map.json").read_text(encoding="utf-8")).get("packages", {})
+    packages = sorted(tuple(key.split(":", 1)) for key in mapped)
+    path = out / "deps-osv.json"
+    previous = json.loads(path.read_text(encoding="utf-8")) if path.exists() else {}
+    # CISA KEV and FIRST's EPSS for the advisories' CVEs: one feed and about 80 requests a day, both free.
+    state = osv.refresh_vulns(packages, previous, now, exploits=lookup)
+    path.write_text(json.dumps(state, sort_keys=True), encoding="utf-8")
+    hit = sum(1 for p in state["packages"].values() if p.get("advisories"))
+    ex = state.get("exploits") or {}
+    log(f"vulnerability history: {len(state['packages'])} packages, {hit} with advisories, {len(state['records'])} advisories cached; "
+        f"{ex.get('scored', 0)} CVEs scored by EPSS, {ex.get('kev', 0)} in CISA KEV")
+
+
+def write_trends(con, out: Path, now: datetime, log) -> None:
+    """Quarterly lines for the watched source repos that the crawl already covers (the rest arrive as it crawls them)."""
+    from mergescout.signals.deps_trends import repo_trends
+    watch_file = out / "watch-repos.txt"
+    watch = watch_file.read_text(encoding="utf-8").split() if watch_file.exists() else []
+    trends = repo_trends(con, watch, now)
+    (out / "deps-trends.json").write_text(json.dumps({"built_at": now.strftime("%Y-%m-%dT%H:%M:%SZ"), "repos": trends},
+                                                     sort_keys=True), encoding="utf-8")
+    log(f"trend lines: {len(trends)} of {len(watch)} watched repos are crawled")
+
+
+def write_package_map(con, out: Path, now: datetime, log) -> None:
+    """Source repos of the ledger's npm and PyPI packages and of the most-used ones; yesterday's file (fetched by the
+    workflow) is reused per package."""
+    import mergescout.signals.bump_ledger as bump_ledger
+    import mergescout.signals.deps_map as deps_map
+    import mergescout.signals.top_packages as top_packages
+    from mergescout.github.graphql import GitHubGraphQL
+    packages = con.execute("SELECT DISTINCT ecosystem, package FROM bump_ledger WHERE ecosystem IN ('npm', 'pypi') "
+                           "AND prs >= $min", {"min": bump_ledger.MIN_PUBLISH}).fetchall()
+    path = out / "deps-map.json"
+    previous = json.loads(path.read_text(encoding="utf-8")).get("packages", {}) if path.exists() else {}
+    # A failed download keeps yesterday's top list; the ledger's packages come first within the lookup time budget.
+    top = top_packages.fetch_top() or [tuple(k.split(":", 1)) for k, v in previous.items() if v.get("top")]
+    # Claimed links are checked against manifests with the workflow's own token (about 1 point per 20 repos).
+    token = os.environ.get("GITHUB_TOKEN") or os.environ.get("MS_RELEASE_TOKEN")
+    gql = GitHubGraphQL(token, max_wait=0, reserve=50) if token else None
+    mapped = deps_map.refresh_map(list(dict.fromkeys([tuple(p) for p in packages] + top)), previous, now, gql=gql)
+    tops = set(top)
+    for key, info in mapped.items():
+        info["top"] = tuple(key.split(":", 1)) in tops
+    path.write_text(json.dumps({"built_at": now.strftime("%Y-%m-%dT%H:%M:%SZ"), "packages": mapped}, sort_keys=True),
+                    encoding="utf-8")
+    links = {}
+    for info in mapped.values():
+        links[info.get("link")] = links.get(info.get("link"), 0) + 1
+    # Source repos the crawler should keep current (seed --watch); only links we can stand behind.
+    watch = sorted({i["repo"].lower() for i in mapped.values() if i.get("repo") and i.get("link") in ("verified", "claimed_checked")})
+    (out / "watch-repos.txt").write_text("".join(f"{r}\n" for r in watch), encoding="utf-8")
+    log(f"package map: {len(mapped)} packages ({len(tops)} most-used), links {links}, {len(watch)} repos to watch")
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(prog="python -m mergescout.snapshot", description="Build and publish the snapshot.")
+    sub = parser.add_subparsers(dest="cmd", required=True)
+    b = sub.add_parser("build", help="Build signals and mergescout.sqlite from all crawl data")
+    b.add_argument("--from-releases", action="store_true")
+    b.add_argument("--out", default=None)
+    p = sub.add_parser("publish", help="Compress the snapshot and publish it as today's data release")
+    p.add_argument("dir")
+    p.add_argument("--keep", type=int, default=7)
+    p.add_argument("--force", action="store_true", help="Publish even if the snapshot shrank by more than half")
+    p.add_argument("--repo", default=os.environ.get("MS_STORE_REPO") or os.environ.get("GITHUB_REPOSITORY"))
+    args = parser.parse_args(argv)
+    cfg = load_config()
+    now = datetime.now(timezone.utc)
+    if args.cmd == "build":
+        from mergescout.signals.__main__ import prepare, write_bundle
+        from mergescout.snapshot.build import build_snapshot
+        out = Path(args.out) if args.out else cfg.data_dir / "snapshot"
+        con = prepare(cfg, out, args.from_releases, now)
+        if con is None:
+            return 1
+        write_bundle(con, out)
+        print(f"snapshot: {build_snapshot(con, out / 'mergescout.sqlite', now)}", flush=True)
+        write_extras(con, out, now)
+        con.close()
+        return 0
+    if args.cmd == "publish":
+        from mergescout.github.releases import Releases
+        from mergescout.snapshot.publish import publish
+        token = os.environ.get("MS_RELEASE_TOKEN")
+        if not token or not args.repo:
+            print("Set MS_RELEASE_TOKEN and MS_STORE_REPO (or GITHUB_REPOSITORY).", file=sys.stderr)
+            return 1
+        try:
+            publish(Releases(token, args.repo), Path(args.dir), now, keep=args.keep, force=args.force)
+        except ValueError as exc:
+            print(exc, file=sys.stderr)
+            return 1
+        return 0
+    return 2
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

mergescout/snapshot/build.py

@@ -0,0 +1,147 @@
+import os
+import sqlite3
+from datetime import datetime
+from pathlib import Path
+
+import duckdb
+
+from mergescout.signals.issues import norm_label_macro
+from mergescout.snapshot.schema import DDL, SCHEMA_VERSION
+from mergescout.snapshot.score import WEIGHTS, Weights
+
+ISO = "CREATE OR REPLACE MACRO iso(x) AS strftime(timezone('UTC', x), '%Y-%m-%dT%H:%M:%SZ')"
+# Issue-template checklist lines ("I affirm", "I have read the Contributing Guide") hide what the issue asks.
+BOILERPLATE = (r"(?im)^.*\b(i affirm|i understand that if|i have (read|searched|checked|confirmed|verified)"
+               r"|code of conduct|this is not a duplicate)\b.*$\n?")
+
+SCORED = """
+CREATE OR REPLACE TABLE snap_issues AS
+WITH lt AS (SELECT repo_id, label, CASE WHEN enough THEN outsider_low END AS fixed_low FROM label_trust),
+il AS (SELECT repo_id, number, unnest(list_transform(labels, lambda x: norm_label(x))) AS label FROM issue_signals),
+fixed AS (SELECT il.repo_id, il.number, max(lt.fixed_low) AS fixed_low FROM il JOIN lt USING (repo_id, label)
+          GROUP BY ALL),
+walls AS (SELECT repo_id, count(*) FILTER (WHERE blocking) AS blocking FROM repo_walls GROUP BY repo_id),
+base AS (
+    SELECT s.*, i.body_excerpt, rm.language, rs.merge_low, rs.review_enough, rs.median_review_hours, rs.newcomer_low,
+        coalesce(w.blocking, 0) AS blocking_walls, f.fixed_low
+    FROM issue_signals s JOIN issues i USING (repo_id, number) JOIN repo_meta rm USING (repo_id)
+    LEFT JOIN repo_stats rs USING (repo_id) LEFT JOIN walls w USING (repo_id) LEFT JOIN fixed f USING (repo_id, number)
+    WHERE s.opportunity OR s.available),
+parts AS (
+    -- Every part is a DOUBLE: SQLite cannot store DuckDB decimals.
+    SELECT *, (CASE WHEN available THEN 1 ELSE 0 END)::DOUBLE AS p_avail,
+        coalesce(merge_low, $prior_merge)::DOUBLE AS p_merge,
+        (CASE WHEN review_enough THEN 1 / (1 + median_review_hours / $reply_half) ELSE $prior_reply END)::DOUBLE
+            AS p_reply,
+        coalesce(fixed_low, newcomer_low, $prior_fixed)::DOUBLE AS p_fixed,
+        (0.5 * least(length(coalesce(body_excerpt, '')), 600) / 600
+          + 0.25 * CASE WHEN regexp_matches(lower(coalesce(body_excerpt, '')),
+                                            '(steps to reproduce|expected|```|error|traceback|repro)') THEN 1 ELSE 0 END
+          + 0.25 * CASE WHEN len(coalesce(labels, [])) > 0 THEN 1 ELSE 0 END)::DOUBLE AS p_quality,
+        (greatest(power($dead_end, coalesce(dead_ends, 0)), $dead_end_floor)
+          * CASE WHEN coalesce(interest_30d, 0) >= $crowded_interest THEN $crowded ELSE 1 END
+          * CASE WHEN blocking_walls > 0 THEN $blocking ELSE 1 END
+          -- Trackers and debates are rarely a task; issues open for years usually hide a reason.
+          * CASE WHEN kind = 'discussion' THEN $discussion ELSE 1 END
+          * CASE WHEN triage_label THEN $untriaged ELSE 1 END
+          * CASE WHEN date_diff('day', created_at, $now) > $very_old_days THEN $very_old
+                 WHEN date_diff('day', created_at, $now) > $old_days THEN $old ELSE 1 END)::DOUBLE AS mult
+    FROM base),
+scored AS (
+    SELECT *, (CASE WHEN available THEN ($w_avail * p_avail + $w_merge * p_merge + $w_reply * p_reply
+                                         + $w_fixed * p_fixed + $w_quality * p_quality) * mult ELSE 0 END)::DOUBLE
+              AS score
+    FROM parts),
+ranked AS (
+    -- Freshness is applied once a day at build time, so queries only sort an index.
+    SELECT *, score * power(0.5, greatest(coalesce(date_diff('day', coalesce(updated_at, created_at), $now), 0), 0)
+                                 / $fresh_half) AS rank_score,
+        row_number() OVER (PARTITION BY repo_id, opportunity ORDER BY score DESC, updated_at DESC) AS repo_rank
+    FROM scored)
+SELECT * FROM ranked WHERE opportunity OR repo_rank <= $per_repo_other
+ORDER BY opportunity DESC, rank_score DESC LIMIT $max_issues
+"""
+
+ISSUE_ROWS = """
+SELECT repo_id, number, title,
+    substr(ltrim(regexp_replace(coalesce(body_excerpt, ''), $boilerplate, '', 'g'), ' ' || chr(10) || chr(13) || chr(9)),
+           1, 300),
+    to_json(coalesce(labels, []))::VARCHAR,
+    language, iso(created_at), iso(updated_at), available::INT, opportunity::INT, blocked_by, assignees,
+    linked_open_prs, dead_ends, claimers, iso(last_claim_at), interest_30d, score, rank_score,
+    p_avail, p_merge, p_reply, p_fixed, p_quality, mult, iso(fetched_at), beginner::INT, kind
+FROM snap_issues WHERE repo_id IN (SELECT repo_id FROM kept_repos)
+"""
+
+REPO_ROWS = """
+SELECT rm.repo_id, rm.name, rm.stars, rm.language, iso(rm.pushed_at), rm.archived::INT, rm.license,
+    rm.has_discussions::INT, rm.open_issues, rm.open_prs, coalesce(ins.n, 0), rs.outsider_prs, rs.merged, rs.closed,
+    rs.merge_rate, rs.merge_low, rs.merge_high, rs.merge_enough::INT, rs.newcomer_merged, rs.newcomer_decided,
+    rs.newcomer_rate, rs.newcomer_low, rs.newcomer_high, rs.newcomer_enough::INT, rs.reviewed, rs.median_review_hours,
+    rs.review_enough::INT, rs.window_days, rs.window_complete::INT, rs.ai_prs, rs.bot_prs, rs.insider_prs,
+    coalesce(w.blocking, 0), coalesce(w.walls_json, '[]'), coalesce(l.labels_json, '[]'), coalesce(n.notes, '[]'),
+    (coalesce(rm.contributing_excerpt, '') <> '')::INT, iso(rm.fetched_at)
+FROM repo_meta rm LEFT JOIN repo_stats rs USING (repo_id)
+LEFT JOIN (SELECT repo_id, count(*) AS n FROM people WHERE kind = 'insider' GROUP BY 1) ins USING (repo_id)
+LEFT JOIN (SELECT repo_id, count(*) FILTER (WHERE blocking) AS blocking,
+                  to_json(list({wall: wall, blocking: blocking, evidence: evidence} ORDER BY wall))::VARCHAR AS walls_json
+           FROM repo_walls GROUP BY 1) w USING (repo_id)
+LEFT JOIN (SELECT repo_id, to_json(list({label: label, closed_n: closed_n, known_fixes: known_fixes,
+                  outsider_fixes: outsider_fixes, outsider_share: outsider_share, low: outsider_low,
+                  high: outsider_high, enough: enough, median_fix_lines: median_fix_lines,
+                  median_fix_files: median_fix_files, open_stale: open_stale} ORDER BY label))::VARCHAR AS labels_json
+           FROM label_trust GROUP BY 1) l USING (repo_id)
+-- Evidence notes are about the repo, so they are stored once per repo, not on every issue.
+LEFT JOIN (SELECT repo_id, any_value(json_extract(evidence_json, '$.notes')::VARCHAR) AS notes
+           FROM candidates GROUP BY 1) n USING (repo_id)
+WHERE rm.repo_id IN (SELECT repo_id FROM kept_repos)
+"""
+
+EXAMPLE_ROWS = """
+SELECT repo_id, number, iso(merged_at), additions + deletions, changed_files,
+    round(epoch(merged_at - created_at) / 86400, 1)
+FROM (SELECT *, row_number() OVER (PARTITION BY repo_id ORDER BY merged_at DESC) AS rn FROM pr_class
+      WHERE author_class = 'outsider' AND state = 'MERGED' AND merged_at IS NOT NULL)
+WHERE rn <= 3 AND repo_id IN (SELECT repo_id FROM kept_repos)
+"""
+
+
+def build_snapshot(con: duckdb.DuckDBPyConnection, path: Path, now: datetime, w: Weights = WEIGHTS,
+                   per_repo_other: int = 30, max_issues: int = 350_000) -> dict:
+    norm_label_macro(con)
+    con.execute(ISO)
+    con.execute(SCORED, {
+        "prior_merge": w.prior_merge, "prior_reply": w.prior_reply, "prior_fixed": w.prior_fixed,
+        "reply_half": w.reply_half_hours, "dead_end": w.dead_end, "dead_end_floor": w.dead_end_floor,
+        "crowded_interest": w.crowded_interest, "crowded": w.crowded, "blocking": w.blocking_wall,
+        "w_avail": w.availability, "w_merge": w.outsider_merge, "w_reply": w.reply, "w_fixed": w.fixed_before,
+        "w_quality": w.issue_quality, "now": now, "fresh_half": w.freshness_half_days, "discussion": w.discussion, "untriaged": w.untriaged,
+        "old_days": w.old_days, "old": w.old, "very_old_days": w.very_old_days, "very_old": w.very_old,
+        "per_repo_other": per_repo_other, "max_issues": max_issues})
+    # A rename between crawls can leave two repo ids with one name; the newest copy wins.
+    con.execute("CREATE OR REPLACE TEMP TABLE kept_repos AS SELECT repo_id FROM repo_meta WHERE name IS NOT NULL "
+                "QUALIFY row_number() OVER (PARTITION BY lower(name) ORDER BY fetched_at DESC NULLS LAST, "
+                "repo_id DESC) = 1")
+    tmp = path.with_name(path.name + ".tmp")
+    tmp.unlink(missing_ok=True)
+    db = sqlite3.connect(tmp)
+    db.executescript(DDL)
+    counts = {}
+    for table, sql in (("repos", REPO_ROWS), ("issues", ISSUE_ROWS), ("outsider_prs", EXAMPLE_ROWS)):
+        rows = con.execute(sql, {"boilerplate": BOILERPLATE} if table == "issues" else None).fetchall()
+        if rows:
+            db.executemany(f"INSERT INTO {table} VALUES ({', '.join('?' * len(rows[0]))})", rows)
+        counts[table] = len(rows)
+    db.execute("INSERT INTO issues_fts (rowid, title, labels, body) SELECT rowid, title, labels, body FROM issues")
+    meta = {"schema_version": SCHEMA_VERSION, "built_at": now.strftime("%Y-%m-%dT%H:%M:%SZ"),
+            "issues": counts["issues"], "repos": counts["repos"],
+            "free_issues": db.execute("SELECT count(*) FROM issues WHERE available = 1").fetchone()[0]}
+    db.executemany("INSERT INTO meta VALUES (?, ?)", [(k, str(v)) for k, v in meta.items()])
+    db.commit()
+    db.execute("INSERT INTO issues_fts (issues_fts) VALUES ('optimize')")
+    db.execute("ANALYZE")
+    db.commit()
+    db.execute("VACUUM")
+    db.close()
+    os.replace(tmp, path)
+    return counts

mergescout/snapshot/publish.py

@@ -0,0 +1,70 @@
+import hashlib
+import json
+import sqlite3
+import tempfile
+from datetime import datetime
+from pathlib import Path
+
+import zstandard
+
+SNAPSHOT, PACKED, MANIFEST, PREFIX = "mergescout.sqlite", "mergescout.sqlite.zst", "manifest.json", "data-"
+
+
+def sha256(path: Path) -> str:
+    h = hashlib.sha256()
+    with path.open("rb") as f:
+        for chunk in iter(lambda: f.read(1 << 20), b""):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+def pack(out: Path) -> dict:
+    src = out / SNAPSHOT
+    db = sqlite3.connect(src)
+    meta = dict(db.execute("SELECT key, value FROM meta").fetchall())
+    db.close()
+    with src.open("rb") as fi, (out / PACKED).open("wb") as fo:
+        # One thread and the known size keep memory small (threads ran out of memory on a 8 GB laptop; same speed).
+        zstandard.ZstdCompressor(level=19).copy_stream(fi, fo, size=src.stat().st_size)
+    manifest = {"schema_version": meta["schema_version"], "built_at": meta["built_at"], "issues": int(meta["issues"]),
+                "repos": int(meta["repos"]), "sha256": sha256(out / PACKED), "size": (out / PACKED).stat().st_size,
+                "sqlite_size": src.stat().st_size}
+    (out / MANIFEST).write_text(json.dumps(manifest, indent=1), encoding="utf-8")
+    return manifest
+
+
+def previous_manifest(releases, tag: str) -> dict | None:
+    # The newest published day other than the one being written.
+    older = sorted((r for r in releases.list_releases() if r["tag_name"].startswith(PREFIX) and r["tag_name"] != tag),
+                   key=lambda r: r["tag_name"], reverse=True)
+    for release in older:
+        asset = next((a for a in releases.assets(release) if a["name"] == MANIFEST), None)
+        if asset:
+            with tempfile.TemporaryDirectory() as tmp:
+                path = Path(tmp) / MANIFEST
+                releases.download_asset(asset, path)
+                return json.loads(path.read_text(encoding="utf-8"))
+    return None
+
+
+def publish(releases, out: Path, now: datetime, keep: int = 7, force: bool = False, min_ratio: float = 0.5,
+            log=print) -> str:
+    manifest = pack(out)
+    if not manifest["issues"] or not manifest["repos"]:
+        raise ValueError("refusing to publish an empty snapshot")
+    tag = f"{PREFIX}{now:%Y-%m-%d}"
+    prev = previous_manifest(releases, tag)
+    # A build that lost part of the crawl data must not replace a good day; --force when the drop is expected.
+    if prev and not force and manifest["issues"] < min_ratio * (prev.get("issues") or 0):
+        raise ValueError(f"refusing to publish: the snapshot shrank from {prev['issues']} to {manifest['issues']} "
+                         "issues (a partial build?); use --force if this is expected")
+    # Data first, manifest last: a reader never sees a manifest whose file is missing (the checksum guards the rest).
+    releases.upload(tag, out / PACKED)
+    releases.upload(tag, out / MANIFEST)
+    data = sorted((r for r in releases.list_releases() if r["tag_name"].startswith(PREFIX)),
+                  key=lambda r: r["tag_name"], reverse=True)
+    for old in data[keep:]:
+        releases.delete_release(old)
+        log(f"removed {old['tag_name']}")
+    log(f"published {tag}: {manifest['issues']} issues, {manifest['size'] / 1e6:.1f} MB")
+    return tag