stackvitals 0.1.0__py3-none-any.whl

mergescout/bench/report.py

@@ -0,0 +1,230 @@
+"""A public report from the crawl's published benchmark files: aggregates only (no repo or person named), and every
+number in a CSV beside it so anyone can recompute it."""
+import argparse
+import csv
+import json
+from datetime import datetime
+from pathlib import Path
+
+MIN_REPOS = 50
+LICENCE = "CC BY 4.0"
+QS = ("p10", "p25", "p50", "p75", "p90", "p50_low", "p50_high")
+METRICS = {"ci": ("median_s", "p90_s", "first_try_share", "rerun_share", "fail_share"),
+           "flow": ("first_review_hours", "merge_hours", "outsider_merge_rate", "size_lines"),
+           "delivery": ("pickup_p75_hours", "review_p75_hours", "cycle_to_merge_p75_hours"),
+           "deploys": ("days_per_week", "days_per_month"),
+           "community": ("first_answer_hours", "answered_1d_share", "newcomer_answered_share", "newcomer_return_rate"),
+           "stars": ("gained_12w", "growth_12w")}
+RECOMPUTE_SQL = """```sql
+-- DuckDB, in the folder with the CSV files: the typical repo's pickup P75 by language, with its range
+SELECT language, repos, pickup_p75_hours_p50, pickup_p75_hours_p50_low, pickup_p75_hours_p50_high,
+       pickup_p75_hours_linearb_good
+FROM read_csv_auto('delivery.csv') WHERE size = 'all' ORDER BY repos DESC;
+```"""
+RECOMPUTE = """```python
+import csv
+rows = list(csv.DictReader(open("ci.csv", encoding="utf-8")))
+rust = next(r for r in rows if r["language"] == "Rust" and r["size"] == "all")
+print(float(rust["median_s_p50"]) / 60, "minutes")  # the typical Rust repo's push wait
+```"""
+
+
+def _load(folder: Path, name: str) -> dict | None:
+    try:
+        return json.loads((folder / f"{name}-benchmarks.json").read_text(encoding="utf-8"))
+    except (OSError, ValueError):
+        return None
+
+
+def _rows(data: dict, metrics) -> list[dict]:
+    out = []
+    for key, g in sorted((data.get("groups") or {}).items()):
+        if not isinstance(g, dict) or (g.get("repos") or 0) < MIN_REPOS:
+            continue
+        lang, size = key.split("|", 1)
+        row = {"language": lang, "size": size, "repos": g["repos"]}
+        for kind in ("sources", "tiers"):
+            if isinstance(g.get(kind), dict):
+                row.update({f"{kind}_{k.replace(' ', '_')}": v for k, v in g[kind].items()})
+        for m in metrics:
+            q = g.get(m) or {}
+            row[f"{m}_repos"] = q.get("repos", "")
+            row.update({f"{m}_{p}": q.get(p, "") for p in QS})
+            if isinstance(q.get("linearb"), dict):
+                row.update({f"{m}_linearb_{band}": q["linearb"].get(band, "") for band in ("elite", "good", "fair")})
+        out.append(row)
+    return out
+
+
+def _write_csv(path: Path, rows: list[dict]) -> None:
+    if not rows:
+        path.unlink(missing_ok=True)  # an earlier build's file would no longer match the report
+        return
+    # Groups can carry different measures (a band share only where the group had the measure): every column, in order.
+    fields = list(dict.fromkeys(k for r in rows for k in r))
+    with path.open("w", encoding="utf-8", newline="") as f:
+        w = csv.DictWriter(f, fieldnames=fields, restval="")
+        w.writeheader()
+        w.writerows(rows)
+
+
+def _min(secs) -> str:
+    return "-" if secs in ("", None) else f"{float(secs) / 60:.1f} min"
+
+
+def _hrs(h) -> str:
+    return "-" if h in ("", None) else (f"{float(h):.0f} h" if float(h) < 48 else f"{float(h) / 24:.1f} days")
+
+
+def _hrs1(h) -> str:
+    return "-" if h in ("", None) else (f"{float(h):.1f} h" if float(h) < 48 else f"{float(h) / 24:.1f} days")
+
+
+def _pct(x) -> str:
+    return "-" if x in ("", None) else f"{float(x):.0%}"
+
+
+def _table(rows: list[dict], by: str, cols: list[tuple[str, str, object]]) -> list[str]:
+    pick = [r for r in rows if (r["size"] == "all") != (by == "size") and (r["language"] == "all") == (by == "size")]
+    if not pick:
+        return []
+    head = ["Language" if by == "language" else "Stars", "Repos"] + [c[0] for c in cols]
+    lines = ["| " + " | ".join(head) + " |", "|" + "---|" * len(head)]
+    tiers = ["100-1k", "1k-10k", "10k+"]
+    order = (lambda r: -int(r["repos"])) if by == "language" else (lambda r: tiers.index(r["size"]) if r["size"] in tiers else 9)
+    for r in sorted(pick, key=order):
+        name = r["language"] if by == "language" else r["size"]
+        lines.append(f"| {name} | {int(r['repos']):,} | " + " | ".join(fmt(r[key]) for _, key, fmt in cols) + " |")
+    return lines
+
+
+def build_report(bench: Path, out: Path) -> Path:
+    """report.md plus one CSV per benchmark file in `out`."""
+    bench, out = Path(bench), Path(out)
+    out.mkdir(parents=True, exist_ok=True)
+    data = {name: _load(bench, name) for name in METRICS}
+    rows = {}
+    for name, d in data.items():
+        if d:
+            rows[name] = _rows(d, METRICS[name])
+            _write_csv(out / f"{name}.csv", rows[name])
+    ci, flow, com = data.get("ci") or {}, data.get("flow") or {}, data.get("community") or {}
+    built = max((d.get("built_at") or "" for d in data.values() if d), default="")[:10] or "date unknown"
+    month = datetime.fromisoformat(built).strftime("%B %Y") if built[:1].isdigit() else ""
+    newcomers = com.get("newcomers") or {}
+    lines = [f"# State of open-source CI and review flow{f' ({month})' if month else ''}", "",
+             "From a daily crawl of public GitHub repos with 100+ stars: "
+             + ", ".join(x for x in (f"CI from {ci['repos']:,} repos" if ci else "",
+                                    f"review flow from {flow['repos']:,} repos" if flow else "",
+                                    f"first answers from {com['repos']:,} repos" if com else "",
+                                    f"{newcomers['newcomers']:,} newcomers in {newcomers['repos']:,} repos" if newcomers else "") if x)
+             + f". Aggregates only: no repo or person is named. Every number below is in the CSV files next to this report, "
+               f"under {LICENCE}. Groups with fewer than {MIN_REPOS} repos are left out. Built {built}.", ""]
+    all_ci = next((r for r in rows.get("ci", []) if r["language"] == "all" and r["size"] == "all"), None)
+    if all_ci:
+        lines += ["## CI: how long a push waits for its checks", "",
+                  f"The typical repo (the median repo) waits {_min(all_ci['median_s_p50'])} from a push until all its checks "
+                  f"finish; one in ten repos waits more than {_min(all_ci['median_s_p90'])}. The typical repo's pushes are green "
+                  f"on the first try {_pct(all_ci['first_try_share_p50'])} of the time ({int(all_ci['repos']):,} repos).", ""]
+        cols = [("Typical push wait", "median_s_p50", _min), ("Slowest 10% of repos", "median_s_p90", _min),
+                ("Green on first try", "first_try_share_p50", _pct)]
+        lines += _table(rows["ci"], "language", cols) + [""] + _table(rows["ci"], "size", cols) + [""]
+    all_flow = next((r for r in rows.get("flow", []) if r["language"] == "all" and r["size"] == "all"), None)
+    if all_flow:
+        lines += ["## Review flow", "",
+                  f"In the typical repo, a pull request that gets reviewed waits {_hrs(all_flow['first_review_hours_p50'])} for its "
+                  f"first review by a maintainer other than its author. Merged pull requests are merged a median "
+                  f"{_hrs(all_flow['merge_hours_p50'])} after opening"
+                  + (" (this counts PRs merged with no review, and comes from a different set of repos)"
+                     if all_flow["merge_hours_p50"] not in ("", None) and all_flow["first_review_hours_p50"] not in ("", None)
+                     and float(all_flow["merge_hours_p50"]) < float(all_flow["first_review_hours_p50"]) else "")
+                  + f". The typical repo merges {_pct(all_flow['outsider_merge_rate_p50'])} of outside contributors' decided PRs "
+                  f"({int(all_flow['repos']):,} repos).", ""]
+        cols = [("First review (reviewed PRs)", "first_review_hours_p50", _hrs), ("To merge (all merged PRs)", "merge_hours_p50", _hrs),
+                ("Outside PRs merged", "outsider_merge_rate_p50", _pct)]
+        lines += _table(rows["flow"], "language", cols) + [""] + _table(rows["flow"], "size", cols) + [""]
+        agents = (flow.get("agent_prs") or {}).get("all|all")
+        if agents:
+            lines += [f"**Coding agents:** in the typical repo that gets both, {_pct(agents['agent']['merged'])} of pull requests "
+                      f"opened by coding agents (often started by the repo's own maintainers) were merged, against {_pct(agents['outsider']['merged'])} of people's outside PRs "
+                      f"({agents['repos_with_agent_prs']:,} repos; {agents['agent']['decided']:,} agent PRs and "
+                      f"{agents['outsider']['decided']:,} outside PRs decided in the last 180 days).", ""]
+        bt = flow.get("odds_backtest") or {}
+        if (bt.get("mae") or {}).get("calibrated") is not None:
+            lines += [f"**Can a repo's record predict a new outside PR?** A repo's past merge rate for outside PRs runs high for "
+                      f"its next month's outside PRs, because many stay open. Corrected by what happened on other repos with a "
+                      f"similar record, it was off by {bt['mae']['calibrated'] * 100:.0f} points on average ({bt['repos']:,} repos, "
+                      f"{bt['prs']:,} PRs; checked on repos the correction was not built from); the raw rate was off by "
+                      f"{bt['mae']['raw'] * 100:.0f} points and the typical repo's rate alone by {bt['mae']['typical_only'] * 100:.0f}.", ""]
+    all_com = next((r for r in rows.get("community", []) if r["language"] == "all" and r["size"] == "all"), None)
+    if all_com or newcomers:
+        lines += ["## Outside contributors and newcomers", ""]
+        if all_com:
+            lines += [f"The typical repo first answers an outside contributor's PR in {_hrs(all_com['first_answer_hours_p50'])}; "
+                      f"it answers {_pct(all_com['answered_1d_share_p50'])} of them within a day ({int(all_com['repos']):,} repos).", ""]
+        if newcomers.get("by_answer"):
+            lines += [f"Of {newcomers['newcomers']:,} newcomers (a first PR in a repo), the share who came back with new work "
+                      f"within {newcomers.get('return_days', 90)} days, by how long their first PR waited for an answer:", "",
+                      "| First answer | Newcomers | Came back | First PR merged |", "|---|---|---|---|"]
+            lines += [f"| {b['bucket']} | {b['newcomers']:,} | {_pct(b['returned'])} | {_pct(b['merged'])} |" for b in newcomers["by_answer"]]
+            lines += ["", "This is a correlation, not a promise.", ""]
+    all_del = next((r for r in rows.get("delivery", []) if r["language"] == "all" and r["size"] == "all"), None)
+    if all_del:
+        def mid(m):
+            # One decimal: P75s of a few hours lose their meaning at whole hours.
+            value, low, high = (all_del.get(m + k) for k in ("_p50", "_p50_low", "_p50_high"))
+            if value in ("", None):
+                return "-"
+            if float(value) >= 48:
+                return _hrs(value)
+            span = f" ({float(low):.1f}-{float(high):.1f})" if low not in ("", None) and high not in ("", None) else ""
+            return f"{float(value):.1f} h{span}"
+        dl = data.get("delivery") or {}
+        lines += ["## Delivery: pickup and review against LinearB's bands", "",
+                  f"The typical repo's P75 pickup (opening to the first maintainer review) is {mid('pickup_p75_hours')}, its P75 "
+                  f"review (first review to merge) {mid('review_p75_hours')} and its P75 from opening to merge "
+                  f"{mid('cycle_to_merge_p75_hours')} ({int(all_del['repos']):,} repos; ranges are 95% bootstrap ranges of the "
+                  f"median). Inside LinearB's good band (pickup up to 4 h, review up to 14 h): "
+                  f"{_pct(all_del.get('pickup_p75_hours_linearb_good'))} of repos for pickup and "
+                  f"{_pct(all_del.get('review_p75_hours_linearb_good'))} for review. {dl.get('bands_note', '')}.", ""]
+        cols = [("Pickup P75 (typical)", "pickup_p75_hours_p50", _hrs1), ("Review P75 (typical)", "review_p75_hours_p50", _hrs1),
+                ("Pickup inside LinearB's good band", "pickup_p75_hours_linearb_good", _pct)]
+        lines += _table(rows["delivery"], "language", cols) + [""] + _table(rows["delivery"], "size", cols) + [""]
+    dep = data.get("deploys") or {}
+    top = (dep.get("groups") or {}).get("all|all") or {}
+    if top.get("sources"):
+        src, tiers = top["sources"], top.get("tiers") or {}
+        lines += ["## Shipping: how often public repos release", "",
+                  f"Of {int(top['repos']):,} sampled repos, {_pct(src.get('releases'))} show their shipping through GitHub "
+                  f"releases, {_pct(src.get('deployments api'))} through production deployments, and {_pct(src.get('none'))} "
+                  "through neither (they ship from another repo or a service GitHub can't see)."
+                  + (f" Among those that show it, by DORA 2023's frequency tiers: {_pct(tiers.get('elite'))} elite, "
+                     f"{_pct(tiers.get('high'))} high, {_pct(tiers.get('medium'))} medium, {_pct(tiers.get('low'))} low."
+                     if tiers else ""), ""]
+    lines += ["## Method and files", ""]
+    for name, d in data.items():
+        if d:
+            lines.append(f"- **{name}.csv** ({d.get('repos', 0):,} repos, built {(d.get('built_at') or 'date unknown')[:10]}): "
+                         f"{d.get('definition', '')}")
+            if d.get("population") or d.get("gaps"):
+                lines.append(f"  Who is counted: {d.get('population') or '-'}. Not covered: {d.get('gaps') or '-'}.")
+    lines += ["", "Each CSV row is a group of repos (language and star range, or `all`); for each measure it gives the number of "
+                  "repos measured, the 10th, 25th, 50th, 75th and 90th percentiles across them, and a 95% range for the median "
+                  "(the 50th) from 400 bootstrap resamples of the group's repos with a fixed seed, so a rebuild gives the same "
+                  "numbers. A public crawl is not every team: private and enterprise repos are not in it. To recompute a number:",
+              "", RECOMPUTE_SQL, "", RECOMPUTE, ""]
+    (out / "report.md").write_text("\n".join(lines), encoding="utf-8")
+    return out / "report.md"
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(prog="python -m mergescout.bench.report", description="Build the public report from benchmark files.")
+    parser.add_argument("bench", help="folder with the *-benchmarks.json files")
+    parser.add_argument("out", help="folder for report.md and the CSV files")
+    args = parser.parse_args(argv)
+    print(build_report(Path(args.bench), Path(args.out)))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

mergescout/bench/shared.py

@@ -0,0 +1,112 @@
+"""Peer groups from opt-in shared figures: issues labelled "benchmark" on this repo, each carrying one repo's rounded CI
+figures (filed from a CI run summary through the "benchmark" issue form); closing an issue withdraws it. Groups are published only with 10+ repos.
+
+    GH_TOKEN=... python -m mergescout.bench.shared --repo owner/name --out out/shared-benchmarks.json
+"""
+import argparse
+import json
+import os
+import re
+import statistics
+import sys
+from datetime import datetime, timezone
+
+import httpx
+
+MIN_GROUP = 10
+RANKS = (0.10, 0.25, 0.50, 0.75, 0.90)
+BLOCK = re.compile(r"```json\s*(\{.*?\})\s*```", re.S)
+SHARES = ("first_try_share", "fail_share", "rerun_share")
+NUMBERS = ("pushes", "median_s", "p90_s")
+KEYS = {"schema", "kind", "id", "month", "language", "tier", "pushes", "median_s", "p90_s", *SHARES}
+
+
+def read_issue(issue: dict) -> dict | None:
+    """The figures in one issue's body, or None when they're missing or don't look like ours."""
+    found = BLOCK.search(issue.get("body") or "")
+    try:
+        doc = json.loads(found[1]) if found else None
+    except ValueError:
+        return None
+    if not isinstance(doc, dict) or not KEYS <= set(doc) or set(doc) - KEYS - {"repo"}:
+        return None
+    ok = (doc["schema"] == 1 and doc["kind"] == "ci-benchmark" and re.fullmatch(r"[0-9a-f]{16}", str(doc["id"]))
+          and re.fullmatch(r"20\d{2}-(0[1-9]|1[0-2])", str(doc["month"])) and isinstance(doc["language"], str) and len(doc["language"]) < 40
+          and doc["tier"] in ("100-1k", "1k-10k", "10k+")
+          and all(isinstance(doc[k], (int, float)) and not isinstance(doc[k], bool) and 0 <= doc[k] <= 1 for k in SHARES)
+          and all(isinstance(doc[k], int) and not isinstance(doc[k], bool) and 0 <= doc[k] < 10**7 for k in NUMBERS) and doc["pushes"] > 0)
+    return doc if ok else None
+
+
+def _q(values: list[float]) -> dict:
+    s = sorted(values)
+    cuts = statistics.quantiles(s, n=100, method="inclusive") if len(s) > 1 else s * 99
+    return {f"p{int(r * 100)}": round(cuts[int(r * 100) - 1], 4) for r in RANKS}
+
+
+def build(docs: list[dict], now: datetime | None = None) -> dict:
+    latest = {}
+    for d in docs:
+        if d and (d["id"] not in latest or d["month"] > latest[d["id"]]["month"]):
+            latest[d["id"]] = d
+    groups = {}
+    for d in latest.values():
+        for key in (f"{d['language']}|{d['tier']}", f"{d['language']}|all", f"all|{d['tier']}", "all|all"):
+            groups.setdefault(key, []).append(d)
+    now = now or datetime.now(timezone.utc)
+    return {"built_at": now.strftime("%Y-%m-%dT%H:%M:%SZ"), "private": True, "repos": len(latest),
+            "about": "Repos that chose to share rounded CI figures (public and private), latest month each.",
+            "groups": {k: {"repos": len(g), "median_s": _q([d["median_s"] for d in g]),
+                           "first_try_share": _q([d["first_try_share"] for d in g]),
+                           "fail_share": _q([d["fail_share"] for d in g])}
+                       for k, g in sorted(groups.items()) if len(g) >= MIN_GROUP}}
+
+
+def one_per_account(issues: list[dict], now: datetime | None = None) -> list[dict]:
+    """Each GitHub account's newest valid figures only: filed figures can't be verified, so one account can't fill a group.
+    An id belongs to the first account that filed it, so nobody can replace another repo's figures; future months are dropped."""
+    month = f"{now or datetime.now(timezone.utc):%Y-%m}"
+    mine, owner = {}, {}
+    for i in sorted(issues, key=lambda i: i.get("created_at") or ""):
+        doc, who = read_issue(i), ((i.get("user") or {}).get("login") or "").lower()
+        if not doc or not who or doc["month"] > month or owner.setdefault(doc["id"], who) != who:
+            continue
+        mine[who] = doc
+    return list(mine.values())
+
+
+def fetch(repo: str, token: str, client: httpx.Client | None = None) -> list[dict]:
+    client = client or httpx.Client(timeout=60)
+    out = []
+    for page in range(1, 51):
+        got = client.get(f"https://api.github.com/repos/{repo}/issues", headers={"Authorization": f"Bearer {token}"},
+                         params={"labels": "benchmark", "state": "open", "per_page": 100, "page": page})
+        got.raise_for_status()
+        got = got.json()
+        out += [i for i in got if "pull_request" not in i]
+        if len(got) < 100:
+            break
+    return out
+
+
+def main(argv=None) -> int:
+    ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
+    ap.add_argument("--repo", required=True, action="append", help="repeatable: every repo whose issues hold figures")
+    ap.add_argument("--out", required=True)
+    args = ap.parse_args(argv)
+    issues = []
+    for repo in args.repo:
+        # A repo the token can't read yet (still private, or gone) is skipped; the others still count.
+        try:
+            issues += fetch(repo, os.environ["GH_TOKEN"])
+        except httpx.HTTPError as exc:
+            print(f"{repo}: skipped ({exc.__class__.__name__})")
+    out = build(one_per_account(issues))
+    with open(args.out, "w", encoding="utf-8") as f:
+        json.dump(out, f, indent=1)
+    print(f"{len(issues)} issues, {out['repos']} repos, {len(out['groups'])} groups published")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

mergescout/bench/stars.py

@@ -0,0 +1,57 @@
+"""Star growth of crawled repos from GitHub's weekly star history (no stargazer identities), as percentiles by language
+and size: the peer line next to a repo's own star trend."""
+from datetime import datetime
+
+from mergescout.bench.ci import TokenPool, _summary, tier
+
+WEEKS = 12
+MIN_GROUP = 20
+DEFINITION = ("per repo: stars gained in the last 12 complete weeks, and that gain as a share of the stars it had "
+              "before them, from GitHub's weekly star history")
+
+
+def star_stats(rows: list, stars: int | None, now: datetime) -> dict | None:
+    """Gain over the last 12 complete weeks; the running week is left out, and younger repos get no figure."""
+    rows = sorted((r for r in rows or [] if isinstance(r, dict) and isinstance(r.get("week"), (int, float))),
+                  key=lambda r: -r["week"])
+    done = [r for r in rows if r["week"] + 7 * 86400 <= now.timestamp()]
+    if len(done) < WEEKS:
+        return None
+    gained = sum(int(r.get("total") or 0) for r in done[:WEEKS])
+    running = sum(int(r.get("total") or 0) for r in rows if r not in done)
+    before = max((stars or 0) - gained - running, 1)
+    return {"gained_12w": gained, "growth_12w": round(gained / before, 4)}
+
+
+def read_stars(pool: TokenPool, repo: dict, now: datetime) -> dict | None:
+    # One request: the first page holds the newest 30 weeks.
+    resp = pool.get(f"/repos/{repo['name']}/stargazers/history", {})
+    if resp is None:
+        raise RuntimeError("GitHub gave no answer after retries")
+    if resp.status_code >= 400:
+        return None
+    stats = star_stats(resp.json(), repo.get("stars"), now)
+    return {"repo": repo["name"], "language": repo.get("language"), "stars": repo.get("stars"), **stats} if stats else None
+
+
+def aggregate_stars(records: list[dict], now: datetime, top_languages: int = 25) -> dict:
+    """Percentiles by language and size; a group is published only with MIN_GROUP repos."""
+    latest = {}
+    for r in sorted(records, key=lambda r: r.get("day") or ""):
+        latest[r["repo"].lower()] = r
+    recs = list(latest.values())
+    counts = {}
+    for r in recs:
+        counts[r.get("language") or "Other"] = counts.get(r.get("language") or "Other", 0) + 1
+    langs = {name for name, _ in sorted(counts.items(), key=lambda kv: -kv[1])[:top_languages]}
+    groups = {}
+    for r in recs:
+        lang = r.get("language") if r.get("language") in langs else "Other"
+        for key in (f"{lang}|{tier(r.get('stars'))}", f"{lang}|all", f"all|{tier(r.get('stars'))}", "all|all"):
+            groups.setdefault(key, []).append(r)
+    out = {key: {"repos": len(items), "gained_12w": _summary([r["gained_12w"] for r in items]),
+                 "growth_12w": _summary([r["growth_12w"] for r in items])}
+           for key, items in groups.items() if len(items) >= MIN_GROUP}
+    return {"version": 1, "built_at": now.strftime("%Y-%m-%dT%H:%M:%SZ"), "repos": len(recs), "definition": DEFINITION,
+            "population": "public GitHub repos with 100+ stars that pushed in the last 30 days (the CI sampler's daily slice)",
+            "gaps": "stars only: no forks, downloads or traffic; no private repos", "groups": out}

mergescout/cli.py

@@ -0,0 +1,89 @@
+import argparse
+import json
+import logging
+import sys
+
+import httpx
+
+from mergescout.github.releases import ReleaseError
+from mergescout.mcp_server import App, NoSnapshot, build_server, live_status, plain
+from mergescout.tool.live import LiveError
+from mergescout.tool.action_pack import build_action_pack
+from mergescout.tool.refs import parse_ref
+
+
+def _print(data) -> None:
+    print(json.dumps(data, indent=1, ensure_ascii=False))
+
+
+def main(argv: list[str] | None = None) -> int:
+    argv = list(sys.argv[1:] if argv is None else argv) or ["serve"]
+    parser = argparse.ArgumentParser(prog="mergescout", description="Find open-source issues you can get merged.",
+                                     epilog="Also: `stackvitals contribute brief owner/repo`, how changes get merged in a repo.")
+    sub = parser.add_subparsers(dest="cmd", required=True)
+    sub.add_parser("serve", help="Run the MCP server on stdio (what Claude Code and Cursor start)")
+    u = sub.add_parser("update", help="Download the newest snapshot")
+    u.add_argument("--force", action="store_true")
+    f = sub.add_parser("find", help="Best free issues")
+    f.add_argument("--lang")
+    f.add_argument("--topic")
+    f.add_argument("--beginner", action="store_true")
+    f.add_argument("--no-walls", action="store_true")
+    f.add_argument("--limit", type=int, default=10)
+    f.add_argument("--json", action="store_true")
+    for name in ("check", "explain", "pack", "health", "track", "untrack"):
+        sub.add_parser(name).add_argument("ref")
+    o = sub.add_parser("outcome")
+    o.add_argument("ref")
+    o.add_argument("outcome")
+    o.add_argument("--note", default="")
+    sub.add_parser("tracked")
+    sub.add_parser("stats")
+    args = parser.parse_args(argv)
+    if args.cmd == "serve":
+        # Request lines from the HTTP library would only clutter the AI client's log.
+        logging.getLogger("httpx").setLevel(logging.WARNING)
+        build_server(App()).run("stdio")
+        return 0
+    app = App(log=lambda msg: print(msg, file=sys.stderr))
+    try:
+        if args.cmd == "update":
+            print(app.store.update(force=args.force))
+        elif args.cmd == "find":
+            out = app.snapshot().find_issues(args.lang, args.topic, args.beginner, 0.0, not args.no_walls, args.limit)
+            if args.json:
+                _print(out)
+            else:
+                print(f"snapshot {out['snapshot']}  ({out['note']})")
+                for r in out["results"]:
+                    print(f"{r['score']:.2f}  {r['issue']}  {r['title']}\n      {r['outsider_merge']}; {r['review']}"
+                          + (f"; walls: {', '.join(r['walls'])}" if r["walls"] else ""))
+        elif args.cmd == "check":
+            _print(live_status(app, parse_ref(args.ref)))
+        elif args.cmd == "explain":
+            ref = parse_ref(args.ref)
+            _print(app.snapshot().explain(ref.repo, ref.number))
+        elif args.cmd == "pack":
+            _print(build_action_pack(app.snapshot(), parse_ref(args.ref)))
+        elif args.cmd == "health":
+            _print(app.snapshot().repo_health(parse_ref(args.ref).repo, detail=True))
+        elif args.cmd == "track":
+            ref = parse_ref(args.ref)
+            _print(app.tracker.track(ref.key, app.live.issue_status(ref.owner, ref.name, ref.number)))
+        elif args.cmd == "untrack":
+            app.tracker.untrack(parse_ref(args.ref).key)
+            _print({"tracked": sorted(app.tracker.tracked())})
+        elif args.cmd == "tracked":
+            _print({"tracked": app.tracker.tracked(), "changes": app.tracker.updates(app.live)})
+        elif args.cmd == "outcome":
+            _print(app.tracker.report_outcome(parse_ref(args.ref).key, args.outcome, args.note))
+        elif args.cmd == "stats":
+            _print(app.snapshot().stats())
+    except (NoSnapshot, ValueError, LiveError, httpx.HTTPError, ReleaseError) as exc:
+        print(plain(exc), file=sys.stderr)
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

mergescout/config.py

@@ -0,0 +1,35 @@
+import os
+import tempfile
+from dataclasses import dataclass
+from pathlib import Path
+
+
+def load_dotenv(path: Path) -> None:
+    # Read KEY=VALUE lines into the environment without overriding values already set.
+    if not path.exists():
+        return
+    for raw in path.read_text(encoding="utf-8").splitlines():
+        line = raw.strip()
+        if not line or line.startswith("#") or "=" not in line:
+            continue
+        key, value = line.split("=", 1)
+        os.environ.setdefault(key.strip(), value.strip().strip("\"").strip("'"))
+
+
+@dataclass(frozen=True)
+class Config:
+    github_token: str | None
+    data_dir: Path
+
+    def require_token(self) -> str:
+        if not self.github_token:
+            raise RuntimeError("GITHUB_TOKEN is not set. Add it to .env (see .env.example).")
+        return self.github_token
+
+
+def load_config(env_file: Path | None = Path(".env")) -> Config:
+    if env_file is not None:
+        load_dotenv(env_file)
+    data_dir = Path(os.environ.get("MS_DATA_DIR") or Path(tempfile.gettempdir()) / "mergescout")
+    data_dir.mkdir(parents=True, exist_ok=True)
+    return Config(github_token=os.environ.get("GITHUB_TOKEN") or None, data_dir=data_dir)

mergescout/crawl/__main__.py

@@ -0,0 +1,97 @@
+import argparse
+import os
+import sqlite3
+import sys
+import time
+from datetime import datetime, timezone
+from pathlib import Path
+
+from mergescout.config import load_config
+from mergescout.crawl.chain import next_step, write_last_run
+from mergescout.crawl.runner import run_crawl
+from mergescout.crawl.seed import read_watch, seed_queue
+from mergescout.crawl.state import CrawlState
+from mergescout.github.graphql import GitHubGraphQL
+from mergescout.github.identities import identities_from_env
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(prog="python -m mergescout.crawl", description="Crawl repos at full depth.")
+    sub = parser.add_subparsers(dest="cmd", required=True)
+    run = sub.add_parser("run", help="Crawl until the time or points budget runs out (safe to stop and re-run)")
+    run.add_argument("--minutes", type=float, default=50)
+    run.add_argument("--limit", type=int, default=None)
+    run.add_argument("--in-flight", type=int, default=12)
+    run.add_argument("--reserve", type=int, default=300, help="Stop when fewer points than this are left")
+    run.add_argument("--hot-days", type=int, default=1, help="Refresh repos with beginner issues after this many days")
+    run.add_argument("--cold-days", type=int, default=7, help="Refresh other repos after this many days")
+    run.add_argument("--max-wait", type=float, default=120,
+                     help="Longest wait for GitHub's rate limit; also capped by the time left in the run")
+    run.add_argument("--grace", type=float, default=8,
+                     help="Minutes after --minutes that running work may still spend sitting out a wait")
+    run.add_argument("--full-every-days", type=int, default=30, help="Redo a full crawl this often; refreshes in between")
+    sub.add_parser("status", help="Show the queue and the last runs")
+    seed = sub.add_parser("seed", help="Queue active repos from the sweep, beginner-friendly repos first")
+    seed.add_argument("--min-stars", type=int, default=100)
+    seed.add_argument("--active-days", type=int, default=90)
+    seed.add_argument("--watch", type=Path, default=None, help="File of owner/name lines to crawl first")
+    sub.add_parser("requeue-failed", help="Give failed repos another try (up to 3 attempts)")
+    bk = sub.add_parser("backup", help="Copy the state file safely, even while it is in use")
+    bk.add_argument("dest")
+    nx = sub.add_parser("next", help="Print which workflow the relay starts next: discover, crawl, crawl-later or none")
+    nx.add_argument("--after-discover", action="store_true")
+    args = parser.parse_args(argv)
+
+    cfg = load_config()
+    state = CrawlState(cfg.data_dir / "crawl.sqlite")
+    now = datetime.now(timezone.utc)
+    if args.cmd == "status":
+        print(f"repos={state.counts()}")
+        for r in state.db.execute("SELECT run_id, started_at, finished_at, repos, points, reason FROM runs "
+                                  "ORDER BY started_at DESC LIMIT 5"):
+            print("run " + " ".join(str(v) for v in tuple(r)))
+        tracked = state.db.execute("SELECT COUNT(*) FROM issue_state").fetchone()[0]
+        print(f"tracked issues={tracked:,}")
+        return 0
+    if args.cmd == "backup":
+        dest = sqlite3.connect(args.dest)
+        with dest:
+            state.db.backup(dest)
+        dest.close()
+        return 0
+    if args.cmd == "next":
+        print(next_step(state, cfg.data_dir, now, after_discover=args.after_discover))
+        return 0
+    if args.cmd == "requeue-failed":
+        print(f"requeued {state.requeue_failed(3)}")
+        return 0
+    if args.cmd == "seed":
+        seed_queue(state, cfg.data_dir, now, min_stars=args.min_stars, active_days=args.active_days,
+                   watch=read_watch(args.watch))
+        return 0
+    try:
+        token = cfg.require_token()
+    except RuntimeError as exc:
+        print(str(exc), file=sys.stderr)
+        return 1
+    run_id = os.environ.get("MS_RUN_ID") or now.strftime("%Y%m%dT%H%M%SZ")
+    print(f"requeued stale={state.requeue_stale(120)} failed={state.requeue_failed(3)}", flush=True)
+    # Any wait GitHub asks for is sat out if it still fits in this run; otherwise that identity steps aside.
+    end = time.monotonic() + (args.minutes + args.grace) * 60
+
+    def allowed_wait() -> float:
+        return max(0.0, min(args.max_wait, end - time.monotonic()))
+    # Personal token plus, when set, the GitHub App and workflow tokens: each with its own limits and wait.
+    identities = identities_from_env(token, allowed_wait, client=GitHubGraphQL)
+    out = run_crawl(state, identities, cfg.data_dir, run_id,
+                    minutes=args.minutes, in_flight=args.in_flight, reserve=args.reserve, limit=args.limit,
+                    hot_days=args.hot_days, cold_days=args.cold_days, full_every_days=args.full_every_days,
+                    log=lambda msg: print(msg, flush=True))
+    for i in identities:
+        print(f"github {i.name}: {i.gate.summary()}", flush=True)
+    write_last_run(cfg.data_dir, out)
+    return 1 if out["repos"] == 0 and out["failed"] else 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())