stackvitals 0.1.0__py3-none-any.whl

mergescout/signals/__main__.py

@@ -0,0 +1,96 @@
+import argparse
+import os
+import sys
+import tarfile
+from datetime import datetime, timezone
+from pathlib import Path
+
+import duckdb
+
+from mergescout.config import load_config
+from mergescout.github.releases import Releases
+from mergescout.signals.evidence import TABLES, build_all
+from mergescout.signals.silver import load_silver
+from mergescout.store.sync import fetch_bronze
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(prog="python -m mergescout.signals", description="Build Tier 1 signals.")
+    sub = parser.add_subparsers(dest="cmd", required=True)
+    b = sub.add_parser("build", help="Read all crawl batches and write signal tables as Parquet")
+    b.add_argument("--from-releases", action="store_true", help="Download crawl bundles from releases first")
+    b.add_argument("--out", default=None)
+    c = sub.add_parser("check", help="Hand-check identity on 30 repos against owner files and public org members")
+    c.add_argument("--signals", required=True)
+    c.add_argument("--out", required=True)
+    o = sub.add_parser("opendigger", help="Compare review times with OpenDigger")
+    o.add_argument("--signals", required=True)
+    o.add_argument("--repos", type=int, default=100)
+    args = parser.parse_args(argv)
+
+    if args.cmd in ("check", "opendigger"):
+        return _validate(args)
+
+    cfg = load_config()
+    out = Path(args.out) if args.out else cfg.data_dir / "signals"
+    con = prepare(cfg, out, args.from_releases, datetime.now(timezone.utc))
+    if con is None:
+        return 1
+    write_bundle(con, out)
+    con.close()
+    return 0
+
+
+def prepare(cfg, out: Path, from_releases: bool, now: datetime) -> duckdb.DuckDBPyConnection | None:
+    # Silver + every signal table; None when there is no crawl data to build from.
+    out.mkdir(parents=True, exist_ok=True)
+    if from_releases:
+        # Same store as the crawler's `store` command: MS_STORE_REPO, else the running repo.
+        token = os.environ.get("MS_RELEASE_TOKEN")
+        repo = os.environ.get("MS_STORE_REPO") or os.environ.get("GITHUB_REPOSITORY")
+        if not token or not repo:
+            print("Set MS_RELEASE_TOKEN and MS_STORE_REPO (or GITHUB_REPOSITORY) to read crawl releases.",
+                  file=sys.stderr)
+            return None
+        fetch_bronze(Releases(token, repo), cfg.data_dir, log=lambda msg: print(msg, flush=True))
+    # A file database lets DuckDB spill to disk instead of running out of memory on the runner.
+    con = duckdb.connect(str(out / "work.duckdb"))
+    con.execute("SET memory_limit = '4GB'")
+    silver = load_silver(con, [cfg.data_dir, cfg.data_dir / "bronze"])
+    print(f"silver: {silver}", flush=True)
+    empty = [t for t in ("repo_meta", "prs", "issues") if not silver.get(t)]
+    if empty:
+        # A wrong store repo or token finds nothing; publishing that would replace the good bundle.
+        print(f"No crawl data for {', '.join(empty)}; not building signals.", file=sys.stderr)
+        con.close()
+        return None
+    print(f"signals: {build_all(con, now)}", flush=True)
+    return con
+
+
+def write_bundle(con: duckdb.DuckDBPyConnection, out: Path) -> Path:
+    with tarfile.open(out / "signals.tar", "w") as tar:
+        for table in TABLES:
+            path = out / f"{table}.parquet"
+            quoted = path.as_posix().replace("'", "''")
+            con.execute(f"COPY {table} TO '{quoted}' (FORMAT parquet, COMPRESSION zstd)")
+            tar.add(path, arcname=path.name)
+    return out / "signals.tar"
+
+
+def _validate(args) -> int:
+    import httpx
+
+    from mergescout.github.graphql import GitHubGraphQL
+    from mergescout.signals.check import run_check, run_opendigger
+    if args.cmd == "opendigger":
+        with httpx.Client(timeout=60) as http:
+            return 0 if run_opendigger(Path(args.signals), args.repos, http)["n"] else 1
+    token = load_config().require_token().strip()
+    with httpx.Client(timeout=60, headers={"Authorization": f"Bearer {token}"}) as http:
+        run_check(Path(args.signals), Path(args.out), GitHubGraphQL(token, max_wait=120), http)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

mergescout/signals/bump_ledger.py

@@ -0,0 +1,258 @@
+"""The upgrade ledger: how Dependabot and Renovate PRs for each package ended across the crawl (merged, failed CI
+against these repos' usual, needed extra commits, closed, reverted), with counts so readers can judge the evidence."""
+import re
+from datetime import datetime, timedelta
+
+import duckdb
+import pyarrow as pa
+
+from mergescout.signals.bumps import MANAGERS, _norm, parse_bump
+
+WINDOW_DAYS = 180
+REVERT_DAYS = 14
+# Backtest flags. Closed matches the dossier (40% predicted 3x later closes on the crawl sample); CI uses the raw
+# failure share until enough CI results exist to test the dossier's comparison with each repo's usual rate.
+CLOSED_FLAG = 0.40
+CI_FLAG = 0.40
+CUTOFFS = (0.30, 0.40, 0.50, 0.60)
+# Cells this thin are left out of the published file; readers see "not enough evidence" below 30 PRs from 10 repos.
+MIN_PUBLISH = 3
+DEFINITION = ("Dependabot and Renovate PRs opened in the last 180 days in crawled repos, per ecosystem, package and step "
+              "(major, minor, patch; 0.x minor steps count as major). Merged; closed without merging (a bot closing its own "
+              "PR for a newer version is not counted); CI failed on the last commit, next to the failures these repos' "
+              "other bot PRs would predict; needed extra commits; reverted or rolled back within 14 days (a revert PR, GitHub's "
+              "revert branch, or a branch naming the rollback). Repos that never merge bot PRs "
+              "are left out. Package and versions come from titles, or from Dependabot branch names for older PRs.")
+SUPERSEDE = timedelta(days=2)
+# Renovate's "update dependency X" does not say npm or PyPI; the repo's main language does (marked as inferred).
+LANGUAGE_ECOSYSTEM = {"Python": "pypi", "JavaScript": "npm", "TypeScript": "npm", "Vue": "npm", "Svelte": "npm",
+                      "Astro": "npm", "CoffeeScript": "npm"}
+# Renovate also names runners, runtimes and Docker images this way ("update dependency ubuntu to v24"): never guessed.
+NOT_PACKAGES = {"ubuntu", "debian", "alpine", "centos", "fedora", "windows", "macos", "node", "nodejs", "node.js", "python",
+                "golang", "go", "rust", "ruby", "java", "openjdk", "temurin", "dotnet", "nginx", "httpd", "postgres",
+                "postgresql", "redis", "mysql", "mariadb", "mongo", "mongodb", "busybox", "docker", "kubernetes", "terraform"}
+REVERT = re.compile(r'^\s*revert\s+"(?P<inner>.+)"\s*$', re.I)
+BRANCH_VERSION = re.compile(r"^(?P<pkg>.+?)-(?P<ver>v?\d+(?:\.\d+)+[\w.+-]*)$")
+RANGE_OPERATOR = re.compile(r"(?:-(?:gte|lte|gt|lt|neq|eq|tw|approx))+$")
+# Names that can exist in each registry; anything else (a hook URL, "jdx/mise" read as PyPI) is not attributed.
+VALID_NAME = {"pypi": re.compile(r"^[a-z0-9]([a-z0-9._-]*[a-z0-9])?$", re.I),
+              "npm": re.compile(r"^(@[a-z0-9][\w.~-]*/)?[a-z0-9~][\w.~-]*$", re.I)}
+FAILED = ("FAILURE", "ERROR")
+
+
+def parse_branch(head: str | None) -> tuple[str, str, str] | None:
+    """(ecosystem, package, new version) from a Dependabot branch, only where it can't be misread: PyPI names never
+    contain "/", and an npm branch without folders; scoped npm names and folders look alike, so those are skipped."""
+    parts = (head or "").split("/")
+    if len(parts) < 3 or parts[0].lower() != "dependabot" or parts[-1].startswith("multi-"):
+        return None
+    eco, m = MANAGERS.get(parts[1]), BRANCH_VERSION.match(parts[-1])
+    if not m or eco not in ("pypi", "npm") or (eco == "npm" and len(parts) != 3):
+        return None
+    # Range updates put the operator in the branch: "setuptools-scm-gte-8.0.0", "boto3-approx-eq-1.35.0"; extras
+    # ("uvicorn[standard]" -> "uvicorn-standard-") can't be told apart from the name, so those are skipped.
+    name = RANGE_OPERATOR.sub("", m["pkg"])
+    return (eco, _norm(eco, name), m["ver"]) if valid_name(eco, name) and not name.endswith("-") else None
+
+
+def valid_name(eco: str | None, name: str | None) -> bool:
+    rule = VALID_NAME.get(eco or "")
+    return bool(name) and (rule is None or bool(rule.match(name)))
+
+
+def _key(name: str | None) -> str:
+    # Names compared loosely when matching a revert to its bump (the revert title has no ecosystem).
+    return re.sub(r"[-_.]+", "-", (name or "").lower())
+
+
+def build_bump_prs(con: duckdb.DuckDBPyConnection) -> int:
+    """One row per dependency-bot PR: parsed from its title, else from its branch (crawl files before titles were kept)."""
+    rows = con.execute("""
+        SELECT p.repo_id, p.number, p.created_at, p.merged_at, p.closed_at, p.state, p.author_login, p.head_ref, p.title,
+               p.ci_state, p.commit_count, rm.language
+        FROM prs p LEFT JOIN repo_meta rm USING (repo_id)
+        WHERE regexp_matches(lower(coalesce(p.author_login, '')), '^(dependabot|renovate)')
+           OR regexp_matches(lower(coalesce(p.head_ref, '')), '^(dependabot|renovate)')""").fetchall()
+    out = []
+    for repo, number, created, merged, closed, state, login, head, title, ci, commits, lang in rows:
+        if title:
+            b, source = parse_bump(title, head, login), "title"
+            if b is None:
+                continue  # Renovate's setup and pin PRs are not upgrades
+        else:
+            found = parse_branch(head)
+            if not found:
+                continue
+            b = {"bot": "dependabot", "ecosystem": found[0], "package": found[1], "from_version": None,
+                 "to_version": found[2], "bump": None, "grouped": False, "security": False, "closed_by_bot": False}
+            source = "branch"
+        inferred = b["ecosystem"] is None and b["package"] is not None and lang in LANGUAGE_ECOSYSTEM \
+            and b["package"].lower() not in NOT_PACKAGES
+        eco = LANGUAGE_ECOSYSTEM[lang] if inferred else b["ecosystem"]
+        if b["package"] and not valid_name(eco, b["package"]):
+            b = {**b, "package": None}
+        out.append({"repo_id": repo, "number": number, "created_at": created, "merged_at": merged, "closed_at": closed,
+                    "state": state, "bot": b["bot"], "ecosystem": eco, "package": _norm(eco, b["package"]) if b["package"] else None,
+                    "from_version": b["from_version"], "to_version": b["to_version"], "bump": b["bump"],
+                    "grouped": b["grouped"], "security": b["security"], "ci_state": ci, "commit_count": commits,
+                    "superseded": bool(b["closed_by_bot"]), "reverted": False, "eco_inferred": inferred, "source": source})
+    _mark_superseded(out)
+    _mark_reverts(con, out)
+    con.register("bump_prs_rows", pa.Table.from_pylist(out, schema=SCHEMA))
+    con.execute("CREATE OR REPLACE TABLE bump_prs AS SELECT * FROM bump_prs_rows")
+    con.unregister("bump_prs_rows")
+    return len(out)
+
+
+def _mark_superseded(rows: list[dict]) -> None:
+    # A bot closes its own PR when a newer version arrives: a close next to a new PR for the same package isn't a rejection.
+    opened = {}
+    for r in rows:
+        if r["package"]:
+            opened.setdefault((r["repo_id"], r["package"]), []).append(r["created_at"])
+    for r in rows:
+        if r["package"] and r["merged_at"] is None and r["closed_at"] is not None:
+            later = [t for t in opened[(r["repo_id"], r["package"])] if t > r["created_at"]]
+            if any(abs(t - r["closed_at"]) <= SUPERSEDE for t in later):
+                r["superseded"] = True
+
+
+def _mark_reverts(con: duckdb.DuckDBPyConnection, rows: list[dict]) -> None:
+    # A merged 'Revert "Bump X from A to B"' within 14 days of the bump's merge is the strongest sign it broke something.
+    reverts = con.execute("""SELECT repo_id, merged_at, title FROM prs
+                             WHERE merged_at IS NOT NULL AND title IS NOT NULL AND lower(title) LIKE 'revert%'""").fetchall()
+    by_bump = {}
+    for r in rows:
+        if r["package"] and r["merged_at"] is not None:
+            by_bump.setdefault((r["repo_id"], _key(r["package"]), r["to_version"]), []).append(r)
+    for repo, when, title in reverts:
+        m = REVERT.match(title)
+        inner = parse_bump(m["inner"], "dependabot/", "dependabot") if m else None
+        if not inner or not inner["package"]:
+            continue
+        for r in by_bump.get((repo, _key(inner["package"]), inner["to_version"]), []):
+            if r["merged_at"] <= when <= r["merged_at"] + timedelta(days=REVERT_DAYS):
+                r["reverted"] = True
+    _mark_branch_rollbacks(con, rows)
+
+
+# GitHub's Revert button names the branch after the PR it undoes: revert-<number>-<that PR's branch>.
+REVERT_BRANCH = re.compile(r"(?i)^revert-(\d+)-")
+
+
+def _branch_name(package: str) -> str:
+    """How a branch names the package: a scoped npm name needs both parts (@types/node as types-node, never node), a Go
+    module its last part without a /v2 suffix, any other name all of it."""
+    parts = package.lstrip("@").split("/")
+    if len(parts) > 1 and re.fullmatch(r"v\d+", parts[-1]):
+        parts = parts[:-1]
+    if package.startswith("@"):
+        return "[-/_]".join(re.escape(p) for p in parts)
+    return re.escape(parts[-1])
+
+
+def _mark_branch_rollbacks(con: duckdb.DuckDBPyConnection, rows: list[dict]) -> None:
+    """Rollbacks the crawl keeps no title for, from the merged PR's branch: GitHub's revert-<number> branches, and
+    branches naming a rollback of the package followed by a version or nothing (rollback-litellm, downgrade-esbuild-0.24);
+    on the crawl's sample this found 27 against the titles' 6 (bench/accuracy/deps)."""
+    merged = {(r["repo_id"], r["number"]): r for r in rows if r["merged_at"] is not None}
+    by_repo = {}
+    for r in merged.values():
+        if r["package"]:
+            by_repo.setdefault(r["repo_id"], []).append(r)
+    found = con.execute("""SELECT repo_id, merged_at, head_ref FROM prs
+                           WHERE merged_at IS NOT NULL AND head_ref IS NOT NULL
+                             AND regexp_matches(lower(head_ref), '(revert|downgrade|roll-?back)')""").fetchall()
+    window = timedelta(days=REVERT_DAYS)
+    for repo, when, ref in found:
+        if m := REVERT_BRANCH.match(ref):
+            r = merged.get((repo, int(m[1])))
+            if r and r["merged_at"] <= when <= r["merged_at"] + window:
+                r["reverted"] = True
+            continue
+        for r in by_repo.get(repo, ()):
+            name = _branch_name(r["package"])
+            if r["merged_at"] <= when <= r["merged_at"] + window and re.search(
+                    rf"(?i)(revert|downgrade|roll-?back)[-/_]+{name}([-/_]v?\d|$)", ref):
+                r["reverted"] = True
+
+
+SCHEMA = pa.schema([
+    ("repo_id", pa.int64()), ("number", pa.int32()), ("created_at", pa.timestamp("us", tz="UTC")),
+    ("merged_at", pa.timestamp("us", tz="UTC")), ("closed_at", pa.timestamp("us", tz="UTC")), ("state", pa.string()),
+    ("bot", pa.string()), ("ecosystem", pa.string()), ("package", pa.string()), ("from_version", pa.string()),
+    ("to_version", pa.string()), ("bump", pa.string()), ("grouped", pa.bool_()), ("security", pa.bool_()),
+    ("ci_state", pa.string()), ("commit_count", pa.int32()), ("superseded", pa.bool_()), ("reverted", pa.bool_()),
+    ("eco_inferred", pa.bool_()), ("source", pa.string())])
+
+
+def build_bump_ledger(con: duckdb.DuckDBPyConnection, now: datetime, window_days: int = WINDOW_DAYS) -> int:
+    """Per ecosystem, package and step size over the window: outcomes with counts, and the CI failures these repos' other
+    bot PRs would predict (each PR's repo rate, leaving the PR out), so a flaky repo doesn't blame a package."""
+    con.execute("""
+        CREATE OR REPLACE TABLE bump_ledger AS
+        WITH quiet AS (
+            -- Repos that never merge bot PRs ignore them; their closes say nothing about the package.
+            SELECT repo_id FROM bump_prs GROUP BY repo_id HAVING count(*) >= 10 AND count(merged_at) = 0),
+        base AS (
+            SELECT repo_id, count(ci_state) AS n, count(*) FILTER (WHERE ci_state IN ('FAILURE', 'ERROR')) AS f
+            FROM bump_prs GROUP BY repo_id),
+        b AS (
+            SELECT p.*, (p.ci_state IN ('FAILURE', 'ERROR'))::INT AS failed, base.n, base.f
+            FROM bump_prs p JOIN base USING (repo_id)
+            WHERE p.package IS NOT NULL AND p.created_at >= $since AND p.repo_id NOT IN (SELECT repo_id FROM quiet))
+        SELECT ecosystem, package, coalesce(bump, 'unknown') AS bump,
+               count(*) AS prs, count(DISTINCT repo_id) AS repos,
+               count(merged_at) AS merged,
+               count(*) FILTER (WHERE merged_at IS NULL AND state = 'CLOSED' AND NOT superseded) AS closed_unmerged,
+               count(ci_state) AS ci_known, sum(failed) FILTER (WHERE ci_state IS NOT NULL) AS ci_failed,
+               sum((f - failed)::DOUBLE / (n - 1)) FILTER (WHERE ci_state IS NOT NULL AND n > 1) AS expected_failed,
+               count(*) FILTER (WHERE commit_count > 1) AS extra_commits,
+               count(*) FILTER (WHERE reverted) AS reverted,
+               bool_or(eco_inferred) AS eco_inferred, min(created_at) AS first_at, max(created_at) AS last_at
+        FROM b GROUP BY ALL""", {"since": now - timedelta(days=window_days)})
+    return con.execute("SELECT count(*) FROM bump_ledger").fetchall()[0][0]
+
+
+def ledger_json(con: duckdb.DuckDBPyConnection, now: datetime) -> dict:
+    """The ledger as the published file: every cell with a few PRs, with its counts."""
+    cols = ("ecosystem", "package", "bump", "prs", "repos", "merged", "closed_unmerged", "ci_known", "ci_failed",
+            "expected_failed", "extra_commits", "reverted", "eco_inferred")
+    rows = con.execute(f"SELECT {', '.join(cols)} FROM bump_ledger WHERE prs >= $min ORDER BY ecosystem, package, bump",
+                       {"min": MIN_PUBLISH}).fetchall()
+    cells = [{k: (round(v, 2) if isinstance(v, float) else v) for k, v in zip(cols, r)} for r in rows]
+    return {"built_at": now.strftime("%Y-%m-%dT%H:%M:%SZ"), "definition": DEFINITION, "window_days": WINDOW_DAYS,
+            "cells": cells}
+
+
+def bump_backtest(con: duckdb.DuckDBPyConnection, now: datetime) -> dict:
+    """Did packages flagged on older upgrades (opened 360-120 days ago) see more trouble on later ones (90-30 days ago)?
+    Logged by the daily build so the dossier's thresholds are checked as evidence grows."""
+    def cells(lo: int, hi: int) -> dict:
+        rows = con.execute("""
+            SELECT ecosystem, package, count(*), count(DISTINCT repo_id),
+                   count(*) FILTER (WHERE merged_at IS NULL AND state = 'CLOSED' AND NOT superseded),
+                   count(*) FILTER (WHERE merged_at IS NOT NULL OR (state = 'CLOSED' AND NOT superseded)),
+                   count(ci_state), count(*) FILTER (WHERE ci_state IN ('FAILURE', 'ERROR'))
+            FROM bump_prs WHERE package IS NOT NULL AND created_at >= $lo AND created_at < $hi GROUP BY ALL""",
+            {"lo": now - timedelta(days=lo), "hi": now - timedelta(days=hi)}).fetchall()
+        return {(r[0], r[1]): r[2:] for r in rows}
+    early, later = cells(360, 120), cells(90, 30)
+    strong = {k for k, v in early.items() if v[0] >= 30 and v[1] >= 10}
+    out = {}
+    for name, cut, flag, num, den in (("closed", CLOSED_FLAG, lambda v, c: v[3] and v[2] / v[3] >= c, 2, 3),
+                                      ("ci", CI_FLAG, lambda v, c: v[4] >= 20 and v[5] / v[4] >= c, 5, 4)):
+        def share(keys, num=num, den=den):
+            d = sum(later[k][den] for k in keys if k in later)
+            return round(sum(later[k][num] for k in keys if k in later) / d, 3) if d else None
+
+        def run(c, flag=flag):
+            flagged = {k for k in strong if flag(early[k], c)}
+            f, o = share(flagged), share(strong - flagged)
+            return {"flagged": len(flagged), "later_flagged": f, "later_others": o,
+                    "ratio": round(f / o, 2) if f is not None and o else None}
+        # Every cutoff, so the dossier's threshold is picked from data (40% gave 1.72x on 2026-10-19, below the 2x bar).
+        out[name] = {"strong": len(strong), **run(cut),
+                     "by_cutoff": {f"{int(c * 100)}%": {k: v for k, v in run(c).items() if k in ("flagged", "ratio")}
+                                   for c in CUTOFFS}}
+    return out

mergescout/signals/bumps.py

@@ -0,0 +1,104 @@
+"""Dependency-update PRs (Dependabot, Renovate) read from their titles and branch names: which package, which versions,
+how big a step. The upgrade ledger counts how these PRs ended across the crawl."""
+import re
+
+from mergescout.crawl.queries import is_dep_bot
+
+# Dependabot's branch says the package manager: dependabot/<manager>/<path>/<package>-<version>.
+MANAGERS = {"npm_and_yarn": "npm", "npm": "npm", "bun": "npm", "pip": "pypi", "pipenv": "pypi", "poetry": "pypi",
+            "uv": "pypi", "pip_compile": "pypi", "github_actions": "actions", "cargo": "cargo", "go_modules": "go",
+            "maven": "maven", "gradle": "maven", "bundler": "rubygems", "composer": "packagist", "nuget": "nuget",
+            "docker": "docker"}
+# Renovate names the kind of dependency in its title ("update rust crate serde", "update actions/checkout action").
+RENOVATE_KINDS = {"rust crate": "cargo", "module": "go", "action": "actions", "docker tag": "docker", "docker digest": "docker",
+                  "digest": None, "container image": "docker", "image": "docker", "helm release": "helm", "chart": "helm",
+                  "gem": "rubygems", "plugin": None, "pre-commit hook": None, "package": None, "dependency": None}
+KINDS = "rust crate|module|docker tag|docker digest|container image|image|helm release|chart|gem|plugin|pre-commit hook|package|dependency"
+WHATS = "action|monorepo|packages|docker tag|docker digest|digest"
+
+# Leading noise: emoji and a "chore(ci)(deps):"-style prefix of any language; "[Security]" tags are removed first.
+PREFIX = re.compile(r"^\W*(?:[^:]{1,40}?:\s+)?")
+SECURITY = re.compile(r"\s*\[security\]\s*", re.I)
+AUTOCLOSED = re.compile(r"\s+-\s+autoclosed$", re.I)
+BASE_BRANCH = re.compile(r"\s+\((?!major\))[^()]*\)$", re.I)
+NOT_UPDATE = re.compile(r"^(?:configure renovate|migrate renovate config|pin\b)", re.I)
+
+DEPENDABOT = re.compile(r"^bump (?P<pkg>\S+) from (?P<old>\S+) to (?P<new>\S+)(?: in .*)?$", re.I)
+DEPENDABOT_RANGE = re.compile(r"^update (?P<pkg>\S+) requirement from (?P<old>.+?) to (?P<new>.+?)(?: in .*)?$", re.I)
+DEPENDABOT_GROUP = re.compile(r"^bump the \S+ group|^bump \S+(?:,| and )|^bump \S+ packages from", re.I)
+ARROW = re.compile(rf"^update (?:(?P<kind>{KINDS}) )?(?P<pkg>\S+) \(\s*(?P<old>\S+)\s*(?:→|➔|->)\s*(?P<new>\S+)\s*\)", re.I)
+RENOVATE = re.compile(rf"^update (?:(?P<kind>{KINDS}) )?(?P<pkg>\S+)(?: (?P<what>{WHATS}))?(?: from (?P<old>\S+))? to (?P<new>\S+)",
+                      re.I)
+RENOVATE_GROUP = re.compile(r"^update all |^lock file maintenance|monorepo", re.I)
+VERSION = re.compile(r"^[\^~=<>]*v?(\d+)(?:\.(\d+))?(?:\.(\d+))?")
+
+
+def _norm(eco: str | None, name: str) -> str:
+    # PyPI names compare case- and separator-insensitively (PEP 503); others are kept as written.
+    return re.sub(r"[-_.]+", "-", name.lower()) if eco == "pypi" else name
+
+
+def _version(text: str) -> str:
+    # "v14.0.3" -> "14.0.3"; ranges ("^0.57.0") stay as written.
+    return text[1:] if re.match(r"^v\d", text) else text
+
+
+def bump_size(old: str | None, new: str | None) -> str | None:
+    """major / minor / patch from two versions; 0.x minor steps count as major, since semver lets them break."""
+    a, b = VERSION.match(old or ""), VERSION.match(new or "")
+    if not a or not b:
+        return None
+    x = [int(v or 0) for v in a.groups()]
+    y = [int(v or 0) for v in b.groups()]
+    if x[0] != y[0] or (x[0] == 0 and x[1] != y[1]):
+        return "major"
+    return "minor" if x[1] != y[1] else "patch"
+
+
+def parse_bump(title: str | None, head: str | None, login: str | None) -> dict | None:
+    """One dependency-update PR as {bot, ecosystem, package, from_version, to_version, bump, grouped, security,
+    closed_by_bot}, or None when it isn't an update (another author, Renovate's setup or pin PRs)."""
+    if not is_dep_bot(login, head) or not title:
+        return None
+    bot = "dependabot" if (head or "").lower().startswith("dependabot") or (login or "").lower().startswith("dependabot") \
+        else "renovate"
+    text = SECURITY.sub(" ", title).strip()
+    closed_by_bot = AUTOCLOSED.search(text) is not None
+    text = PREFIX.sub("", AUTOCLOSED.sub("", text)).strip()
+    if NOT_UPDATE.match(text):
+        return None
+    out = {"bot": bot, "ecosystem": None, "package": None, "from_version": None, "to_version": None, "bump": None,
+           "grouped": False, "security": SECURITY.search(title) is not None, "closed_by_bot": closed_by_bot}
+    if bot == "dependabot":
+        parts = (head or "").split("/")
+        eco = out["ecosystem"] = MANAGERS.get(parts[1]) if len(parts) > 1 else None
+        # One package in several folders comes on a "multi-" branch without versions in the title.
+        if DEPENDABOT_GROUP.match(text) or "/multi-" in (head or ""):
+            return {**out, "grouped": True}
+        if m := DEPENDABOT.match(text):
+            return {**out, "package": _norm(eco, m["pkg"]), "from_version": m["old"], "to_version": m["new"],
+                    "bump": bump_size(m["old"], m["new"])}
+        if m := DEPENDABOT_RANGE.match(text):
+            return {**out, "package": _norm(eco, m["pkg"]), "from_version": m["old"], "to_version": m["new"], "bump": "range"}
+        return out
+    if m := ARROW.match(text):
+        eco = RENOVATE_KINDS.get((m["kind"] or "image").lower())
+        return {**out, "ecosystem": eco, "package": _norm(eco, m["pkg"]), "from_version": _version(m["old"]),
+                "to_version": _version(m["new"]), "bump": bump_size(m["old"], m["new"])}
+    text = BASE_BRANCH.sub("", text)
+    grouped = RENOVATE_GROUP.search(text) is not None
+    m = RENOVATE.match(text)
+    if not m:
+        # A Renovate update without "to <version>" is a named group ("update frontend dependencies").
+        return {**out, "grouped": True}
+    kind = (m["what"] or m["kind"] or "dependency").lower()
+    eco, new = RENOVATE_KINDS.get(kind), _version(m["new"])
+    if "digest" in kind:
+        bump = "digest"
+    elif m["old"]:
+        bump = bump_size(m["old"], new)
+    else:
+        # Renovate writes only the new major for major updates ("to v19"), the full version otherwise.
+        bump = "major" if "(major)" in text.lower() or re.fullmatch(r"v?\d+", m["new"]) else "nonmajor"
+    return {**out, "ecosystem": eco, "package": None if grouped else _norm(eco, m["pkg"]), "grouped": grouped,
+            "from_version": _version(m["old"]) if m["old"] else None, "to_version": new, "bump": bump}

mergescout/signals/check.py

@@ -0,0 +1,141 @@
+import csv
+import random
+import re
+from concurrent.futures import ThreadPoolExecutor
+from datetime import datetime, timezone
+from pathlib import Path
+
+import duckdb
+import httpx
+
+from mergescout.crawl.queries import RL
+from mergescout.owners import owner_logins
+
+OWNER_FILES = (".github/CODEOWNERS", "CODEOWNERS", "docs/CODEOWNERS", "MAINTAINERS", "MAINTAINERS.md", "OWNERS")
+
+
+def sample_repos(repo_stats: list[dict], n: int = 30, seed: int = 7) -> list[int]:
+    tiers = [(100, 999), (1000, 9999), (10000, None)]
+    rng = random.Random(seed)
+    picked = []
+    for lo, hi in tiers:
+        pool = sorted(r["repo_id"] for r in repo_stats if r["outsider_prs"] >= 20 and r["stars"] >= lo
+                      and (hi is None or r["stars"] <= hi))
+        picked += rng.sample(pool, min(n // len(tiers), len(pool)))
+    return picked
+
+
+def score_identity(predicted: dict[str, str], confirmed_insiders: set[str], pr_counts: dict[str, int]) -> dict:
+    insiders = [u for u, k in predicted.items() if k == "insider"]
+    missed = sorted(u for u, k in predicted.items() if k == "outsider" and u in confirmed_insiders)
+    outsider_prs = sum(pr_counts.get(u, 0) for u, k in predicted.items() if k == "outsider")
+    confirmed = [u for u in insiders if u in confirmed_insiders]
+    # Unconfirmed insiders are not errors: most employees keep their org membership private.
+    return {"insider_confirmed": len(confirmed), "insider_checkable": len(confirmed),
+            "insider_unconfirmed": len(insiders) - len(confirmed), "missed_insiders": missed,
+            "missed_pr_share": (sum(pr_counts.get(u, 0) for u in missed) / outsider_prs) if outsider_prs else 0.0}
+
+
+def spearman(a: list[float], b: list[float]) -> float:
+    def ranks(xs):
+        order = sorted(range(len(xs)), key=lambda i: xs[i])
+        r = [0.0] * len(xs)
+        for rank, i in enumerate(order):
+            r[i] = float(rank)
+        return r
+    ra, rb = ranks(a), ranks(b)
+    n = len(a)
+    d2 = sum((x - y) ** 2 for x, y in zip(ra, rb))
+    return 1 - 6 * d2 / (n * (n * n - 1))
+
+
+OWNER_QUERY = "query($owner: String!, $name: String!) { " + RL + " repository(owner: $owner, name: $name) { " + " ".join(
+    f'f{i}: object(expression: "HEAD:{f}") {{ ... on Blob {{ text }} }}' for i, f in enumerate(OWNER_FILES)) + " } }"
+
+
+def _rows(con: duckdb.DuckDBPyConnection, sql: str) -> list[dict]:
+    cur = con.execute(sql)
+    cols = [d[0] for d in cur.description]
+    return [dict(zip(cols, r)) for r in cur.fetchall()]
+
+
+def _parquet(signals: Path, table: str) -> str:
+    quoted = (signals / f"{table}.parquet").as_posix().replace("'", "''")
+    return f"read_parquet('{quoted}')"
+
+
+def run_check(signals: Path, out: Path, gql, http: httpx.Client, log=print) -> dict:
+    con = duckdb.connect()
+    stats = _rows(con, f"SELECT repo_id, name, stars, outsider_prs FROM {_parquet(signals, 'repo_stats')} "
+                       "WHERE name IS NOT NULL AND stars IS NOT NULL AND outsider_prs IS NOT NULL")
+    people = _rows(con, f"SELECT repo_id, lower(login) AS login, kind, prs, activity FROM {_parquet(signals, 'people')} "
+                        "WHERE kind IN ('insider', 'outsider')")
+    by_repo = {}
+    for p in people:
+        by_repo.setdefault(p["repo_id"], []).append(p)
+    names = {s["repo_id"]: s for s in stats}
+    rows, missed_prs, outsider_prs = [], 0, 0
+    for repo_id in sample_repos(stats):
+        repo = names[repo_id]
+        owner, name = repo["name"].split("/", 1)
+        data = gql.query(OWNER_QUERY, {"owner": owner, "name": name}).data["repository"] or {}
+        texts = [(data.get(f"f{i}") or {}).get("text") or "" for i in range(len(OWNER_FILES))]
+        confirmed = owner_logins("\n".join(texts)) | {owner.lower()}
+        ppl = by_repo.get(repo_id, [])
+        top = [p["login"] for kind in ("insider", "outsider")
+               for p in sorted((p for p in ppl if p["kind"] == kind), key=lambda p: -p["activity"])[:20]]
+
+        def member(login, owner=owner):
+            # 204 means a public member of the owning org (users and private members answer 404).
+            return login, http.get(f"https://api.github.com/orgs/{owner}/public_members/{login}").status_code == 204
+        with ThreadPoolExecutor(8) as pool:
+            confirmed |= {login for login, yes in pool.map(member, [u for u in top if u not in confirmed]) if yes}
+        s = score_identity({p["login"]: p["kind"] for p in ppl}, confirmed, {p["login"]: p["prs"] for p in ppl})
+        repo_outsider_prs = sum(p["prs"] for p in ppl if p["kind"] == "outsider")
+        missed_prs += sum(p["prs"] for p in ppl if p["login"] in s["missed_insiders"])
+        outsider_prs += repo_outsider_prs
+        rows.append({"repo": repo["name"], "stars": repo["stars"],
+                     "insiders": sum(p["kind"] == "insider" for p in ppl),
+                     "outsiders": sum(p["kind"] == "outsider" for p in ppl),
+                     "owner_files": sum(bool(x) for x in texts),
+                     "insider_confirmed": s["insider_confirmed"], "insider_unconfirmed": s["insider_unconfirmed"],
+                     "missed_insiders": " ".join(s["missed_insiders"]), "missed_pr_share": round(s["missed_pr_share"], 4)})
+        log(f"{repo['name']}: {s['insider_confirmed']} insiders confirmed, missed {s['missed_insiders']} "
+            f"({s['missed_pr_share']:.1%} of outsider PRs)")
+    out.parent.mkdir(parents=True, exist_ok=True)
+    with out.open("w", newline="", encoding="utf-8") as f:
+        writer = csv.DictWriter(f, fieldnames=list(rows[0]) if rows else ["repo"])
+        writer.writeheader()
+        writer.writerows(rows)
+    total = {"repos": len(rows), "missed_pr_share": missed_prs / outsider_prs if outsider_prs else 0.0}
+    log(f"total: {total['repos']} repos, missed insiders wrote {total['missed_pr_share']:.2%} of outsider PRs")
+    return total
+
+
+def _last_month_median(doc: dict, now: datetime) -> float | None:
+    # OpenDigger keys months as "YYYY-MM"; the current month is still incomplete.
+    q = doc.get("quantile_2") or {}
+    months = sorted(k for k in q if re.fullmatch(r"\d{4}-\d{2}", k) and k < now.strftime("%Y-%m"))
+    return q[months[-1]] if months else None
+
+
+def run_opendigger(signals: Path, repos: int, http: httpx.Client, log=print, now: datetime | None = None) -> dict:
+    now = now or datetime.now(timezone.utc)
+    con = duckdb.connect()
+    ours = _rows(con, f"SELECT name, median_review_hours FROM {_parquet(signals, 'repo_stats')} "
+                      f"WHERE review_enough AND name IS NOT NULL ORDER BY reviewed DESC LIMIT {int(repos)}")
+
+    def fetch(row):
+        resp = http.get(f"https://oss.open-digger.cn/github/{row['name']}/change_request_response_time.json")
+        return row, (resp.json() if resp.status_code == 200 else None)
+    with ThreadPoolExecutor(8) as pool:
+        results = list(pool.map(fetch, ours))
+    sample = next((doc for _, doc in results if doc), None)
+    if sample is None or "quantile_2" not in sample:
+        log(f"OpenDigger has no usable data (keys: {sorted(sample or {})[:12]})")
+        return {"n": 0, "spearman": None}
+    pairs = [(row["median_review_hours"], _last_month_median(doc, now) * 24) for row, doc in results
+             if doc and _last_month_median(doc, now) is not None]
+    rho = spearman([a for a, _ in pairs], [b for _, b in pairs]) if len(pairs) > 2 else None
+    log(f"OpenDigger: {len(pairs)} of {len(ours)} repos matched, Spearman {None if rho is None else round(rho, 3)}")
+    return {"n": len(pairs), "spearman": rho}