stackvitals 0.1.0__py3-none-any.whl

mergescout/signals/osv.py

@@ -0,0 +1,232 @@
+"""Vulnerability history per package from OSV (free, no key): every advisory ever, merged across aliases (GHSA, PYSEC,
+CVE are often the same flaw), with the ones still unfixed and any malware record. Records are cached by their modified
+time, so a daily run fetches only new or changed ones."""
+import re
+from concurrent.futures import ThreadPoolExecutor
+from datetime import datetime, timedelta
+from urllib.parse import quote
+
+from mergescout.signals.bumps import _norm
+from mergescout.signals.deps_map import http_get_json, http_post_json
+
+API = "https://api.osv.dev"
+ECOSYSTEMS = {"pypi": "PyPI", "npm": "npm"}
+QUERY_BATCH = 1000
+# A first run would fetch every advisory at once (about 2,300 for 300 packages took 3 minutes); later runs only new ones.
+MAX_FETCH = 4000
+RANK = {"CRITICAL": 4, "HIGH": 3, "MODERATE": 2, "MEDIUM": 2, "LOW": 1}
+
+
+def _ids(packages: list[tuple[str, str]], post_json) -> dict[str, dict[str, str]]:
+    # Advisory ids and their modified time per package; a package with many advisories answers in pages.
+    out = {}
+    wanted = [(eco, name) for eco, name in packages if eco in ECOSYSTEMS]
+    for i in range(0, len(wanted), QUERY_BATCH):
+        chunk = wanted[i:i + QUERY_BATCH]
+        queries = [{"package": {"name": name, "ecosystem": ECOSYSTEMS[eco]}} for eco, name in chunk]
+        results = post_json(f"{API}/v1/querybatch", {"queries": queries}).get("results") or []
+        for (eco, name), query, result in zip(chunk, queries, results):
+            found = {v["id"]: v.get("modified") for v in (result or {}).get("vulns") or []}
+            token = (result or {}).get("next_page_token")
+            while token:
+                more = (post_json(f"{API}/v1/querybatch", {"queries": [{**query, "page_token": token}]}).get("results") or [{}])[0]
+                found.update({v["id"]: v.get("modified") for v in (more or {}).get("vulns") or []})
+                token = (more or {}).get("next_page_token")
+            out[f"{eco}:{name}"] = found
+    return out
+
+
+def _facts(record: dict, keys: set[str]) -> dict:
+    # The few fields the history needs; the advisory text is not kept.
+    pkgs, bounds, bad, spans = {}, {}, {}, {}
+    mal = _is_malware(record)
+    for a in record.get("affected") or []:
+        pkg = a.get("package") or {}
+        eco = next((k for k, v in ECOSYSTEMS.items() if v == pkg.get("ecosystem")), None)
+        key = f"{eco}:{_norm(eco, pkg.get('name') or '')}" if eco else None
+        if key in keys:
+            events = [e for r in a.get("ranges") or [] for e in r.get("events") or []]
+            pkgs[key] = pkgs.get(key, False) or any("fixed" in e for e in events)
+            # "last_affected" bounds the range without naming a fix: newer releases are not affected.
+            bounds[key] = bounds.get(key, []) + [e["last_affected"] for e in events if e.get("last_affected")]
+            # Each [introduced, fixed) span, so a PR's version can be checked and the fixing version named.
+            for r in a.get("ranges") or []:
+                if r.get("type") in ("ECOSYSTEM", "SEMVER"):
+                    start = None
+                    for e in r.get("events") or []:
+                        if "introduced" in e:
+                            start = e["introduced"]
+                        elif "fixed" in e and start is not None:
+                            spans.setdefault(key, []).append([start, e["fixed"]])
+                            start = None
+                        elif "last_affected" in e and start is not None:
+                            # Up to and including that version, with no fix named.
+                            spans.setdefault(key, []).append([start, e["last_affected"], True])
+                            start = None
+                    if start is not None:
+                        spans.setdefault(key, []).append([start, None])
+            if mal:
+                # A malware record lists the bad versions; a range open from "0" means every version (OSV's rule).
+                every = any(e.get("introduced") == "0" for e in events) and not any("fixed" in e or "last_affected" in e
+                                                                                    for e in events)
+                bad[key] = ["all"] if every or "all" in bad.get(key, []) else \
+                    sorted(set(bad.get(key, [])) | set(a.get("versions") or []), key=_version_key)[-50:]
+    # OSV keeps CVEs the supplier disputes and says so in the text; only the flag is kept.
+    text = f"{record.get('summary') or ''} {record.get('details') or ''}".lower()
+    return {"rv": RECORD, "id": record.get("id"), "aliases": record.get("aliases") or [], "published": record.get("published"),
+            "modified": record.get("modified"), "withdrawn": bool(record.get("withdrawn")),
+            "severity": ((record.get("database_specific") or {}).get("severity") or "").upper() or None, "pkgs": pkgs,
+            "bounds": bounds, "ranges": spans, "disputed": "disputed" in text, **({"bad": bad} if mal else {})}
+
+
+def _is_malware(record: dict) -> bool:
+    return any(n.startswith("MAL-") for n in (record.get("id") or "", *(record.get("aliases") or [])))
+
+
+# The record format: a record built by older code is read again (last_affected spans came in version 2).
+RECORD = 2
+# Pre-releases sort before their release and post-releases after it (PEP 440 and semver agree on that much).
+PHASES = {"dev": 0, "a": 1, "alpha": 1, "b": 2, "beta": 2, "c": 3, "rc": 3, "pre": 3, "preview": 3, "post": 5}
+
+
+def version_key(v: str | None) -> tuple:
+    """(epoch, release padded to 4, phase, number): 2.0.0rc1 < 2.0.0 < 2.0.0.post1, 1!0.1 > 9.9."""
+    s = (v or "").strip().lower().removeprefix("v")
+    epoch = 0
+    if "!" in s:
+        head, s = s.split("!", 1)
+        epoch = int(head) if head.isdigit() else 0
+    m = re.match(r"(\d+(?:\.\d+)*)(.*)", s)
+    if not m:
+        return (epoch, (0, 0, 0, 0), 4, 0)
+    release = tuple(int(x) for x in m.group(1).split(".")[:4])
+    release += (0,) * (4 - len(release))
+    rest = m.group(2).split("+", 1)[0]
+    tag = re.search(r"(dev|alpha|beta|preview|pre|post|rc|a|b|c)[.-]?(\d*)", rest)
+    if tag:
+        return (epoch, release, PHASES[tag.group(1)], int(tag.group(2) or 0))
+    # Any other suffix after a dash (1.0.0-next.3) is a semver pre-release.
+    return (epoch, release, 3 if rest.startswith("-") else 4, 0)
+
+
+def _version_key(v: str) -> tuple:
+    return version_key(v)
+
+
+def _cves(group: list[dict]) -> list[str]:
+    return sorted({n for x in group for n in (x["id"], *x["aliases"]) if n.startswith("CVE-")})
+
+
+def _summary(key: str, facts: list[dict], now: datetime, exploits: dict | None = None) -> dict:
+    live = [f for f in facts if not f["withdrawn"] and key in f["pkgs"]]
+    # One flaw under several ids (aliases) counts once: group ids that share any id or alias.
+    groups, seen = [], {}
+    for f in live:
+        names = {f["id"], *f["aliases"]}
+        hit = {seen[n] for n in names if n in seen}
+        merged = [f]
+        for g in sorted(hit, reverse=True):
+            merged += groups.pop(g)
+        groups.append(merged)
+        seen = {n: i for i, g in enumerate(groups) for x in g for n in (x["id"], *x["aliases"])}
+    since = (now - timedelta(days=730)).strftime("%Y-%m-%d")
+    recent = (now - timedelta(days=90)).strftime("%Y-%m-%d")
+    severity, out = {}, {"advisories": len(groups), "last_2y": 0, "last_90d": 0, "unfixed": 0, "latest": None, "ids": []}
+    malware, last_affected, bad = set(), set(), set()
+    out["disputed"] = 0
+    kev, epss_open, epss_max = [], None, None
+    affects = []
+    open_label = None
+    for g in groups:
+        first = min((x["published"] or "")[:10] for x in g)
+        out["last_2y"] += first >= since
+        out["last_90d"] += first >= recent
+        # A malware record lists bad versions and never has a fix; it is reported as malware, not as unfixed.
+        mal = {n for x in g for n in (x["id"], *x["aliases"]) if n.startswith("MAL-")}
+        malware |= mal
+        if mal:
+            bad |= {v for x in g for v in (x.get("bad") or {}).get(key, [])}
+        fixed = any(x["pkgs"][key] for x in g)
+        bound = [v for x in g for v in (x.get("bounds") or {}).get(key, [])]
+        if mal or fixed:
+            pass
+        elif any(x.get("disputed") for x in g):
+            out["disputed"] += 1
+        elif bound:
+            # Open only while the newest release is inside the bound; the dossier compares with the current version.
+            last_affected.add(max(bound, key=_version_key))
+        else:
+            out["unfixed"] += 1
+        # Exploited in the wild (KEV), and the chance of it (EPSS): an open advisory's matters most.
+        for cve in _cves(g) if exploits is not None else ():
+            e = exploits.get(cve) or {}
+            if e.get("kev"):
+                kev.append({"id": cve, "fixed": bool(fixed)})
+            if "epss" in e:
+                score = {"id": cve, "score": e["epss"], "percentile": e.get("percentile")}
+                epss_max = max(epss_max or score, score, key=lambda s: s["score"])
+                if not (mal or fixed or bound):
+                    epss_open = max(epss_open or score, score, key=lambda s: s["score"])
+        spans = [s for x in g for s in (x.get("ranges") or {}).get(key, [])]
+        label = max((x["severity"] for x in g if x["severity"]), key=lambda s: RANK.get(s, 0), default=None)
+        disputed = any(x.get("disputed") for x in g)
+        if spans and not mal:
+            affects.append({"id": g[0]["id"], "ranges": spans, "severity": label, "disputed": disputed})
+        out["latest"] = max(out["latest"] or "", first) or None
+        if label:
+            severity[label] = severity.get(label, 0) + 1
+            # The worst advisory still open (no fix, or a range ending at a bound), for a repo's severity policy.
+            if not (mal or fixed or bound or disputed) and RANK.get(label, 0) > RANK.get(open_label or "", 0):
+                open_label = label
+    out["severity"], out["open_severity"] = severity, open_label
+    out["affects"] = affects
+    out["malware"], out["malware_ids"] = bool(malware), sorted(malware)
+    out["malware_versions"] = ["all"] if "all" in bad else sorted(bad, key=_version_key)[-50:]
+    out["last_affected"] = sorted(last_affected, key=_version_key)
+    out["ids"] = [g[0]["id"] for g in sorted(groups, key=lambda g: min(x["published"] or "" for x in g), reverse=True)[:5]]
+    if exploits is not None:
+        out["kev"] = sorted({k["id"]: k for k in kev}.values(), key=lambda k: k["id"])
+        out["epss_open"], out["epss_max"] = epss_open, epss_max
+    return out
+
+
+def _fetch(get_json, advisory_id: str) -> dict | None:
+    # A record lost after retries is left out of the cache, so the next run asks for it again.
+    try:
+        return get_json(f"{API}/v1/vulns/{quote(advisory_id, safe='')}")
+    except Exception:  # noqa: BLE001
+        return None
+
+
+def refresh_vulns(packages: list[tuple[str, str]], previous: dict, now: datetime, post_json=http_post_json,
+                  get_json=http_get_json, workers: int = 8, max_fetch: int = MAX_FETCH, exploits=None) -> dict:
+    """{"records": cached facts by advisory id, "packages": history per "ecosystem:name"}; only new or changed
+    advisories are fetched again. exploits: a lookup cves -> {cve: {"epss", "percentile", "kev"}} (signals.exploits);
+    without one, no exploit evidence is added."""
+    keys = {f"{eco}:{name}" for eco, name in packages if eco in ECOSYSTEMS}
+    ids = _ids(packages, post_json)
+    old = previous.get("records") or {}
+    # Records cached before ranges' bounds (or malware versions) were kept are read once more.
+    stale = lambda r: r.get("rv") != RECORD or (_is_malware(r) and "bad" not in r)  # noqa: E731
+    todo = sorted({i for found in ids.values() for i, mod in found.items()
+                   if (old.get(i) or {}).get("modified") != mod or stale(old.get(i) or {})})
+    complete = len(todo) <= max_fetch
+    todo = todo[:max_fetch]
+    with ThreadPoolExecutor(workers) as pool:
+        fetched = list(pool.map(lambda i: _fetch(get_json, i), todo))
+    records = {i: old[i] for found in ids.values() for i in found if i in old and i not in todo}
+    # The cache keeps the batch answer's modified time: the record itself may spell it with other precision.
+    modified = {i: mod for found in ids.values() for i, mod in found.items()}
+    records.update({i: {**_facts(r, keys), "modified": modified[i]} for i, r in zip(todo, fetched) if r})
+    found = None
+    if exploits is not None:
+        cves = sorted({n for r in records.values() for n in (r["id"], *r["aliases"]) if n.startswith("CVE-")})
+        found = exploits(cves) or {}
+    packages_out = {key: _summary(key, [records[i] for i in ids.get(key, {}) if i in records], now, found)
+                    for key in sorted(keys)}
+    # Not complete: some advisories wait for the next run, so counts may still grow.
+    out = {"built_at": now.strftime("%Y-%m-%dT%H:%M:%SZ"), "complete": complete, "records": records, "packages": packages_out}
+    if found is not None:
+        out["exploits"] = {"cves": len(cves), "scored": sum("epss" in e for e in found.values()),
+                           "kev": sum(bool(e.get("kev")) for e in found.values())}
+    return out

mergescout/signals/owners_watch.py

@@ -0,0 +1,80 @@
+"""Who can publish each package, watched daily: npm maintainers and the publisher of the latest release (and whether it came
+through trusted CI publishing), PyPI owners and organization. Registries keep no history of this; the changes are the
+signal (a takeover usually starts with a new maintainer or a release published outside the usual CI). Emails are never kept."""
+import time
+from concurrent.futures import ThreadPoolExecutor
+from datetime import datetime
+from urllib.parse import quote
+
+from mergescout.signals.deps_map import http_get_json
+
+MAX_EVENTS = 20
+
+
+def _snapshot(eco: str, name: str, version: str | None, get_json) -> dict | None:
+    if eco == "npm":
+        d = get_json(f"https://registry.npmjs.org/{quote(name, safe='@')}/latest")
+        if not d:
+            return None
+        user = d.get("_npmUser") or {}
+        trusted = bool(user.get("trustedPublisher") or (d.get("dist") or {}).get("attestations"))
+        return {"version": d.get("version"), "publisher": user.get("name"), "trusted": trusted,
+                "maintainers": sorted(m.get("name") for m in d.get("maintainers") or [] if m.get("name"))}
+    if eco == "pypi" and version:
+        d = get_json(f"https://pypi.org/pypi/{quote(name, safe='')}/{quote(version, safe='')}/json")
+        if not d:
+            return None
+        own = d.get("ownership") or {}
+        return {"version": version, "organization": own.get("organization"),
+                "maintainers": sorted(r.get("user") for r in own.get("roles") or [] if r.get("user"))}
+    return None
+
+
+def _events(old: dict, new: dict, day: str) -> list[dict]:
+    out = []
+    added = sorted(set(new.get("maintainers") or []) - set(old.get("maintainers") or []))
+    removed = sorted(set(old.get("maintainers") or []) - set(new.get("maintainers") or []))
+    if added:
+        out.append({"kind": "maintainer_added", "date": day, "who": added})
+    if removed:
+        out.append({"kind": "maintainer_removed", "date": day, "who": removed})
+    if old.get("trusted") and new.get("trusted") is False:
+        out.append({"kind": "trusted_lost", "date": day, "version": new.get("version")})
+    if old.get("publisher") and new.get("publisher") and old["publisher"] != new["publisher"]:
+        out.append({"kind": "publisher_changed", "date": day, "from": old["publisher"], "to": new["publisher"]})
+    if old.get("organization") != new.get("organization") and ("organization" in old or "organization" in new):
+        out.append({"kind": "organization_changed", "date": day, "from": old.get("organization"), "to": new.get("organization")})
+    return out
+
+
+def refresh_owners(packages: list[tuple[str, str, str | None]], previous: dict, now: datetime, get_json=http_get_json,
+                   workers: int = 16, budget_s: float = 900) -> dict:
+    """Today's snapshot per package and the dated changes since watching began; a package not read today (an error or
+    the time budget) keeps yesterday's entry and gets no event."""
+    day = now.strftime("%Y-%m-%d")
+    deadline = time.time() + budget_s
+    old = previous.get("packages") or {}
+
+    def read(item):
+        eco, name, version = item
+        if time.time() > deadline:
+            return None
+        try:
+            return _snapshot(eco, name, version, get_json)
+        except Exception:  # noqa: BLE001
+            return None
+    with ThreadPoolExecutor(workers) as pool:
+        snaps = list(pool.map(read, packages))
+    out = {}
+    for (eco, name, _), snap in zip(packages, snaps):
+        key, before = f"{eco}:{name}", old.get(f"{eco}:{name}")
+        if snap is None:
+            if before:
+                out[key] = before
+            continue
+        if before:
+            events = (before.get("events") or []) + _events(before.get("current") or {}, snap, day)
+            out[key] = {"current": snap, "since": before.get("since", day), "events": events[-MAX_EVENTS:]}
+        else:
+            out[key] = {"current": snap, "since": day, "events": []}
+    return {"built_at": now.strftime("%Y-%m-%dT%H:%M:%SZ"), "packages": out}

mergescout/signals/repo_stats.py

@@ -0,0 +1,96 @@
+import math
+from datetime import datetime, timedelta
+
+import duckdb
+
+from mergescout.signals.rules import RULES, Rules
+
+
+def wilson(k: int, n: int, z: float = 1.96) -> tuple[float, float] | None:
+    if n == 0:
+        return None
+    p = k / n
+    centre = p + z * z / (2 * n)
+    margin = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n))
+    denom = 1 + z * z / n
+    return (centre - margin) / denom, min(1.0, (centre + margin) / denom)
+
+
+def _wilson_macros(con: duckdb.DuckDBPyConnection) -> None:
+    con.execute("""
+        CREATE OR REPLACE MACRO wilson_lo(k, n) AS CASE WHEN n = 0 THEN NULL ELSE
+            ((k::DOUBLE / n + 1.9208 / n) - 1.96 * sqrt((k::DOUBLE / n) * (1 - k::DOUBLE / n) / n + 0.9604 / (n * n)))
+            / (1 + 3.8416 / n) END;
+        CREATE OR REPLACE MACRO wilson_hi(k, n) AS CASE WHEN n = 0 THEN NULL ELSE least(1.0,
+            ((k::DOUBLE / n + 1.9208 / n) + 1.96 * sqrt((k::DOUBLE / n) * (1 - k::DOUBLE / n) / n + 0.9604 / (n * n)))
+            / (1 + 3.8416 / n)) END;""")
+
+
+def build_repo_stats(con: duckdb.DuckDBPyConnection, now: datetime, rules: Rules = RULES) -> None:
+    _wilson_macros(con)
+    since = now - timedelta(days=rules.window_days)
+    con.execute("""
+        CREATE OR REPLACE TABLE repo_stats AS
+        WITH ranked AS (
+            SELECT repo_id, created_at, row_number() OVER (PARTITION BY repo_id ORDER BY created_at DESC) AS rn
+            FROM prs WHERE created_at >= $since),
+        -- Full crawls keep the newest PRs up to the cap; refreshes can add older ones, which must not widen the window.
+        span AS (
+            SELECT p.repo_id, coalesce(c.seen, 0) >= $cap AS truncated,
+                CASE WHEN coalesce(c.seen, 0) >= $cap THEN c.cap_start ELSE $since END AS window_start
+            FROM (SELECT DISTINCT repo_id FROM prs) p
+            LEFT JOIN (SELECT repo_id, count(*) AS seen, min(created_at) FILTER (WHERE rn <= $cap) AS cap_start
+                       FROM ranked GROUP BY repo_id) c USING (repo_id)),
+        -- Prefer the first person's review; older batches only know the very first review.
+        w AS (SELECT * REPLACE (coalesce(first_human_review_at, first_review_at) AS first_review_at,
+                                coalesce(first_human_review_by, first_review_by) AS first_review_by)
+              FROM pr_class JOIN span USING (repo_id) WHERE created_at >= window_start),
+        firsts AS (
+            SELECT repo_id, author_login, min(merged_at) AS first_merge FROM pr_class
+            WHERE author_class = 'outsider' AND merged_at IS NOT NULL GROUP BY ALL),
+        o AS (
+            SELECT w.repo_id,
+                count(*) AS outsider_prs,
+                count(*) FILTER (WHERE state = 'MERGED') AS merged,
+                count(*) FILTER (WHERE state = 'CLOSED') AS closed,
+                count(*) FILTER (WHERE state = 'OPEN') AS open,
+                count(*) FILTER (WHERE state IN ('MERGED', 'CLOSED')
+                                 AND (f.first_merge IS NULL OR w.created_at <= f.first_merge)) AS newcomer_decided,
+                count(*) FILTER (WHERE w.merged_at IS NOT NULL AND w.merged_at = f.first_merge) AS newcomer_merged,
+                count(*) FILTER (WHERE first_review_at IS NOT NULL
+                                 AND first_review_by IS DISTINCT FROM w.author_login AND rv.kind = 'insider') AS reviewed,
+                median(epoch(first_review_at - created_at) / 3600) FILTER (
+                    WHERE first_review_at IS NOT NULL AND first_review_by IS DISTINCT FROM w.author_login
+                          AND rv.kind = 'insider')
+                    AS median_review_hours
+            FROM w LEFT JOIN firsts f USING (repo_id, author_login)
+            -- Only a maintainer's review counts as a response; review bots answer within minutes.
+            LEFT JOIN people rv ON rv.repo_id = w.repo_id AND rv.login = w.first_review_by
+            WHERE w.author_class = 'outsider' GROUP BY w.repo_id),
+        mix AS (
+            SELECT repo_id, count(*) AS window_prs,
+                count(*) FILTER (WHERE author_class = 'ai') AS ai_prs,
+                count(*) FILTER (WHERE author_class = 'bot') AS bot_prs,
+                count(*) FILTER (WHERE author_class = 'insider') AS insider_prs
+            FROM w GROUP BY repo_id)
+        SELECT span.repo_id, rm.name, rm.stars,
+            -- When the PR cap cut history short, say how many days we really cover.
+            CASE WHEN span.truncated THEN date_diff('day', span.window_start, $now) ELSE $window END AS window_days,
+            NOT span.truncated AS window_complete,
+            coalesce(o.outsider_prs, 0) AS outsider_prs, coalesce(o.merged, 0) AS merged,
+            coalesce(o.closed, 0) AS closed, coalesce(o.open, 0) AS open,
+            o.merged::DOUBLE / nullif(o.merged + o.closed, 0) AS merge_rate,
+            wilson_lo(o.merged, o.merged + o.closed) AS merge_low, wilson_hi(o.merged, o.merged + o.closed) AS merge_high,
+            coalesce(o.merged + o.closed, 0) >= $min_decided AS merge_enough,
+            coalesce(o.newcomer_decided, 0) AS newcomer_decided, coalesce(o.newcomer_merged, 0) AS newcomer_merged,
+            o.newcomer_merged::DOUBLE / nullif(o.newcomer_decided, 0) AS newcomer_rate,
+            wilson_lo(o.newcomer_merged, o.newcomer_decided) AS newcomer_low,
+            wilson_hi(o.newcomer_merged, o.newcomer_decided) AS newcomer_high,
+            coalesce(o.newcomer_decided, 0) >= $min_decided AS newcomer_enough,
+            coalesce(o.reviewed, 0) AS reviewed, o.median_review_hours,
+            coalesce(o.reviewed, 0) >= $min_reviewed AS review_enough,
+            coalesce(mix.window_prs, 0) AS window_prs, coalesce(mix.ai_prs, 0) AS ai_prs,
+            coalesce(mix.bot_prs, 0) AS bot_prs, coalesce(mix.insider_prs, 0) AS insider_prs
+        FROM span LEFT JOIN o USING (repo_id) LEFT JOIN mix USING (repo_id) LEFT JOIN repo_meta rm USING (repo_id)""",
+        {"since": since, "now": now, "cap": rules.pr_cap, "window": rules.window_days,
+         "min_decided": rules.min_decided, "min_reviewed": rules.min_reviewed})

mergescout/signals/rules.py

@@ -0,0 +1,70 @@
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class Rules:
+    window_days: int = 365
+    min_decided: int = 20
+    min_closed: int = 10
+    min_reviewed: int = 5
+    claim_days: int = 14
+    interest_days: int = 30
+    stale_days: int = 180
+    pr_cap: int = 300
+
+
+RULES = Rules()
+
+# Accounts that act like bots but are typed as users; extend when the hand-check finds more.
+KNOWN_BOTS = frozenset({
+    "miss-islington", "bors", "homu", "k8s-ci-robot", "k8s-triage-robot", "k8s-merge-robot", "pytorchbot",
+    "facebook-github-bot", "googlebot", "google-cla", "copybara-service", "tensorflow-jenkins", "rustbot",
+    "rust-highfive", "rust-timer", "nodejs-github-bot", "chromium-wpt-export-bot", "mergify", "kodiakhq",
+    "allcontributors", "stale", "netlify", "vercel", "coveralls", "codecov-commenter", "sonarcloud",
+    "openshift-ci-robot", "openshift-merge-robot", "ti-chi-bot", "ti-srebot", "elasticmachine", "apache-bot",
+    "copilot-pull-request-reviewer", "coderabbitai", "gemini-code-assist", "sourcery-ai", "greptile-apps",
+    "home-assistant", "welcome", "linear", "similar-issues-ai", "oss-sentinel-ai", "hfla-graphql-app",
+    # CI accounts that open failing-test issues (seen 2026-10-16).
+    "kibanamachine", "elasticsearchmachine", "brave-builds", "gordontheturtle",
+})
+# Comments carry no account type, so bots that only comment are known by their names (seen on 2,545 repos).
+BOT_NAME_PATTERN = r"(bot|-ci|github-actions|-automation|-integration)$"
+UNKNOWN_LOGINS = frozenset({"ghost"})
+# Accounts that are AI coding agents, not ordinary automation; their PRs count as AI-written.
+AI_AGENT_LOGINS = frozenset({"copilot-swe-agent", "devin-ai-integration", "chatgpt-codex-connector", "cursor-agent",
+                             "sweep-ai", "openhands-agent", "openhands-ai"})
+# Plain words a person can also use as a login: AI only when the account is a bot.
+AI_AGENT_BOT_NAMES = frozenset({"copilot", "claude", "codex", "cursor", "devin"})
+
+# Real comments also say "...speaker and would like to work on this" and "I'll open a PR" (symfony#66111).
+CLAIM_PATTERN = (r"(i('d| would) (like|love) to (work on|take|tackle|pick)"
+                 r"|\band would (like|love) to (work on|take|tackle|pick up) (this|it)\b"
+                 r"|(can|could|may) i (work on|take|pick|tackle|try)"
+                 r"|assign (this|it) to me|please assign|^/assign|i('m| am) (working|taking) (on )?(this|it)"
+                 r"|let me (take|work on|try|pick)|i('ll| will) (take|work on|pick up|try) (this|it)"
+                 r"|i('ll| will) (open|submit|send|raise|create) (a )?(pr|pull request)\b)")
+WAIT_PATTERN = (r"(please (wait|hold)|on hold|blocked (by|on)|needs? (design|discussion|decision|triage)"
+                r"|not (ready|accepting)|we('re| are) (still )?(discussing|deciding)|do not (open|send) (a )?(pr|pull))")
+# Surnames that end in "bot" (seen: adamrtalbot); RE2 has no lookbehind, so they are excluded separately.
+HUMAN_BOT_NAMES = r"(talbot|abbot|cabot|sabot)$"
+# Labels (after norm_label) that say the issue is not ready for a PR, e.g. "status: blocked", "needs info".
+# Anchored: real labels "triage accepted", "never stale", "question bank" and "triage needs pr" mean ready or a topic.
+PREFIX = r"^((status|state|type|kind|resolution|issue|s|t) )?"
+WAITING = r"(design|discussion|decision|info|information|feedback|repro|reproduction|investigation|confirmation)"
+# Also labels saying someone is on it ("in progress", "claimed", "gssoc assigned") or it is done ("fixed").
+WAIT_LABEL_PATTERN = (rf"{PREFIX}(blocked|on hold|wontfix|won t fix|invalid|duplicate|not planned|"
+                      rf"maybe later|(\d+ )?in progress|work in progress|wip|claimed( \w+)?|"
+                      rf"taken|has pr|pr open|pr submitted|fixed|resolved( pending close)?|(wait|waiting) for review)$"
+                      rf"|\bneeds? (more )?{WAITING}\b|\b(awaiting|waiting for|waiting on|pending|requires) "
+                      rf"({WAITING}|response|reply)\b|\btriage needs {WAITING}\b"
+                      r"|^(gssoc|gsoc|ssoc|hacktoberfest|kwoc|swoc|jwoc|iwoc|dwoc|osci)\w* assigned$")
+# "Not settled yet" labels (triage, question, proposal). Repos add them to every new issue or keep them after a
+# decision (golang: "Proposal" + "Proposal-Accepted"), so a beginner or help label added later outweighs them.
+TRIAGE_LABEL_PATTERN = (rf"{PREFIX}(triage|untriaged|unconfirmed|question|discussion|in discussion|rfc|proposal)$"
+                        r"|\bneeds? triage\b|\b(awaiting|waiting for|pending|requires) triage\b"
+                        r"|\btriage (needed|pending|required)\b")
+# Titles like "[RFC] ..." or "Epic: ..." and labels like "epic" mark trackers and debates, not tasks.
+KINDS = r"(epic|rfc|discussion|tracking|tracker|meta|roadmap|proposal|umbrella|poll|survey)"
+DISCUSSION_TITLE = rf"^\s*[\[(]\s*{KINDS}\s*[\])]|^\s*{KINDS}\s*:|\btracking issue\b|^\s*dependency dashboard\s*$"
+LABEL_KINDS = r"(epic|tracking|tracker|roadmap|umbrella|poll|survey)"
+DISCUSSION_LABEL = rf"^((type|kind) )?{LABEL_KINDS}$"

mergescout/signals/silver.py

@@ -0,0 +1,33 @@
+from pathlib import Path
+
+import duckdb
+import pyarrow as pa
+
+from mergescout.crawl.writer import SCHEMAS
+
+# One row per key survives: the most recently fetched copy.
+KEYS = {"repo_meta": ["repo_id"], "prs": ["repo_id", "number"], "issues": ["repo_id", "number"],
+        "comments": ["repo_id", "number", "comment_id", "created_at"], "closed_issues": ["repo_id", "number"],
+        "issue_events": ["repo_id", "number", "event", "observed_at"]}
+
+
+def load_silver(con: duckdb.DuckDBPyConnection, roots: list[Path]) -> dict[str, int]:
+    con.execute("SET TimeZone = 'UTC'")  # day counts must not depend on the machine's clock zone
+    counts = {}
+    for table, key in KEYS.items():
+        files = sorted(str(p) for root in roots for p in (root / "crawl" / table).glob("*.parquet"))
+        con.register(f"{table}_empty", pa.Table.from_pylist([], schema=SCHEMAS[table]))
+        source = f"SELECT * FROM {table}_empty"
+        if files:
+            con.read_parquet(files, union_by_name=True).create_view(f"{table}_raw", replace=True)
+            # BY NAME fills columns that older batches do not have with nulls.
+            source += f" UNION ALL BY NAME SELECT * FROM {table}_raw"
+        con.execute(f"""
+            CREATE OR REPLACE TABLE {table} AS
+            SELECT * EXCLUDE (rn) FROM (
+                SELECT *, row_number() OVER (PARTITION BY {", ".join(key)}
+                                             ORDER BY fetched_at DESC NULLS LAST, run_id DESC) AS rn
+                FROM ({source})) WHERE rn = 1""")
+        # fetchall, not fetchone: DuckDB 2 refuses any other work while a result is half read.
+        counts[table] = con.execute(f"SELECT count(*) FROM {table}").fetchall()[0][0]
+    return counts

mergescout/signals/top_packages.py

@@ -0,0 +1,26 @@
+"""The most-used npm and PyPI packages (by repos depending on them, from ecosyste.ms), so the evidence covers what people
+add, not only what crawled repos happen to upgrade."""
+from mergescout.signals.bump_ledger import valid_name
+from mergescout.signals.bumps import _norm
+from mergescout.signals.deps_map import http_get_json
+
+REGISTRIES = {"npm": "npmjs.org", "pypi": "pypi.org"}
+URL = ("https://packages.ecosyste.ms/api/v1/registries/{registry}/packages"
+       "?sort=dependent_repos_count&order=desc&per_page=100&page={page}")
+
+
+def fetch_top(n: int = 1000, get_json=None) -> list[tuple[str, str]]:
+    """The top n of each registry, most-used first; a page that fails ends that registry's list for today."""
+    get_json = get_json or http_get_json
+    out = []
+    for eco, registry in REGISTRIES.items():
+        for page in range(1, -(-n // 100) + 1):
+            try:
+                rows = get_json(URL.format(registry=registry, page=page))
+            except Exception:  # noqa: BLE001
+                break
+            if not isinstance(rows, list) or not rows:
+                break
+            out += [(eco, _norm(eco, r["name"])) for r in rows
+                    if isinstance(r, dict) and isinstance(r.get("name"), str) and valid_name(eco, r["name"])]
+    return list(dict.fromkeys(out))

mergescout/signals/triage.py

@@ -0,0 +1,71 @@
+import hashlib
+from datetime import datetime, timedelta
+
+import duckdb
+
+from mergescout.signals.triage_math import buckets, calibration, fit
+
+DEFINITION = ("outside contributors' PRs opened 30 to 395 days ago; outcome = merged within 30 days of opening; "
+              "every feature is as of the PR's opening: the author's earlier PRs and merges here, and the repo's rate over "
+              "outside PRs opened 30 to 395 days before it. "
+              "Checked on one repo in five, never seen in training.")
+MIN_REPO_PRS = 10
+
+
+def training_rows(con: duckdb.DuckDBPyConnection, now: datetime) -> list[dict]:
+    """One row per outside PR whose 30-day outcome is known, with only what was known when it was opened."""
+    rows = con.execute("""
+        WITH outs AS (
+            SELECT repo_id, number, created_at, author_login, additions, deletions, closing_issue_numbers,
+                   (merged_at IS NOT NULL AND merged_at <= created_at + INTERVAL 30 DAY) AS merged30
+            FROM pr_class WHERE author_class = 'outsider' AND author_login IS NOT NULL),
+        -- The repo's rate as of each PR: outside PRs opened 30-395 days earlier, whose 30-day outcome was known by then.
+        rated AS (
+            SELECT o.*, count(*) OVER w AS past_n, sum(merged30::INT) OVER w AS past_k FROM outs o
+            WINDOW w AS (PARTITION BY repo_id ORDER BY created_at
+                         RANGE BETWEEN INTERVAL 395 DAY PRECEDING AND INTERVAL 30 DAY PRECEDING)),
+        base AS (SELECT * FROM rated WHERE created_at >= $lo AND created_at <= $hi),
+        merges AS (SELECT repo_id, author_login, merged_at FROM pr_class WHERE merged_at IS NOT NULL AND author_login IS NOT NULL),
+        -- Running totals read "as of" each PR's opening: cheap even for prolific authors.
+        here AS (SELECT repo_id, author_login, merged_at,
+                        row_number() OVER (PARTITION BY repo_id, author_login ORDER BY merged_at) AS done FROM merges),
+        opened AS (SELECT repo_id, author_login, created_at,
+                          row_number() OVER (PARTITION BY repo_id, author_login ORDER BY created_at) AS done
+                   FROM pr_class WHERE author_login IS NOT NULL),
+        counted AS (
+            SELECT b.*, coalesce(h.done, 0) AS here_merged, coalesce(o.done, 0) AS here_opened
+            FROM base b
+            ASOF LEFT JOIN here h ON h.repo_id = b.repo_id AND h.author_login = b.author_login AND b.created_at > h.merged_at
+            ASOF LEFT JOIN opened o ON o.repo_id = b.repo_id AND o.author_login = b.author_login AND b.created_at > o.created_at)
+        SELECT repo_id, number, here_merged,
+               coalesce(additions, 0) + coalesce(deletions, 0) AS lines,
+               coalesce(len(closing_issue_numbers), 0) > 0 AS linked_issue,
+               -- GitHub reports the author's association as of the query, which leaks the outcome; history does not.
+               here_opened = 0 AS first_timer,
+               CASE WHEN past_n >= $min_repo THEN past_k::DOUBLE / past_n END AS repo_rate,
+               merged30 AS merged
+        FROM counted""",
+        {"lo": now - timedelta(days=395), "hi": now - timedelta(days=30), "min_repo": MIN_REPO_PRS}).fetchall()
+    cols = ("repo", "number", "here_merged", "lines", "linked_issue", "first_timer", "repo_rate", "merged")
+    return [dict(zip(cols, r)) for r in rows]
+
+
+def _held_out(repo_id: int) -> bool:
+    # One repo in five, picked by a real hash so neighbouring ids don't land together.
+    return int(hashlib.sha1(str(repo_id).encode()).hexdigest(), 16) % 5 == 0
+
+
+def build_triage_model(con: duckdb.DuckDBPyConnection, now: datetime) -> dict:
+    """Fit on four repos in five, check calibration on the fifth, then publish a model fitted on all of them."""
+    rows = []
+    for r in training_rows(con, now):
+        feats = buckets(r["here_merged"], r["lines"], r["linked_issue"], r["first_timer"], r["repo_rate"])
+        rows.append({**feats, "merged": bool(r["merged"]), "repo": r["repo"]})
+    train = [r for r in rows if not _held_out(r["repo"])]
+    test = [r for r in rows if _held_out(r["repo"])]
+    evaluation = {"train_prs": len(train), "test_prs": len(test)}
+    if train and test:
+        evaluation |= calibration(fit(train), test)
+    model = fit(rows) if rows else fit([])
+    return {**model, "built_at": now.strftime("%Y-%m-%dT%H:%M:%SZ"), "definition": DEFINITION, "prs": len(rows),
+            "evaluation": evaluation}