stackvitals 0.1.0__py3-none-any.whl

mergescout/signals/community_bench.py

@@ -0,0 +1,149 @@
+"""How public repos treat outside contributors, from the crawl: how fast a first answer comes, and whether newcomers come
+back. Per-repo facts become percentiles by language and size; the newcomer table backs the "answer first-timers" lever."""
+from datetime import datetime, timedelta
+
+import duckdb
+
+from mergescout.bench.ci import _pct, quantiles, tier
+
+MIN_PRS = 10
+MIN_NEWCOMERS = 10
+MIN_GROUP = 50
+# A repo's first 60 days of crawled history can't tell a newcomer from someone who came before it.
+HISTORY_DAYS = 60
+RETURN_DAYS = 90
+METRICS = ("first_answer_hours", "answered_1d_share", "newcomer_answered_share", "newcomer_return_rate", "bus_factor", "top_share")
+DEPENDENCY_DAYS, MIN_MERGED = 180, 20
+BUCKETS = ("within 1 day", "1-3 days", "3-14 days", "over 14 days", "no answer")
+DEFINITION = ("Outside contributors are people who are not owners, members, collaborators, bots or AI agents. A first "
+              "answer is the first review by someone other than the author, a merge or a close. Per repo: median hours "
+              "to the first answer and the share answered within a day, for outside PRs opened in the last 90 days "
+              "(10+ PRs); for newcomers (an outside contributor's first PR in a repo with 60+ days of history before "
+              "it), the share answered within 14 days and the share who opened another PR 1 to 90 days later (10+ newcomers). "
+              "Accounts opening 20+ PRs in a repo on one day are left out as automated. Who carries the work: PRs by "
+              "people (anyone but bots and AI agents) merged in the last 180 days (20+); the top person's share, and the "
+              "bus factor, the fewest people behind half of them.")
+
+ANSWER = """least(CASE WHEN p.first_human_review_by IS DISTINCT FROM p.author_login THEN p.first_human_review_at END,
+                  p.merged_at, p.closed_at)"""
+# One account opening 20+ PRs in a repo on one day is automated (bat: 73 in a day); it would swamp the repo's rates.
+BURST = """burst AS (SELECT DISTINCT repo_id, author_login FROM pr_class WHERE author_class = 'outsider'
+                     GROUP BY repo_id, author_login, CAST(created_at AS DATE) HAVING count(*) >= 20)"""
+NEWCOMERS = f"""
+    WITH span AS (SELECT repo_id, min(created_at) AS start FROM pr_class GROUP BY repo_id),
+    {BURST},
+    outside AS (SELECT p.*, {ANSWER} AS answered_at FROM pr_class p
+                WHERE p.author_class = 'outsider' AND NOT EXISTS (
+                    SELECT 1 FROM burst b WHERE b.repo_id = p.repo_id AND b.author_login = p.author_login)),
+    ranked AS (SELECT o.*, row_number() OVER w AS k
+               FROM outside o WINDOW w AS (PARTITION BY o.repo_id, o.author_login ORDER BY o.created_at, o.number)),
+    firsts AS (SELECT r.* FROM ranked r JOIN span s USING (repo_id)
+               WHERE r.k = 1 AND r.created_at >= s.start + INTERVAL {HISTORY_DAYS} DAY)
+    -- Coming back is new work at least a day later: PRs split minutes apart are one contribution.
+    SELECT f.*, (SELECT min(o.created_at) FROM outside o WHERE o.repo_id = f.repo_id AND o.author_login = f.author_login
+                 AND o.created_at >= f.created_at + INTERVAL 1 DAY) AS next_at
+    FROM firsts f"""
+
+
+def newcomer_returns(con: duckdb.DuckDBPyConnection, now: datetime) -> dict:
+    """Newcomers whose return can be judged (first PR 120+ days ago), by how long their first PR waited for an answer."""
+    judged = f"WITH judged AS (SELECT * FROM ({NEWCOMERS}) WHERE created_at <= $cut)"
+    cut = {"cut": now - timedelta(days=RETURN_DAYS + 30)}
+    back = f"avg((next_at IS NOT NULL AND next_at <= created_at + INTERVAL {RETURN_DAYS} DAY)::INT)"
+    by_answer = con.execute(f"""{judged}
+        SELECT CASE WHEN answered_at IS NULL THEN 4 WHEN answered_at - created_at <= INTERVAL 1 DAY THEN 0
+                    WHEN answered_at - created_at <= INTERVAL 3 DAY THEN 1 WHEN answered_at - created_at <= INTERVAL 14 DAY THEN 2
+                    ELSE 3 END AS b, count(*), {back}, avg((merged_at IS NOT NULL)::INT)
+        FROM judged GROUP BY 1 ORDER BY 1""", cut).fetchall()
+    by_outcome = con.execute(f"""{judged}
+        SELECT CASE WHEN merged_at IS NOT NULL THEN 'merged' WHEN state = 'CLOSED' THEN 'closed unmerged' ELSE 'still open' END,
+               count(*), {back}
+        FROM judged GROUP BY 1 ORDER BY 2 DESC""", cut).fetchall()
+    n, repos = con.execute(f"{judged} SELECT count(*), count(DISTINCT repo_id) FROM judged", cut).fetchall()[0]
+    return {"newcomers": n, "repos": repos, "return_days": RETURN_DAYS,
+            "by_answer": [{"bucket": BUCKETS[b], "newcomers": c, "returned": round(r, 3), "merged": round(m, 3)}
+                          for b, c, r, m in by_answer],
+            "by_outcome": [{"outcome": o, "newcomers": c, "returned": round(r, 3)} for o, c, r in by_outcome]}
+
+
+def repo_metrics(con: duckdb.DuckDBPyConnection, now: datetime) -> list[dict]:
+    """The facts behind each repo's community score; a fact without enough PRs or newcomers is None."""
+    params = {"lo": now - timedelta(days=90), "hi": now - timedelta(days=1), "seen": now - timedelta(days=14),
+              "judged": now - timedelta(days=RETURN_DAYS + 30), "dep": now - timedelta(days=DEPENDENCY_DAYS)}
+    rows = con.execute(f"""
+        WITH {BURST},
+        recent AS (
+            SELECT p.repo_id, {ANSWER} AS answered_at, p.created_at FROM pr_class p
+            WHERE p.author_class = 'outsider' AND p.created_at >= $lo AND p.created_at < $hi AND NOT EXISTS (
+                SELECT 1 FROM burst b WHERE b.repo_id = p.repo_id AND b.author_login = p.author_login)),
+        answers AS (
+            SELECT repo_id, count(*) AS prs,
+                   median(epoch(answered_at - created_at) / 3600) FILTER (WHERE answered_at IS NOT NULL) AS first_answer_hours,
+                   avg((answered_at IS NOT NULL AND answered_at <= created_at + INTERVAL 1 DAY)::INT) AS answered_1d_share
+            FROM recent GROUP BY repo_id),
+        new AS ({NEWCOMERS}),
+        newcomers AS (
+            SELECT repo_id,
+                   count(*) FILTER (WHERE created_at <= $seen) AS seen,
+                   avg((answered_at IS NOT NULL AND answered_at <= created_at + INTERVAL 14 DAY)::INT)
+                       FILTER (WHERE created_at <= $seen) AS newcomer_answered_share,
+                   count(*) FILTER (WHERE created_at <= $judged) AS judged,
+                   avg((next_at IS NOT NULL AND next_at <= created_at + INTERVAL {RETURN_DAYS} DAY)::INT)
+                       FILTER (WHERE created_at <= $judged) AS newcomer_return_rate
+            FROM new GROUP BY repo_id),
+        -- Who carries the work, as the app measures it: people's merged PRs, the fewest people behind half of them.
+        by_person AS (
+            SELECT repo_id, author_login, count(*) AS n FROM pr_class
+            WHERE merged_at >= $dep AND author_role IN ('insider', 'outsider') GROUP BY repo_id, author_login),
+        running AS (
+            SELECT repo_id, n, sum(n) OVER (PARTITION BY repo_id) AS total,
+                   sum(n) OVER (PARTITION BY repo_id ORDER BY n DESC, author_login ROWS UNBOUNDED PRECEDING) AS upto,
+                   row_number() OVER (PARTITION BY repo_id ORDER BY n DESC, author_login) AS k
+            FROM by_person),
+        dependency AS (
+            SELECT repo_id, max(total) AS merged, max(n) / max(total) AS top_share,
+                   min(k) FILTER (WHERE upto * 2 >= total) AS bus_factor
+            FROM running GROUP BY repo_id)
+        SELECT rm.repo_id, rm.language, rm.stars, a.prs, a.first_answer_hours, a.answered_1d_share,
+               n.seen, n.newcomer_answered_share, n.judged, n.newcomer_return_rate, d.merged, d.top_share, d.bus_factor
+        FROM repo_meta rm LEFT JOIN answers a USING (repo_id) LEFT JOIN newcomers n USING (repo_id)
+            LEFT JOIN dependency d USING (repo_id)
+        WHERE coalesce(a.prs, 0) >= {MIN_PRS} OR coalesce(n.seen, 0) >= {MIN_NEWCOMERS}
+            OR coalesce(d.merged, 0) >= {MIN_MERGED}""", params).fetchall()
+    out = []
+    for repo_id, lang, stars, prs, hours, share_1d, seen, answered, judged, returned, merged, top, bus in rows:
+        enough_prs, enough_seen, enough_judged = (prs or 0) >= MIN_PRS, (seen or 0) >= MIN_NEWCOMERS, (judged or 0) >= MIN_NEWCOMERS
+        enough_merged = (merged or 0) >= MIN_MERGED
+        out.append({"repo_id": repo_id, "language": lang, "stars": stars,
+                    "first_answer_hours": round(hours, 2) if enough_prs and hours is not None else None,
+                    "answered_1d_share": round(share_1d, 4) if enough_prs else None,
+                    "newcomer_answered_share": round(answered, 4) if enough_seen else None,
+                    "newcomer_return_rate": round(returned, 4) if enough_judged else None,
+                    "bus_factor": int(bus) if enough_merged else None, "top_share": round(top, 4) if enough_merged else None})
+    return out
+
+
+def community_benchmarks(con: duckdb.DuckDBPyConnection, now: datetime, top_languages: int = 25) -> dict:
+    """Percentiles of the community facts by language and size (groups need MIN_GROUP repos), plus the newcomer table."""
+    repos = repo_metrics(con, now)
+    counts = {}
+    for r in repos:
+        counts[r["language"] or "Other"] = counts.get(r["language"] or "Other", 0) + 1
+    langs = {name for name, _ in sorted(counts.items(), key=lambda kv: -kv[1])[:top_languages]}
+    groups = {}
+    for r in repos:
+        lang = r["language"] if r["language"] in langs else "Other"
+        for key in (f"{lang}|{tier(r['stars'])}", f"{lang}|all", f"all|{tier(r['stars'])}", "all|all"):
+            groups.setdefault(key, []).append(r)
+    out = {}
+    for key, items in groups.items():
+        if len(items) < MIN_GROUP:
+            continue
+        entry = {"repos": len(items)}
+        for m in METRICS:
+            values = [float(r[m]) for r in items if r[m] is not None]
+            if len(values) >= MIN_GROUP:
+                entry[m] = {"repos": len(values), **quantiles(values)}
+        out[key] = entry
+    return {"version": 1, "built_at": now.strftime("%Y-%m-%dT%H:%M:%SZ"), "repos": len(repos), "definition": DEFINITION, "population": 'public GitHub repos in the daily crawl with enough outside pull requests and issues to measure', "gaps": 'no private or enterprise teams; GitHub only; a newcomer is someone new to that repo, not to open source',
+            "groups": out, "newcomers": newcomer_returns(con, now)}

mergescout/signals/delivery_bench.py

@@ -0,0 +1,103 @@
+"""Delivery benchmarks from the crawl: per repo the P75 of pickup, review and cycle to merge (the suite's nearest-rank
+rule, so a repo's own Delivery page compares like with like), per group the spread with a bootstrap range on the median
+and the share of repos inside LinearB's published bands, with the population and the gaps stated beside the numbers."""
+import math
+from datetime import datetime, timedelta
+
+import duckdb
+
+from mergescout.bench.ci import BOOTSTRAP_RESAMPLES as RESAMPLES
+from mergescout.bench.ci import BOOTSTRAP_SEED as SEED
+from mergescout.bench.ci import _pct, bootstrap_median, tier
+
+MIN_PRS = 10  # PRs a repo needs in the window
+MIN_VALUES = 5  # values a repo needs for one metric
+MIN_GROUP = 50
+METRICS = {"pickup_p75_hours": "pickup", "review_p75_hours": "review", "cycle_to_merge_p75_hours": "cycle"}
+# LinearB's 2026 bands on P75 hours: elite below the first, good up to the second, fair up to the third.
+LINEARB = {"pickup_p75_hours": (1, 4, 16), "review_p75_hours": (3, 14, 24)}
+DEFINITION = ("per repo, PRs opened in the last 90 days by people (bots and AI agents left out), repos with 10+ such PRs: "
+              "the P75 (nearest rank) of hours from opening to the first review by a maintainer other than the author "
+              "(pickup), from that review to the merge (review), and from opening to the merge (cycle to merge); a repo "
+              "needs 5+ values for a metric. Groups by language and stars need 50+ repos.")
+POPULATION = ("public GitHub repos in the daily crawl that had 10+ pull requests by people in the last 90 days; GitHub "
+              "only, any CI")
+GAPS = ("no private or enterprise teams; no GitLab or Bitbucket; pickup counts from opening, not from leaving draft (the "
+        "crawl does not keep draft history), so repos that open drafts look slower than LinearB's own pickup; cycle to "
+        "merge has no coding or deploy time, so it is not LinearB's full cycle time")
+BANDS_NOTE = ("LinearB's 2026 benchmark bands on P75 (pickup: elite under 1 h, good up to 4 h, fair up to 16 h; review: "
+              "elite under 3 h, good up to 14 h, fair up to 24 h); each share is the part of the group's repos inside that "
+              "band or a better one")
+
+
+def _p75(values: list[float]) -> float:
+    s = sorted(values)
+    return s[min(len(s) - 1, max(0, math.ceil(0.75 * len(s)) - 1))]
+
+
+def per_repo(con: duckdb.DuckDBPyConnection, now: datetime, window_days: int = 90) -> list[dict]:
+    rows = con.execute("""
+        WITH w AS (
+            SELECT p.*, coalesce(p.first_human_review_at, p.first_review_at) AS review_at,
+                   coalesce(p.first_human_review_by, p.first_review_by) AS review_by
+            FROM pr_class p WHERE p.created_at >= $since AND p.author_class IN ('insider', 'outsider'))
+        SELECT w.repo_id, any_value(rm.language), any_value(rm.stars), count(*),
+            list(epoch(w.review_at - w.created_at) / 3600) FILTER (
+                WHERE w.review_at IS NOT NULL AND w.review_by IS DISTINCT FROM w.author_login AND rv.kind = 'insider'),
+            list(epoch(w.merged_at - w.review_at) / 3600) FILTER (
+                WHERE w.merged_at IS NOT NULL AND w.review_at IS NOT NULL AND w.review_at <= w.merged_at
+                  AND w.review_by IS DISTINCT FROM w.author_login AND rv.kind = 'insider'),
+            list(epoch(w.merged_at - w.created_at) / 3600) FILTER (WHERE w.merged_at IS NOT NULL)
+        FROM w LEFT JOIN people rv ON rv.repo_id = w.repo_id AND rv.login = w.review_by
+        LEFT JOIN repo_meta rm ON rm.repo_id = w.repo_id
+        GROUP BY w.repo_id HAVING count(*) >= $min_prs""",
+        {"since": now - timedelta(days=window_days), "min_prs": MIN_PRS}).fetchall()
+    out = []
+    for repo_id, language, stars, prs, pickup, review, cycle in rows:
+        rec = {"repo_id": repo_id, "language": language, "stars": stars, "prs": prs}
+        for metric, values in zip(METRICS, (pickup, review, cycle)):
+            values = [v for v in values or [] if v is not None and v >= 0]
+            rec[metric] = round(_p75(values), 4) if len(values) >= MIN_VALUES else None
+        out.append(rec)
+    return out
+
+
+def _entry(metric: str, values: list[float]) -> dict:
+    low, high = bootstrap_median(values)
+    entry = {"repos": len(values), **{f"p{int(q * 100)}": _pct(values, q) for q in (0.1, 0.25, 0.5, 0.75, 0.9)},
+             "p50_low": low, "p50_high": high}
+    if metric in LINEARB:
+        elite, good, fair = LINEARB[metric]
+        n = len(values)
+        entry["linearb"] = {"elite": round(sum(v < elite for v in values) / n, 4), "good": round(sum(v <= good for v in values) / n, 4),
+                            "fair": round(sum(v <= fair for v in values) / n, 4)}
+    return entry
+
+
+def delivery_benchmarks(con: duckdb.DuckDBPyConnection, now: datetime, window_days: int = 90, top_languages: int = 25) -> dict:
+    """Delivery percentiles of public repos by language and size; groups need 50+ repos with the metric."""
+    repos = per_repo(con, now, window_days)
+    counts = {}
+    for r in repos:
+        counts[r["language"] or "Other"] = counts.get(r["language"] or "Other", 0) + 1
+    langs = {name for name, _ in sorted(counts.items(), key=lambda kv: -kv[1])[:top_languages]}
+    groups = {}
+    for r in repos:
+        lang = r["language"] if r["language"] in langs else "Other"
+        for key in (f"{lang}|{tier(r['stars'])}", f"{lang}|all", f"all|{tier(r['stars'])}", "all|all"):
+            groups.setdefault(key, []).append(r)
+    out = {}
+    for key, items in groups.items():
+        if len(items) < MIN_GROUP:
+            continue
+        entry = {"repos": len(items)}
+        for metric in METRICS:
+            values = [r[metric] for r in items if r[metric] is not None]
+            if len(values) >= MIN_GROUP:
+                entry[metric] = _entry(metric, values)
+        out[key] = entry
+    return {"version": 1, "built_at": now.strftime("%Y-%m-%dT%H:%M:%SZ"), "window_days": window_days, "repos": len(repos),
+            "definition": DEFINITION, "population": POPULATION, "gaps": GAPS, "bands_note": BANDS_NOTE,
+            "method": {"percentile": "nearest rank", "bootstrap_resamples": RESAMPLES, "bootstrap_seed": SEED,
+                       "range": "95% of the resampled medians"},
+            "groups": out}

mergescout/signals/deps_map.py

@@ -0,0 +1,261 @@
+"""Package -> source repo map from deps.dev (free, anonymous): the default version, release history, licence, advisories,
+and how the repo link is known. A signed link is "verified"; a claimed one is checked against the repo's own manifest
+(user decision D1), since anyone can point a package at a popular repo."""
+import configparser
+import http.client
+import json
+import re
+import time
+import tomllib
+import urllib.error
+import urllib.request
+from concurrent.futures import ThreadPoolExecutor
+from datetime import datetime, timedelta
+from urllib.parse import quote
+
+from mergescout.github.graphql import GraphQLError, RateLimited
+from mergescout.signals.bumps import _norm
+
+API = "https://api.deps.dev"
+SYSTEMS = {"npm": "NPM", "pypi": "PYPI"}
+BATCH = 500
+CLAIM_BATCH = 20
+SAFE = re.compile(r"^[\w.-]+$")
+SAFE_DIR = re.compile(r"^[\w.@+-]+(/[\w.@+-]+)*$")
+SKIPPED = object()
+# Pre-release markers in PyPI (1.0.0.dev6, 2.0rc1) and npm (6.0.0-beta.3) versions.
+PRERELEASE = re.compile(r"(\d(a|b|rc|c)\d+|\.?dev\d*|\.?post\d*\.dev|-[0-9A-Za-z])", re.I)
+# Dropped connections and resets come as several types; all of them are worth a retry.
+NETWORK = (OSError, http.client.HTTPException)
+
+
+def http_get_json(url: str, tries: int = 3) -> dict | None:
+    # deps.dev states no rate limit; a short back-off covers its rare 5xx answers. 404 means "not known".
+    for attempt in range(tries):
+        try:
+            with urllib.request.urlopen(urllib.request.Request(url, headers={"User-Agent": "mergescout"}), timeout=30) as r:
+                return json.loads(r.read())
+        except urllib.error.HTTPError as exc:
+            if exc.code == 404:
+                return None
+            if attempt == tries - 1:
+                raise
+        except NETWORK:
+            if attempt == tries - 1:
+                raise
+        time.sleep(2 ** attempt)
+    return None
+
+
+def http_post_json(url: str, body: dict, tries: int = 3) -> dict:
+    data = json.dumps(body).encode()
+    for attempt in range(tries):
+        try:
+            req = urllib.request.Request(url, data=data, headers={"Content-Type": "application/json", "User-Agent": "mergescout"})
+            with urllib.request.urlopen(req, timeout=60) as r:
+                return json.loads(r.read())
+        except NETWORK:
+            if attempt == tries - 1:
+                raise
+            time.sleep(2 ** attempt)
+    return {}
+
+
+def lookup_packages(packages: list[tuple[str, str]], get_json=http_get_json, post_json=http_post_json,
+                    now: datetime | None = None, workers: int = 8, budget_s: float | None = None) -> dict[tuple[str, str], dict]:
+    """Per (ecosystem, name): default version, releases, repo and link kind, licences, advisories, deprecation.
+    deps.dev slows down after many calls (23 s vs 333 s for the same 300 packages), so a time budget stops early."""
+    wanted = list(dict.fromkeys((eco, name) for eco, name in packages if eco in SYSTEMS))
+    deadline = time.time() + budget_s if budget_s else None
+
+    def fetch(pair):
+        # Past the budget nothing more is asked; those packages keep their previous answer.
+        if deadline and time.time() > deadline:
+            return SKIPPED
+        try:
+            return get_json(f"{API}/v3/systems/{SYSTEMS[pair[0]]}/packages/{quote(pair[1], safe='')}")
+        except Exception:  # noqa: BLE001
+            # One lost answer after retries only skips that package (it keeps its last answer).
+            return SKIPPED
+    with ThreadPoolExecutor(workers) as pool:
+        found = list(pool.map(fetch, wanted))
+    year_ago = ((now or datetime.now()) - timedelta(days=365)).strftime("%Y-%m-%d")
+    out, keys = {}, []
+    for (eco, name), pkg in zip(wanted, found):
+        if pkg is SKIPPED:
+            continue
+        info = {"ecosystem": eco, "package": name, "version": None, "repo": None, "link": None, "licenses": [],
+                "advisories": [], "deprecated": False, "deprecated_reason": None, "status": None, "releases_12m": 0,
+                "last_release": None, "first_release": None}
+        vs = [v for v in (pkg or {}).get("versions") or [] if v.get("publishedAt")]
+        if vs:
+            dates = sorted(v["publishedAt"][:10] for v in vs)
+            default = next((v for v in vs if v.get("isDefault")), max(vs, key=lambda v: v["publishedAt"]))
+            stable = [v for v in vs if not PRERELEASE.search(v["versionKey"]["version"])]
+            # A pre-release default (httpx 1.0.0.dev6) often lacks the repo link; the newest stable release speaks for the package.
+            if PRERELEASE.search(default["versionKey"]["version"]) and stable:
+                default = max(stable, key=lambda v: v["publishedAt"])
+            # Nightly and dev builds are not releases; a package with only pre-releases keeps them.
+            counted = sorted(v["publishedAt"][:10] for v in stable) or dates
+            info.update(version=default["versionKey"]["version"], first_release=dates[0], last_release=counted[-1],
+                        releases_12m=sum(d >= year_ago for d in counted), release_rule=RELEASE_RULE)
+            keys.append((eco, name, info["version"]))
+        out[(eco, name)] = info
+    by_key = {(SYSTEMS[e], n, v): (e, n) for e, n, v in keys}
+    for i in range(0, len(keys), BATCH):
+        body = {"requests": [{"versionKey": {"system": SYSTEMS[e], "name": n, "version": v}} for e, n, v in keys[i:i + BATCH]]}
+        token = None
+        # deps.dev answers 100 versions per page and says where to go on; answers are matched by key, not position.
+        while True:
+            page = post_json(f"{API}/v3alpha/versionbatch", {**body, "pageToken": token} if token else body)
+            for resp in page.get("responses") or []:
+                k = ((resp or {}).get("request") or {}).get("versionKey") or {}
+                found = by_key.get((k.get("system"), k.get("name"), k.get("version")))
+                if found and (resp or {}).get("version"):
+                    _fill(out[found], resp["version"])
+            token = page.get("nextPageToken")
+            if not token:
+                break
+    return out
+
+
+def _fill(info: dict, v: dict) -> None:
+    repos = [p for p in v.get("relatedProjects") or [] if p.get("relationType") == "SOURCE_REPO"]
+    # A signed build or publish attestation proves the link; plain metadata is only what the publisher typed.
+    repos.sort(key=lambda p: p.get("relationProvenance") == "UNVERIFIED_METADATA")
+    if repos:
+        project = repos[0]["projectKey"]["id"]
+        info["repo"] = project.removeprefix("github.com/")
+        info["link"] = "claimed" if repos[0].get("relationProvenance") == "UNVERIFIED_METADATA" else "verified"
+    info["licenses"] = v.get("licenses") or []
+    info["advisories"] = [a.get("id") for a in v.get("advisoryKeys") or [] if a.get("id")]
+    info["deprecated"] = bool(v.get("isDeprecated"))
+    # The registry's own words, cut short: shown as a quote, never acted on.
+    reason = " ".join((v.get("deprecatedReason") or "").split())[:160]
+    info["deprecated_reason"] = reason or None if info["deprecated"] else None
+    info["status"] = (v.get("projectStatus") or {}).get("status")
+
+
+def manifest_name(path: str, text: str | None) -> str | None:
+    """The package name a manifest declares: package.json, pyproject.toml ([project] or Poetry) or setup.cfg."""
+    if not text:
+        return None
+    try:
+        if path.endswith("package.json"):
+            return json.loads(text).get("name")
+        if path.endswith("pyproject.toml"):
+            doc = tomllib.loads(text)
+            return (doc.get("project") or {}).get("name") or ((doc.get("tool") or {}).get("poetry") or {}).get("name")
+        if path.endswith("setup.cfg"):
+            cfg = configparser.ConfigParser()
+            cfg.read_string(text)
+            return cfg.get("metadata", "name", fallback=None)
+        if path.endswith("setup.py"):
+            return _setup_py_name(text)
+    except (ValueError, tomllib.TOMLDecodeError, configparser.Error, AttributeError):
+        return None
+    return None
+
+
+def _setup_py_name(text: str) -> str | None:
+    # setup(name="x"), or setup(name=NAME) with NAME = "x" in the same file; anything computed is not guessed.
+    call = text[text.find("setup("):] if "setup(" in text else ""
+    if m := re.search(r"""\bname\s*=\s*['"]([^'"]+)['"]""", call):
+        return m.group(1)
+    if m := re.search(r"\bname\s*=\s*([A-Za-z_]\w*)\s*[,)\n]", call):
+        const = re.search(rf"""^\s*{m.group(1)}\s*=\s*['"]([^'"]+)['"]""", text, re.M)
+        return const.group(1) if const else None
+    return None
+
+
+def _npm_directory(name: str, get_json) -> str | None:
+    try:
+        d = get_json(f"https://registry.npmjs.org/{quote(name, safe='@')}/latest")
+        repo = d.get("repository") if isinstance(d, dict) else None
+        raw = repo.get("directory") if isinstance(repo, dict) else None
+    except Exception:  # noqa: BLE001
+        return None
+    # Publishers write anything here (numbers, lists): only a plain string is used.
+    path = raw.strip().removeprefix("./").strip("/") if isinstance(raw, str) else ""
+    # The folder goes into the GitHub query, so only plain path names are used.
+    return path if SAFE_DIR.match(path) and ".." not in path.split("/") else None
+
+
+def _candidates(eco: str, name: str, directory: str | None = None) -> list[str]:
+    # Where projects keep the manifest that publishes the package, from the layouts of 33 real unconfirmed links.
+    if eco == "pypi":
+        folders = ["", f"{name}/", f"{name.replace('-', '_')}/", f"libs/{name}/", f"packages/{name}/", "python/", f"{name}-python/"]
+        return list(dict.fromkeys([f"{d}{f}" for d in folders for f in ("pyproject.toml", "setup.py")] + ["setup.cfg"]))
+    short = name.split("/")[-1]
+    scoped = name.lstrip("@").replace("/", "-")
+    first = [f"{directory}/package.json"] if directory else []
+    return list(dict.fromkeys(first + ["package.json", f"packages/{short}/package.json", f"packages/{scoped}/package.json",
+                               f"{short}/package.json", "npm/package.json"]))
+
+
+def check_claims(infos: dict, gql, get_json=None) -> int:
+    """Claimed GitHub links become "claimed_checked" when the repo's manifest names the package, else
+    "claimed_unconfirmed"; about 1 GraphQL point per 25 repos. With get_json, npm's repository.directory is looked in
+    first (monorepos). Returns how many were checked."""
+    todo = [i for i in infos.values() if i.get("link") == "claimed" and i.get("repo") and "/" in i["repo"]
+            and all(SAFE.match(part) for part in i["repo"].split("/", 1))]
+    for i in [x for x in infos.values() if x.get("link") == "claimed" and x not in todo]:
+        i["link"] = "claimed_unconfirmed"
+    npm = [i for i in todo if i["ecosystem"] == "npm"] if get_json is not None else []
+    with ThreadPoolExecutor(16) as pool:
+        for info, path in zip(npm, pool.map(lambda i: _npm_directory(i["package"], get_json), npm)):
+            info["directory"] = path
+    checked = 0
+    for start in range(0, len(todo), CLAIM_BATCH):
+        chunk = todo[start:start + CLAIM_BATCH]
+        lines, files = [], {}
+        for r, info in enumerate(chunk):
+            owner, name = info["repo"].split("/", 1)
+            lines.append(f'  r{r}: repository(owner: "{owner}", name: "{name}") {{')
+            for f, path in enumerate(_candidates(info["ecosystem"], info["package"], info.get("directory"))):
+                files[(r, f)] = path
+                lines.append(f'    f{f}: object(expression: "HEAD:{path}") {{ ... on Blob {{ text }} }}')
+            lines.append("  }")
+        query = "query {\n  rateLimit { cost remaining resetAt }\n" + "\n".join(lines) + "\n}"
+        try:
+            data = gql.query(query, {}, partial=True).data
+        except (GraphQLError, RateLimited):
+            # The rest stay "claimed" (not yet checked) and are tried on the next run.
+            return checked
+        checked += len(chunk)
+        for r, info in enumerate(chunk):
+            repo = data.get(f"r{r}") or {}
+            names = {manifest_name(path, ((repo.get(f"f{f}") or {}).get("text"))) for (rr, f), path in files.items() if rr == r}
+            match = {_norm(info["ecosystem"], n) for n in names if n} & {_norm(info["ecosystem"], info["package"])}
+            info["link"] = "claimed_checked" if match else "claimed_unconfirmed"
+    return checked
+
+
+MAX_AGE_DAYS = 7
+# How releases are counted; answers from an older rule are looked up again (2: pre-releases no longer count).
+RELEASE_RULE = 2
+
+
+def refresh_map(packages: list[tuple[str, str]], previous: dict, now: datetime, gql=None, budget_s: float = 600,
+                lookup=lookup_packages, check=check_claims, get_json=http_get_json) -> dict:
+    """Yesterday's map with new and week-old packages looked up again (within the time budget) and their claimed
+    links checked when a GitHub token is at hand; a package skipped by the budget keeps its last answer."""
+    stamp = now.strftime("%Y-%m-%dT%H:%M:%SZ")
+    stale_before = (now - timedelta(days=MAX_AGE_DAYS)).strftime("%Y-%m-%dT%H:%M:%SZ")
+    keep, todo = {}, []
+    for eco, name in dict.fromkeys(packages):
+        old = previous.get(f"{eco}:{name}")
+        # Answers built on a pre-release (before stable versions were preferred) are asked again.
+        if (old and (old.get("fetched_at") or "") >= stale_before and not PRERELEASE.search(old.get("version") or "")
+                and old.get("release_rule", 1) >= RELEASE_RULE):
+            keep[f"{eco}:{name}"] = old
+        else:
+            todo.append((eco, name))
+            if old:
+                keep[f"{eco}:{name}"] = old
+    fresh = lookup(todo, now=now, budget_s=budget_s)
+    if gql is not None:
+        check(fresh, gql, get_json=get_json)
+    for (eco, name), info in fresh.items():
+        keep[f"{eco}:{name}"] = {**info, "fetched_at": stamp}
+    return keep

mergescout/signals/deps_trends.py

@@ -0,0 +1,117 @@
+"""How a package's source repo treats outside work, quarter by quarter, from the crawl's PR history (12 months): outside
+PRs merged, first maintainer answer, how much merging rests on one person; compared with repos of similar size."""
+from datetime import datetime
+
+import duckdb
+
+from mergescout.bench.ci import tier
+
+MIN_DECIDED = 5
+MIN_RESPONSES = 5
+MIN_MERGES = 5
+# Changes smaller than these are "steady": quarters of a few dozen PRs move around on their own.
+RATE_STEP = 0.15
+SPEED_RATIO = 1.5
+# Automation that merges (Actions, merge queues, any login the crawl saw acting as a bot) is not one person merging.
+BOT_MERGER = ("(lower({col}) LIKE '%[bot]' OR lower({col}) IN ('github-actions', 'github-merge-queue', 'mergify', 'kodiakhq', "
+              "'bors', 'dependabot', 'renovate') OR lower({col}) IN (SELECT lower(login) FROM people WHERE kind = 'bot'))")
+
+
+def _quarters(now: datetime, n: int = 4) -> list[tuple[int, int]]:
+    # The last n complete quarters, oldest first (the running quarter is partial).
+    year, q = now.year, (now.month - 1) // 3 + 1
+    out = []
+    for _ in range(n):
+        q -= 1
+        if q == 0:
+            year, q = year - 1, 4
+        out.append((year, q))
+    return out[::-1]
+
+
+def repo_trends(con: duckdb.DuckDBPyConnection, repos: list[str], now: datetime) -> dict[str, dict]:
+    """Per repo ("owner/name"): quarterly lines, their direction and the latest quarter's place among similar repos."""
+    quarters = _quarters(now)
+    lo = datetime(quarters[0][0], 3 * quarters[0][1] - 2, 1, tzinfo=now.tzinfo)
+    end_y, end_q = (now.year, (now.month - 1) // 3 + 1)
+    hi = datetime(end_y, 3 * end_q - 2, 1, tzinfo=now.tzinfo)
+    wanted = sorted({r.lower() for r in repos})
+    if not wanted:
+        return {}
+    rows = con.execute(f"""
+        WITH wanted AS (SELECT repo_id, lower(name) AS repo, stars FROM repo_meta WHERE lower(name) IN (SELECT unnest($repos))),
+        w AS (SELECT p.*, wanted.repo, year(timezone('UTC', p.created_at)) AS y, quarter(timezone('UTC', p.created_at)) AS q
+              FROM pr_class p JOIN wanted USING (repo_id) WHERE p.created_at >= $lo AND p.created_at < $hi)
+        SELECT w.repo, w.y, w.q,
+               count(*) FILTER (WHERE author_class = 'outsider') AS opened,
+               count(*) FILTER (WHERE author_class = 'outsider' AND state IN ('MERGED', 'CLOSED')) AS decided,
+               count(*) FILTER (WHERE author_class = 'outsider' AND merged_at IS NOT NULL) AS merged,
+               median(epoch(first_human_review_at - created_at) / 3600) FILTER (
+                   WHERE author_class = 'outsider' AND first_human_review_at >= created_at AND pe.kind = 'insider') AS response,
+               count(*) FILTER (WHERE author_class = 'outsider' AND first_human_review_at >= created_at AND pe.kind = 'insider')
+                   AS responses,
+               count(*) FILTER (WHERE merged_at IS NOT NULL AND merged_by IS NOT NULL
+                                AND NOT {BOT_MERGER.format(col='merged_by')}) AS merges
+        FROM w LEFT JOIN people pe ON pe.repo_id = w.repo_id AND pe.login = w.first_human_review_by
+        GROUP BY ALL""", {"repos": wanted, "lo": lo, "hi": hi}).fetchall()
+    tops = {}
+    for repo, y, q, n in con.execute(f"""
+            SELECT lower(rm.name), year(timezone('UTC', p.created_at)), quarter(timezone('UTC', p.created_at)), count(*)
+            FROM pr_class p JOIN repo_meta rm USING (repo_id)
+            WHERE lower(rm.name) IN (SELECT unnest($repos)) AND p.merged_at IS NOT NULL AND p.merged_by IS NOT NULL
+              AND NOT {BOT_MERGER.format(col='p.merged_by')} AND p.created_at >= $lo AND p.created_at < $hi
+            GROUP BY 1, 2, 3, p.merged_by""", {"repos": wanted, "lo": lo, "hi": hi}).fetchall():
+        tops[(repo, y, q)] = max(tops.get((repo, y, q), 0), n)
+    stars = dict(con.execute("SELECT lower(name), stars FROM repo_meta WHERE lower(name) IN (SELECT unnest($repos))",
+                             {"repos": wanted}).fetchall())
+    cells = {(r[0], r[1], r[2]): r for r in rows}
+    out = {}
+    for repo in wanted:
+        if repo not in stars:
+            continue
+        lines = []
+        for y, q in quarters:
+            r = cells.get((repo, y, q))
+            opened, decided, merged, response, responses, merges = r[3:] if r else (0, 0, 0, None, 0, 0)
+            lines.append({"quarter": f"{y}Q{q}", "outside_opened": opened, "outside_decided": decided,
+                          "outside_merge_rate": round(merged / decided, 2) if decided >= MIN_DECIDED else None,
+                          "first_response_hours": round(response, 1) if responses >= MIN_RESPONSES else None,
+                          "merges": merges,
+                          "top_merger_share": round(tops.get((repo, y, q), 0) / merges, 2) if merges >= MIN_MERGES else None})
+        out[repo] = {"stars": stars[repo], "quarters": lines, "direction": _direction(lines)}
+    _peers(out)
+    return out
+
+
+def _direction(lines: list[dict]) -> dict:
+    """Earlier half against later half of the year, in plain words; None when either half has no value."""
+    def halves(key):
+        a = [x[key] for x in lines[:2] if x[key] is not None]
+        b = [x[key] for x in lines[2:] if x[key] is not None]
+        return (sum(a) / len(a), sum(b) / len(b)) if a and b else (None, None)
+    out = {}
+    a, b = halves("outside_merge_rate")
+    out["outside_merge_rate"] = None if a is None else "falling" if b - a <= -RATE_STEP else "rising" if b - a >= RATE_STEP else "steady"
+    a, b = halves("first_response_hours")
+    out["first_response_hours"] = None if a is None else ("slower" if b >= a * SPEED_RATIO and b - a > 24 else
+                                                          "faster" if a >= b * SPEED_RATIO and a - b > 24 else "steady")
+    return out
+
+
+def _peers(out: dict) -> None:
+    # The latest quarter against mapped repos of the same stars tier (all of them when a tier has under 5).
+    latest = {repo: t["quarters"][-1] for repo, t in out.items()}
+    for repo, t in out.items():
+        same = [r for r in out if r != repo and tier(out[r]["stars"]) == tier(t["stars"])]
+        pool = same if len(same) >= 5 else [r for r in out if r != repo]
+        t["peers"] = {}
+        for key, better, words in (("outside_merge_rate", 1, ("merges outside PRs more than most similar repos",
+                                                               "merges outside PRs less than most similar repos")),
+                                   ("first_response_hours", -1, ("faster than most similar repos", "slower than most similar repos"))):
+            mine = latest[repo][key]
+            others = [latest[r][key] for r in pool if latest[r][key] is not None]
+            if mine is None or not others:
+                continue
+            ahead = sum(1 for v in others if (v - mine) * better > 0) / len(others)
+            place = words[1] if ahead >= 0.75 else words[0] if ahead <= 0.25 else "about typical for similar repos"
+            t["peers"][key] = {"place": place, "peers": len(others)}