stackvitals 0.1.0__py3-none-any.whl

stackvitals/deps/dossier.py

@@ -0,0 +1,341 @@
+"""One package's dossier: use / monitor / investigate, and the evidence lines behind it, each with its counts, dates and
+source. Never "safe": the evidence can only say what happened, not what will."""
+import re
+from datetime import datetime, timedelta
+
+from mergescout.signals.bumps import _norm
+
+# Upgrade rates are shown from 30 PRs in 10 repos; thinner cells only say how much evidence there is.
+MIN_PRS, MIN_REPOS, MIN_CI = 30, 10, 20
+EXCESS_FAIL = 0.10
+# Full-crawl backtest (2026-10-20): flagged at 30% closed saw 1.45x the later closes of others, 40% 1.74x, 50% 1.79x, 60% 2.1x (13).
+CLOSED_SHARE = 0.40
+REVERT_SHARE = 0.03
+STALE_DAYS = 730
+ONE_MERGER = 0.8
+SIZES = {"major": "Major upgrades", "minor": "Minor upgrades", "patch": "Patch upgrades", "nonmajor": "Minor or patch upgrades",
+         "range": "Range updates", "unknown": "Upgrades (size unknown)", "combined": "Upgrades (sizes combined)"}
+# A steady merge rate is a good sign only when outside PRs actually get merged.
+GOOD_MERGE_RATE = 0.3
+# Of 440 crawled repos with 20+ outside PRs last quarter, half had 24%+ still open and 1 in 10 had 70%+ (2026-10-18).
+MIN_OPENED = 20
+STILL_OPEN = 0.7
+# A fall under 15 points from the first quarter shown to the last is noise (22 of 102 "falling" repos, 2026-10-18).
+FALL_STEP = 0.15
+# 3+ advisories first published in 90 days: 4% of mapped packages (nltk, vllm, axios, electron), 2026-10-18.
+BURST = 3
+# EPSS at 10% or more: a common action line, a few percent of all scored CVEs (FIRST's own guidance names 10%).
+EPSS_ALARM = 0.1
+BAD_STATUS = {"ARCHIVED", "QUARANTINED", "DEPRECATED"}
+
+
+def _pct(x: float) -> int:
+    return round(100 * x)
+
+
+def _line(signal: str, text: str, source: str) -> dict:
+    return {"signal": signal, "text": text, "source": source}
+
+
+def dossier(ecosystem: str, name: str, ev: dict, now: datetime, versions: list[str] | None = None) -> dict:
+    eco = ecosystem.lower()
+    pkg = _norm(eco, name)
+    key = f"{eco}:{pkg}"
+    m, cells, osv = ev["map"].get(key), ev["ledger"].get(key, []), ev["osv"].get(key)
+    out = {"ecosystem": eco, "package": pkg, "verdict": None, "reasons": [], "lines": [], "repo": (m or {}).get("repo"),
+           "osv": {"open_severity": (osv or {}).get("open_severity")},
+           "link": (m or {}).get("link"), "as_of": ev.get("built_at")}
+    if not m and not cells:
+        out["lines"].append(_line("info", f"{pkg} is not in the evidence set yet: packages join when crawled repos upgrade them.",
+                                  "ledger"))
+        return out
+    lines = (_osv_lines(osv, ev.get("osv_complete", True), (m or {}).get("version"), versions) + _status_lines(m) + _link_lines(m, now)
+             + _release_lines(m, now))
+    upgrade_lines, upgrade_history = _ledger_lines(cells)
+    trend_lines, trend_history = _trend_lines(ev["trends"].get(((m or {}).get("repo") or "").lower()), m)
+    lines += upgrade_lines + trend_lines + _owner_lines((ev.get("owners") or {}).get(key), now)
+    out["lines"] = lines
+    alarms = [x["text"] for x in lines if x["signal"] == "alarm"]
+    warns = [x["text"] for x in lines if x["signal"] == "warn"]
+    if alarms:
+        out["verdict"], out["reasons"] = "investigate", alarms
+    elif warns:
+        out["verdict"], out["reasons"] = "monitor", warns
+    elif not (upgrade_history or trend_history):
+        # Too little history for any verdict (real data: 925 of 2,266 packages); saying "monitor" would mean nothing.
+        out["reasons"] = ["Not enough evidence yet: too few upgrades seen and no maintainer trend to judge by."]
+    elif not any(x["signal"] == "good" and x["source"] in ("ledger", "crawl") for x in lines):
+        # Real data: gradio's only history was a low upgrade merge rate; "use" with no good evidence would have no reasons.
+        out["reasons"] = ["Not enough evidence for a verdict: the history seen shows nothing clearly good or bad."]
+    else:
+        # History leads: how upgrades and maintainers behaved says more than an empty advisory list.
+        good = sorted((x for x in lines if x["signal"] == "good"), key=lambda x: x["source"] not in ("ledger", "crawl"))
+        out["verdict"], out["reasons"] = "use", [x["text"] for x in good][:3]
+    return out
+
+
+def _vkey(v: str | None) -> tuple:
+    from mergescout.signals.osv import version_key
+    return version_key(v)
+
+
+def _in_span(v: str, span: list) -> bool:
+    """Whether a version sits in an advisory's [introduced, fixed) span, or [introduced, last_affected] when inclusive."""
+    start, end, inclusive = span[0], span[1], len(span) > 2 and span[2]
+    if _vkey(v) < _vkey(start):
+        return False
+    return end is None or (_vkey(v) <= _vkey(end) if inclusive else _vkey(v) < _vkey(end))
+
+
+def affected_by(osv: dict | None, versions: list[str], disputed: bool = True) -> list[dict]:
+    """The advisories whose spans hold one of these versions (a version that doesn't parse is never judged)."""
+    known = [v for v in versions or [] if re.match(r"v?\d", v or "")]
+    return [a for a in (osv or {}).get("affects") or [] if (disputed or not a.get("disputed"))
+            and any(_in_span(v, s) for v in known for s in a["ranges"])]
+
+
+def _osv_lines(osv: dict | None, complete: bool, version: str | None = None,
+               versions: list[str] | None = None) -> list[dict]:
+    if osv is None:
+        return [_line("info", "Not checked against OSV yet.", "osv")]
+    out = []
+    ids = osv.get("ids") or []
+    if osv.get("malware"):
+        mal = ", ".join((osv.get("malware_ids") or [])[:2]) or next((i for i in ids if i.startswith("MAL-")), ids[0] if ids else "a MAL- record")
+        bad = osv.get("malware_versions") or []
+        if bad == ["all"]:
+            what = "every published version is listed as malicious"
+        elif bad:
+            listed = ", ".join(bad[-5:]) + (f" and {len(bad) - 5} more" if len(bad) > 5 else "")
+            what = f"version{'s' if len(bad) > 1 else ''} {listed} {'were' if len(bad) > 1 else 'was'} malicious"
+            # In a PR the version being installed matters, not the registry's current one.
+            hit = [v for v in versions or [] if v in bad]
+            if versions:
+                what += f"; the version installed here ({', '.join(hit or versions)}) is {'one' if hit else 'not one'} of them"
+            elif version:
+                what += f"; the current version {version} is {'one of them' if version in bad else 'not one of them'}"
+        else:
+            what = "a published version was malicious"
+        out.append(_line("alarm", f"OSV has a malware record for it ({mal}): {what}.", "osv"))
+    # Exploited in the wild beats any severity label; EPSS says how likely an open one is to be exploited soon.
+    e = osv.get("epss_open")
+    for k in osv.get("kev") or []:
+        if k.get("fixed"):
+            out.append(_line("warn", f"{k['id']} was exploited in the wild (CISA KEV); it is fixed in newer releases, so "
+                                     "stay on a fixed version.", "kev"))
+        else:
+            odds = f"; EPSS puts the chance of exploitation in the next 30 days at {e['score']:.0%}" if e and e["id"] == k["id"] else ""
+            out.append(_line("alarm", f"{k['id']} is being exploited in the wild (CISA KEV) and has no fixed version yet{odds}.",
+                             "kev"))
+    if e and e.get("score", 0) >= EPSS_ALARM and not any(k["id"] == e["id"] for k in osv.get("kev") or []):
+        out.append(_line("alarm", f"An open advisory ({e['id']}) has a {e['score']:.0%} chance of being exploited in the next "
+                                  "30 days (EPSS).", "epss"))
+    # In a PR, the version being installed is checked against each advisory's affected spans.
+    for v in versions or []:
+        hit = [(a["id"], s) for a in affected_by(osv, [v]) for s in a["ranges"] if _in_span(v, s)]
+        if hit:
+            named = "; ".join(f"{i}, " + ("affected up to " + s[1] + ", no fix named" if len(s) > 2 and s[2]
+                                          else "fixed in " + s[1] if s[1] else "no fixed version yet") for i, s in hit[:3])
+            more = f" and {len(hit) - 3} more" if len(hit) > 3 else ""
+            out.append(_line("alarm", f"Version {v}, the one installed here, is affected by {named}{more} (OSV).", "osv"))
+    if osv.get("unfixed"):
+        n = osv["unfixed"]
+        out.append(_line("alarm", f"{n} {'advisory has' if n == 1 else 'advisories have'} no fixed version yet (OSV).", "osv"))
+    # A range that ends at a "last affected" version is still open only while the newest release is inside it.
+    inside = [b for b in osv.get("last_affected") or [] if version and _vkey(version) <= _vkey(b)]
+    if inside:
+        out.append(_line("alarm", f"The newest release {version} is still inside an advisory's affected range (up to "
+                                  f"{inside[-1]}, OSV).", "osv"))
+    if osv.get("disputed"):
+        out.append(_line("info", f"{osv['disputed']} advisory is disputed by the maintainers (OSV)." if osv["disputed"] == 1
+                         else f"{osv['disputed']} advisories are disputed by the maintainers (OSV).", "osv"))
+    recent = osv.get("last_90d") or 0
+    if recent >= BURST:
+        out.append(_line("warn", f"{recent} advisories in the last 90 days: it is under active security scrutiny, so stay on "
+                                 "the newest version (OSV).", "osv"))
+    loading = "" if complete else " History still loading."
+    if osv.get("advisories"):
+        n = osv["advisories"]
+        out.append(_line("info", f"{n} {'advisory' if n == 1 else 'advisories'} in total, {osv.get('last_2y', 0)} in the last 2 years "
+                                 f"(latest {osv.get('latest')}, OSV).{loading}", "osv"))
+    else:
+        out.append(_line("good", f"No advisories in OSV.{loading}", "osv"))
+    return out
+
+
+def _status_lines(m: dict | None) -> list[dict]:
+    if not m:
+        return []
+    out = []
+    if m.get("deprecated"):
+        why = f': "{m["deprecated_reason"]}"' if m.get("deprecated_reason") else ""
+        end = "" if why and why[-2] in ".!?" else "."
+        out.append(_line("alarm", f"Its registry marks the default version deprecated{why}{end}", "deps.dev"))
+    if (m.get("status") or "").upper() in BAD_STATUS:
+        out.append(_line("alarm", f"PyPI marks the project {m['status'].lower()}.", "deps.dev"))
+    return out
+
+
+def _link_lines(m: dict | None, now: datetime) -> list[dict]:
+    if not m:
+        return [_line("warn", "Its source repo isn't known yet.", "deps.dev")]
+    repo, link = m.get("repo"), m.get("link")
+    if not repo:
+        # Established packages sometimes live off GitHub (beautifulsoup4, docutils); a young one with no source is the risky case.
+        young = (m.get("first_release") or "") >= (now - timedelta(days=STALE_DAYS)).strftime("%Y-%m-%d")
+        return [_line("alarm" if young else "warn",
+                      f"No source repo is linked{' and it is under 2 years old' if young else ''}, so its maintainer history "
+                      "can't be checked.", "deps.dev")]
+    if link == "verified":
+        return [_line("info", f"Source repo {repo}, proven by a signed build or publish attestation (deps.dev).", "deps.dev")]
+    if link == "claimed_checked":
+        return [_line("info", f"Source repo {repo}: the package names it and that repo's manifest names the package back.",
+                      "deps.dev + GitHub")]
+    if link == "claimed_unconfirmed":
+        # Monorepos keep the manifest in a subfolder (swiper, postcss-lab-function, typescript): a warning only for young packages.
+        young = (m.get("first_release") or "") >= (now - timedelta(days=STALE_DAYS)).strftime("%Y-%m-%d")
+        if young:
+            return [_line("warn", f"It names {repo} as its source, but no manifest we checked there names it back and the "
+                                  "package is under 2 years old: check the real source before trusting this history.",
+                          "deps.dev + GitHub")]
+        return [_line("info", f"It names {repo} as its source; that repo's top-level manifest doesn't name it (monorepos and "
+                              "generated packages often do this).", "deps.dev + GitHub")]
+    return [_line("warn", f"It names {repo} as its source; not checked yet.", "deps.dev")]
+
+
+def _release_lines(m: dict | None, now: datetime) -> list[dict]:
+    if not m or not m.get("last_release"):
+        return []
+    last = m["last_release"]
+    if last < (now - timedelta(days=STALE_DAYS)).strftime("%Y-%m-%d"):
+        return [_line("warn", f"There has been no release since {last}.", "deps.dev")]
+    return [_line("info", f"{m.get('releases_12m', 0)} releases in the last 12 months, latest {last} (version {m.get('version')}).",
+                  "deps.dev")]
+
+
+def _ledger_lines(cells: list[dict]) -> tuple[list[dict], bool]:
+    strong = [c for c in cells if c["prs"] >= MIN_PRS and c["repos"] >= MIN_REPOS]
+    if not strong and cells and sum(c["prs"] for c in cells) >= MIN_PRS and max(c["repos"] for c in cells) >= MIN_REPOS:
+        # Real data: httpx had 33 upgrade PRs from 23 repos split across sizes; together they are enough evidence.
+        keys = ("prs", "merged", "closed_unmerged", "ci_known", "ci_failed", "expected_failed", "extra_commits", "reverted")
+        strong = [{"bump": "combined", "repos": max(c["repos"] for c in cells), "eco_inferred": all(c.get("eco_inferred") for c in cells),
+                   **{k: sum(c.get(k) or 0 for c in cells) for k in keys}}]
+    if not strong:
+        if not cells:
+            return [_line("info", "No Dependabot or Renovate upgrades of it seen in crawled repos yet.", "ledger")], False
+        prs, repos = sum(c["prs"] for c in cells), max(c["repos"] for c in cells)
+        return [_line("info", f"Not enough upgrade evidence yet ({prs} PRs from {repos} repos; rates are shown from "
+                              f"{MIN_PRS} PRs in {MIN_REPOS} repos).", "ledger")], False
+    out = []
+    for c in sorted(strong, key=lambda c: list(SIZES).index(c["bump"]) if c["bump"] in SIZES else 99):
+        label = SIZES.get(c["bump"], f"{c['bump'].capitalize()} upgrades")
+        fail = c["ci_failed"] / c["ci_known"] if c["ci_known"] >= MIN_CI else None
+        usual = (c.get("expected_failed") or 0) / c["ci_known"] if fail is not None else None
+        closed, reverted = c["closed_unmerged"] / c["prs"], c["reverted"] / c["prs"]
+        trouble = (fail is not None and fail - usual >= EXCESS_FAIL) or closed >= CLOSED_SHARE or reverted >= REVERT_SHARE
+        if trouble and fail is not None and fail - usual >= EXCESS_FAIL:
+            text = (f"{label} failed CI in {_pct(fail)}% of {c['ci_known']} PRs from {c['repos']} repos; "
+                    f"usually {_pct(usual)}% in these repos")
+        else:
+            text = f"{label}: {_pct(c['merged'] / c['prs'])}% of {c['prs']} PRs in {c['repos']} repos merged"
+            if fail is not None:
+                text += f"; CI failed in {_pct(fail)}% (usually {_pct(usual)}% in these repos)"
+        if closed >= CLOSED_SHARE:
+            text += f"; {_pct(closed)}% closed without merging"
+        if c["reverted"]:
+            text += f"; {c['reverted']} reverted within 14 days"
+        if c.get("extra_commits"):
+            text += f"; {c['extra_commits']} needed extra commits"
+        if c.get("eco_inferred"):
+            # Renovate's "update dependency X" names no ecosystem; a guess from the repo's language can count another package
+            # (real data: Docker "ubuntu" bumps counted as npm's "ubuntu"), so it is shown but never decides.
+            out.append(_line("info", text + " (crawl, last 180 days; ecosystem guessed from the repo's language).", "ledger"))
+            continue
+        merged = c["merged"] / c["prs"]
+        out.append(_line("warn" if trouble else "good" if merged >= 0.5 else "info", text + " (crawl, last 180 days).", "ledger"))
+    return out, any(not c.get("eco_inferred") for c in strong)
+
+
+def _trend_lines(trend: dict | None, m: dict | None) -> tuple[list[dict], bool]:
+    if not m or not m.get("repo"):
+        return [], False
+    if not trend:
+        return [_line("info", "Its source repo isn't crawled yet; maintainer trends arrive once it is.", "crawl")], False
+    q, d = trend.get("quarters") or [], trend.get("direction") or {}
+    rates = [x["outside_merge_rate"] for x in q if x.get("outside_merge_rate") is not None]
+    hours = [x["first_response_hours"] for x in q if x.get("first_response_hours") is not None]
+    out, last = [], (q[-1] if q else {})
+    opened, decided = last.get("outside_opened") or 0, last.get("outside_decided") or 0
+    # Merge rates count decided PRs only; a quarter where most outside PRs got no decision is the real signal (dask: 71 of 89).
+    if opened >= MIN_OPENED and (opened - decided) / opened >= STILL_OPEN:
+        out.append(_line("warn", f"{opened - decided} of {opened} outside PRs opened last quarter are still open.", "crawl"))
+    quiet = 0
+    for x in reversed(q):
+        if (x.get("outside_opened") or 0) > 0:
+            break
+        quiet += 1
+    known = [x for x in q if x.get("outside_merge_rate") is not None]
+    if 2 <= quiet < len(q) and known:
+        # Real data (httpx): 23-28 outside PRs a quarter, then none at all; the empty quarters are the news.
+        k = known[-1]
+        out.append(_line("warn", f"No outside pull requests were opened in the last {quiet} quarters; before that, about "
+                                 f"{_pct(k['outside_merge_rate'])}% of them were merged ({k['quarter']}).", "crawl"))
+    elif not rates and not hours and last.get("top_merger_share") is None:
+        out.append(_line("info", "Its repo had too few outside PRs in the last year to show a trend.", "crawl"))
+    n = f" ({decided} decided)" if decided else ""
+    if d.get("outside_merge_rate") == "falling" and rates:
+        out.append(_line("warn" if rates[0] - rates[-1] >= FALL_STEP else "info",
+                         f"Outside PRs merged fell from {_pct(rates[0])}% to {_pct(rates[-1])}% over the last 4 quarters{n}.", "crawl"))
+    elif d.get("outside_merge_rate") and rates:
+        good = rates[-1] >= GOOD_MERGE_RATE
+        out.append(_line("good" if good else "info", f"Outside PRs merged: {d['outside_merge_rate']}, about {_pct(rates[-1])}% "
+                                                    f"last quarter{n}.", "crawl"))
+    if d.get("first_response_hours") == "slower" and hours:
+        out.append(_line("warn", f"First maintainer answers slowed from about {hours[0]:.0f} to {hours[-1]:.0f} hours.", "crawl"))
+    elif d.get("first_response_hours") and hours:
+        out.append(_line("good", f"First maintainer answers: {d['first_response_hours']}, about {hours[-1]:.0f} hours last quarter.",
+                         "crawl"))
+    top = (q[-1] if q else {}).get("top_merger_share")
+    # One lead merging and a place among peers are facts to know, not warnings (real data: one lead is normal, and big
+    # libraries merge fewer drive-by PRs); only a worsening trend warns.
+    if top is not None and top >= ONE_MERGER:
+        out.append(_line("info", f"Merging rests on one person: {_pct(top)}% of last quarter's merges.", "crawl"))
+    names = {"outside_merge_rate": "Outside PRs merged", "first_response_hours": "First maintainer answers"}
+    for metric, peer in (trend.get("peers") or {}).items():
+        out.append(_line("info", f"{names.get(metric, metric)} against {peer['peers']} similar repos: {peer['place']}.", "crawl"))
+    return out, any(v for v in d.values())
+
+
+OWNER_DAYS = 90
+
+
+def _owner_lines(watch: dict | None, now: datetime) -> list[dict]:
+    """Recent changes in who publishes it (a takeover usually starts here), and who does now."""
+    if not watch:
+        return []
+    out, recent = [], (now - timedelta(days=OWNER_DAYS)).strftime("%Y-%m-%d")
+    for e in watch.get("events") or []:
+        if e["date"] < recent:
+            continue
+        if e["kind"] == "trusted_lost":
+            out.append(_line("alarm", f"On {e['date']}, release {e.get('version')} was published without the trusted CI "
+                                      "publishing used before.", "registry watch"))
+        elif e["kind"] == "maintainer_added":
+            out.append(_line("warn", f"New maintainer{'s' if len(e['who']) > 1 else ''} added on {e['date']}: "
+                                     f"{', '.join(e['who'])}.", "registry watch"))
+        elif e["kind"] == "organization_changed":
+            out.append(_line("warn", f"The owning organization changed on {e['date']}: {e.get('from')} to {e.get('to')}.",
+                             "registry watch"))
+        elif e["kind"] == "maintainer_removed":
+            out.append(_line("info", f"Maintainer{'s' if len(e['who']) > 1 else ''} removed on {e['date']}: {', '.join(e['who'])}.",
+                             "registry watch"))
+    cur = watch.get("current") or {}
+    names = ", ".join(cur.get("maintainers") or []) or "none listed"
+    if "trusted" in cur:
+        how = "came through trusted CI publishing" if cur.get("trusted") else f"was published by {cur.get('publisher')}"
+        out.append(_line("info", f"Latest release {cur.get('version')} {how}; maintainers: {names} (watched since "
+                                 f"{watch.get('since')}).", "registry watch"))
+    else:
+        org = f", organization {cur['organization']}" if cur.get("organization") else ""
+        out.append(_line("info", f"Owners on PyPI: {names}{org} (watched since {watch.get('since')}).", "registry watch"))
+    return out

stackvitals/deps/evidence.py

@@ -0,0 +1,44 @@
+"""The dependency evidence the daily pipeline publishes: upgrade ledger, package map, trend lines, vulnerability history."""
+from pathlib import Path
+
+from stackvitals.ci import bench as ci_bench
+
+FILES = {"ledger": ("deps-ledger.json", "cells"), "map": ("deps-map.json", "packages"),
+         "trends": ("deps-trends.json", "repos"), "osv": ("deps-osv.json", "packages"),
+         "owners": ("deps-owners.json", "packages")}
+
+
+def _validator(key: str):
+    # A file without its main part is ignored, so a bad download can never break a check.
+    return lambda d: d if isinstance(d, dict) and isinstance(d.get(key), (dict, list)) else None
+
+
+def load_evidence(token: str | None, cache_dir: Path, client=None) -> dict | None:
+    """All five files (each cached a day); None without the package map, which every dossier starts from."""
+    parts = {name: ci_bench.load(token, cache_dir, client=client, name=file, url_env=f"STACKVITALS_DEPS_{name.upper()}_URL",
+                                 validate=_validator(key), default_url=None)
+             for name, (file, key) in FILES.items()}
+    if not parts["map"]:
+        return None
+    return index_evidence(parts["ledger"] or {}, parts["map"], parts["trends"] or {}, parts["osv"] or {}, parts["owners"] or {})
+
+
+def index_evidence(ledger: dict, mapped: dict, trends: dict, osv: dict, owners: dict | None = None) -> dict:
+    cells = {}
+    for c in ledger.get("cells") or []:
+        if isinstance(c, dict) and c.get("package") and c.get("ecosystem"):
+            cells.setdefault(f"{c['ecosystem']}:{c['package']}", []).append(c)
+    return {"ledger": cells, "map": mapped.get("packages") or {},
+            "trends": {k.lower(): v for k, v in (trends.get("repos") or {}).items()},
+            "osv": osv.get("packages") or {}, "osv_complete": osv.get("complete", True),
+            "owners": (owners or {}).get("packages") or {},
+            "built_at": {"ledger": ledger.get("built_at"), "map": mapped.get("built_at"), "trends": trends.get("built_at"),
+                         "osv": osv.get("built_at"), "owners": (owners or {}).get("built_at")}}
+
+
+def load_ledger(token: str | None, cache_dir: Path, client=None) -> dict | None:
+    """Only the upgrade ledger, indexed by "ecosystem:package" (cached a day); None when it can't be read."""
+    file, key = FILES["ledger"]
+    data = ci_bench.load(token, cache_dir, client=client, name=file, url_env="STACKVITALS_DEPS_LEDGER_URL",
+                         validate=_validator(key), default_url=None)
+    return index_evidence(data, {}, {}, {})["ledger"] if data else None

stackvitals/deps/fixplan.py

@@ -0,0 +1,40 @@
+"""Guided fixes (OSV-Scanner's guided remediation, without its resolver): for each finding, the smallest upgrade that
+fixes it, how big a jump that is (a major one may break things), and the command that makes it in that ecosystem."""
+import re
+
+COMMANDS = {
+    "npm": "npm install {name}@{to}",
+    "PyPI": "pip install \"{name}>={to}\"  (uv: uv add \"{name}>={to}\")",
+    "Go": "go get {name}@v{to}",
+    "crates.io": "cargo update -p {name} --precise {to}",
+    "RubyGems": "bundle update {name} --conservative",
+    "Packagist": "composer require {name}:^{to}",
+    "NuGet": "dotnet add package {name} --version {to}",
+    "Maven": "set {name} to {to} in the pom or Gradle file",
+    "Pub": "dart pub upgrade {name}",
+    "Hex": "mix deps.update {name}",
+    "GitHub Actions": "use {name}@v{to} (or pin its commit) in the workflow",
+}
+
+
+def _parts(version: str) -> list[int]:
+    nums = [int(p) for p in re.findall(r"\d+", str(version).lstrip("v"))[:3]]
+    return nums + [0] * (3 - len(nums))
+
+
+def bump(current: str, to: str) -> str:
+    """"major", "minor" or "patch": how far the upgrade moves (0.x minors count as major, as semver treats them)."""
+    a, b = _parts(current), _parts(to)
+    if b[0] != a[0] or (a[0] == 0 and b[1] != a[1]):
+        return "major"
+    return "minor" if b[1] != a[1] else "patch"
+
+
+def plan(finding: dict) -> dict | None:
+    """{"to", "bump", "command"} for a finding with a fixed version; None when no fix exists yet."""
+    to = finding.get("fixed")
+    if not to or finding.get("malware"):
+        return None
+    eco, name = finding.get("ecosystem") or "", finding.get("package") or ""
+    template = COMMANDS.get(eco, "upgrade {name} to {to}")
+    return {"to": to, "bump": bump(finding.get("version") or "0", to), "command": template.format(name=name, to=str(to).lstrip("v"))}

stackvitals/deps/guarddog.py

@@ -0,0 +1,32 @@
+"""Opt-in: DataDog's GuardDog on each npm or PyPI package version a pull request adds or upgrades. It downloads the
+package and checks its code for what malware does (install hooks that fetch and run code, obfuscation, data sent out);
+on a 2026 npm benchmark it was right on 97% of what it flagged and caught 90% of the malware. Pinned, run through uv."""
+import json
+import re
+import subprocess
+
+VERSION = "3.2.0"
+MAX_PACKAGES = 10
+ECOSYSTEMS = {"npm": "npm", "pypi": "pypi"}
+# A plain npm or PyPI name (npm may have a scope): never an option, a path or a URL, which GuardDog would also take.
+NAME = re.compile(r"(@[A-Za-z0-9][\w.-]*/)?[A-Za-z0-9][\w.-]*")
+
+
+def scan(ecosystem: str, name: str, version: str, timeout: int = 180) -> list[str] | None:
+    """The rules GuardDog's scan of one package version matched, or None when it couldn't run."""
+    # Names come from a pull request's manifest: options go first and `--` ends them, so a name can't act as one.
+    if ecosystem not in ECOSYSTEMS or not NAME.fullmatch(name or "") or not version or version.startswith("-"):
+        return None
+    try:
+        run = subprocess.run(["uvx", "--quiet", "--from", f"guarddog=={VERSION}", "guarddog", ECOSYSTEMS[ecosystem], "scan",
+                              f"--version={version}", "--output-format=json", "--", name],
+                             capture_output=True, timeout=timeout)
+    except (OSError, subprocess.TimeoutExpired):
+        return None
+    try:
+        doc = json.loads(run.stdout.decode("utf-8", "replace") or "null")
+    except ValueError:
+        return None
+    if not isinstance(doc, dict):
+        return None
+    return sorted(rule for rule, hits in (doc.get("results") or {}).items() if hits)

stackvitals/deps/hygiene.py

@@ -0,0 +1,47 @@
+"""Dependency-update hygiene from the pull requests Team Flow already keeps, the view Dependabot doesn't give: open update
+PRs by age, security updates waiting more than 48 hours (the UK MoJ's rule) unless the team accepted the risk until a
+date, how fast updates merge, and whether the bot went quiet (Dependabot pauses itself after 90 days without
+interaction)."""
+from datetime import datetime, timedelta
+from statistics import median
+
+from mergescout.signals.bumps import parse_bump
+from stackvitals.core.time import iso
+
+SECURITY_HOURS = 48
+
+
+def _age(stamp: str, now: datetime) -> float:
+    return (now - datetime.fromisoformat(stamp.replace("Z", "+00:00"))).total_seconds() / 3600
+
+
+def hygiene(flow: dict, decisions: dict, now: datetime) -> dict | None:
+    """None when no update bot has opened a pull request here."""
+    def updates(rows):
+        for p in rows or []:
+            b = parse_bump(p.get("ti"), p.get("head"), p.get("au"))
+            if b:
+                yield p, b
+    opened = list(updates(flow.get("open")))
+    closed = list(updates(flow.get("closed")))
+    if not opened and not closed:
+        return None
+    rows = []
+    for p, b in opened:
+        security = b["security"] or any("security" in (label or "").lower() for label in p.get("lb") or [])
+        decided = decisions.get(f"package:{b['package']}") if b.get("package") else None
+        row = {"pr": p["n"], "package": b["package"], "ecosystem": b["ecosystem"], "from": b["from_version"],
+               "to": b["to_version"], "bump": b["bump"], "bot": b["bot"], "security": security,
+               "age_hours": round(_age(p["cr"], now), 1)}
+        if decided and decided.get("verb") == "accept":
+            row["decision"] = {k: decided.get(k) for k in ("verb", "by", "until", "reason")}
+        rows.append(row)
+    rows.sort(key=lambda r: -r["age_hours"])
+    since = iso(now - timedelta(days=90))
+    merged = [_age(p["cr"], now) - _age(p["mg"], now) for p, _b in closed if p.get("mg") and p["mg"] >= since]
+    last = max(p["cr"] for p, _b in opened + closed)
+    return {"open": rows,
+            "security_overdue": [r for r in rows if r["security"] and r["age_hours"] > SECURITY_HOURS and not r.get("decision")],
+            "merged_90d": len(merged), "merge_median_hours": round(median(merged), 1) if merged else None,
+            "bots": sorted({b["bot"] for _p, b in opened + closed}), "last_opened": last,
+            "quiet_days": int(_age(last, now) // 24)}

stackvitals/deps/imports.py

@@ -0,0 +1,113 @@
+"""Which packages a repo's own code imports, for Python, JavaScript/TypeScript and Rust, read in the team's checkout:
+
+    stackvitals deps imports [path] --out imports.json      # in a workflow step before StackVitals (imports-json input)
+
+It answers "does our code import the package an advisory names?", not "does our code call the vulnerable function":
+an import is a cheap, honest signal for which advisories to look at first, and a package that isn't imported directly
+may still run through another package. Tests, vendored code and build output are left out.
+"""
+import json
+import os
+import re
+from pathlib import Path
+
+SKIP = re.compile(r"(^|/)(node_modules|vendor|third_party|dist|build|target|\.git|\.venv|venv|site-packages|__pycache__|"
+                  r"tests?|__tests__|fixtures?)(/|$)")
+# `from x import y`, or everything after `import` on the line (`import a as b, c`).
+PY = re.compile(r"^\s*(?:from\s+([A-Za-z_][\w.]*)\s+import|import\s+([^\n#;]+))", re.M)
+# Rust: `use x::`, `use x;`, `extern crate x` and paths like `tokio::spawn` (2018 edition needs no `use`).
+RS = re.compile(r"(?:\bextern\s+crate\s+([a-z_][a-z0-9_]*)|\buse\s+([a-z_][a-z0-9_]*)\s*(?:;|\bas\b)"
+                r"|(?<![\w:])([a-z_][a-z0-9_]*)::)")
+RS_OWN = {"std", "core", "alloc", "crate", "self", "super", "proc_macro"}
+JS = re.compile(r"""(?:\bfrom\s*|\brequire\s*\(\s*|\bimport\s*\(\s*|^\s*import\s+)['"]([^'"]+)['"]""", re.M)
+JS_FILES = (".js", ".mjs", ".cjs", ".jsx", ".ts", ".tsx", ".mts", ".cts", ".vue", ".svelte", ".astro")
+# Distributions whose import name differs from the package name (the common ones). A dotted name must match itself or
+# what's under it; a package with a hyphen and no entry here can't be told apart, so it reads "can't tell".
+PY_ALIASES = {"pyyaml": ("yaml",), "beautifulsoup4": ("bs4",), "pillow": ("PIL",), "scikit-learn": ("sklearn",),
+              "scikit-image": ("skimage",), "python-dateutil": ("dateutil",), "opencv-python": ("cv2",),
+              "opencv-python-headless": ("cv2",), "protobuf": ("google.protobuf",), "pyjwt": ("jwt",),
+              "python-jose": ("jose",), "msgpack-python": ("msgpack",), "attrs": ("attr", "attrs"),
+              "python-multipart": ("multipart",), "pycryptodome": ("Crypto",), "pycryptodomex": ("Cryptodome",),
+              "google-cloud-storage": ("google.cloud.storage",), "typing-extensions": ("typing_extensions",),
+              "psycopg2-binary": ("psycopg2",), "grpcio": ("grpc",), "grpcio-tools": ("grpc_tools",),
+              "pyopenssl": ("OpenSSL",), "gitpython": ("git",), "mysqlclient": ("MySQLdb",), "pyserial": ("serial",),
+              "pyzmq": ("zmq",), "python-magic": ("magic",), "websocket-client": ("websocket",),
+              "python-slugify": ("slugify",), "pygithub": ("github",), "ruamel-yaml": ("ruamel.yaml",),
+              "python-dotenv": ("dotenv",), "pymupdf": ("fitz",), "faiss-cpu": ("faiss",)}
+MAX_FILES = 20000
+
+
+def _norm(name: str) -> str:
+    return re.sub(r"[-_.]+", "-", name).lower()
+
+
+def scan(root: Path) -> dict:
+    """{"pypi": [modules], "npm": [package names], "crates.io": [crate names]} imported by the repo's own source files."""
+    py, js, rs, seen = set(), set(), set(), 0
+    for dirpath, dirs, files in os.walk(root):
+        rel = Path(dirpath).relative_to(root).as_posix()
+        if rel != "." and SKIP.search(rel):
+            dirs[:] = []
+            continue
+        for f in files:
+            if seen >= MAX_FILES:
+                break
+            path = Path(dirpath) / f
+            if f.endswith(".py"):
+                seen += 1
+                for a, b in PY.findall(path.read_text(encoding="utf-8", errors="replace")):
+                    for part in ([a] if a else b.split(",")):
+                        mod = (part.split() or [""])[0].strip("()")
+                        if re.fullmatch(r"[A-Za-z_][\w.]*", mod):
+                            py.add(mod)
+            elif f.endswith(".rs"):
+                seen += 1
+                for found in RS.findall(path.read_text(encoding="utf-8", errors="replace")):
+                    rs.add(next(x for x in found if x))
+            elif f.endswith(JS_FILES) and not f.endswith(".d.ts"):
+                seen += 1
+                for spec in JS.findall(path.read_text(encoding="utf-8", errors="replace")):
+                    if spec.startswith((".", "/")) or spec.startswith("node:"):
+                        continue
+                    parts = spec.split("/")
+                    js.add("/".join(parts[:2]) if spec.startswith("@") else parts[0])
+    return {"pypi": sorted(py), "npm": sorted(js), "crates.io": sorted(rs - RS_OWN), "files": seen}
+
+
+def imported(doc: dict | None, ecosystem: str | None, package: str) -> bool | None:
+    """True when the code imports the package, False when it doesn't, None when there's nothing to tell (no file read,
+    an ecosystem this doesn't cover, or a Python package whose import name isn't known)."""
+    if not isinstance(doc, dict):
+        return None
+    eco = (ecosystem or "").lower()
+    if eco == "npm":
+        return package in set(doc.get("npm") or [])
+    if eco == "crates.io":
+        return package.replace("-", "_") in set(doc.get("crates.io") or [])
+    if eco == "pypi":
+        mods = set(doc.get("pypi") or [])
+        names = PY_ALIASES.get(_norm(package))
+        if names:
+            return any(n in mods or any(m.startswith(n + ".") for m in mods) for n in names)
+        tops = {_norm(m.split(".")[0]) for m in mods}
+        if _norm(package) in tops:
+            return True
+        # Only a plain name is safe to call not imported; "foo-bar" may be imported under any name.
+        return False if re.fullmatch(r"[A-Za-z0-9_]+", package) else None
+    return None
+
+
+def main(argv: list[str]) -> int:
+    import argparse
+    ap = argparse.ArgumentParser(prog="stackvitals deps imports", description=__doc__.split("\n\n")[0])
+    ap.add_argument("path", nargs="?", default=".")
+    ap.add_argument("--out", help="write the JSON here (default: print it)")
+    args = ap.parse_args(argv)
+    doc = scan(Path(args.path))
+    text = json.dumps(doc, indent=1)
+    if args.out:
+        Path(args.out).write_text(text + "\n", encoding="utf-8")
+        print(f"{len(doc['pypi'])} Python modules, {len(doc['npm'])} npm packages and {len(doc['crates.io'])} crates imported, from {doc['files']} files: {args.out}")
+    else:
+        print(text)
+    return 0