saferun-cli 0.0.2__py3-none-any.whl

saferun/engine/osv.py

@@ -0,0 +1,291 @@
+"""Dependency vulnerability scanning with OSV-Scanner (v2 CLI), run INSIDE the analyzer container (Phase 5B). Never on the host.
+
+Invocation (argv, no shell), in a container with NO network, on the sanitized project copy (read-only /workspace), with the
+offline OSV database directory mounted read-only at /osv-db (see osvdb.py; refreshed only by `saferun osv-db update`):
+    /usr/bin/env XDG_CACHE_HOME=/osv-db /usr/local/bin/osv-scanner scan source --format json --no-resolve
+        --offline --offline-vulnerabilities --lockfile=./<lockfile> ...
+SafeRun never installs or fetches dependencies and never runs install scripts: OSV-Scanner only *reads* files.
+Scans are always offline: nothing about the project leaves the machine. Without a database for the needed ecosystem the tool errors
+and that is reported as an ERROR (the verdict can never be PASS), with the command that fixes it. The `osv_offline` option is kept only
+for compatibility; online per-scan queries are intentionally not supported (see PROJECT.md, Phase 5B).
+Exit codes: 0 = no vulnerabilities, 1 = vulnerabilities found (JSON on stdout), 128 = no packages found,
+anything else = error.
+
+Terminology (verified against osv-scanner 2.7.0): a *lockfile* records resolved, exact versions; a *manifest* /
+requirements file declares wanted versions. Scanned, by exact file name:
+  lockfiles    - Python: poetry.lock, Pipfile.lock, pdm.lock, uv.lock; npm: package-lock.json, yarn.lock, pnpm-lock.yaml
+  requirements - Python: requirements.txt ONLY. It is a manifest, not a lockfile: OSV-Scanner (which labels every
+                 input "lockfile" in its JSON) reads it, but an unpinned entry such as `flask>=1.0` is evaluated as
+                 the exact version 1.0 - results for it are unreliable (a note is added), and transitive
+                 dependencies are not covered (--no-resolve).
+Other manifests (package.json, Pipfile, pyproject.toml) and other names (requirements-dev.txt, ...) are NOT scanned;
+a manifest with no scannable file beside it is reported as a coverage gap (INCOMPLETE/SKIPPED), never as clean.
+"""
+from __future__ import annotations
+
+import json
+import os
+import re
+import time
+from pathlib import Path
+
+from .context import CheckContext
+from .model import CheckResult, CheckStatus, Finding, PackageRef, Severity, clean_text
+from .paths import rel_path
+from ..sandbox import SandboxError
+from ..sandbox.analyzer import OSV_EXE, AnalyzerIntegrityError
+from . import osvdb
+from .tools import ToolUnavailable
+
+DB_MOUNT = "/osv-db"
+
+NAME = "dependency-vulnerabilities"
+PY_LOCKFILES = ("poetry.lock", "Pipfile.lock", "pdm.lock", "uv.lock")
+PY_REQUIREMENTS = ("requirements.txt",)   # a manifest, not a lockfile
+NPM_LOCKFILES = ("package-lock.json", "yarn.lock", "pnpm-lock.yaml")
+SUPPORTED = frozenset(PY_LOCKFILES + PY_REQUIREMENTS + NPM_LOCKFILES)
+VENDORED = frozenset({"node_modules", ".venv", "venv", "site-packages", ".git"})
+MAX_LOCKFILES = 50
+MAX_MANIFEST_BYTES = 1024 * 1024
+_NAME_OK = re.compile(r"^[A-Za-z0-9._@/+~-][A-Za-z0-9._@/ +~-]*$")
+
+
+def _basename(p: str) -> str:
+    return p.rsplit("/", 1)[-1]
+
+
+def _dir(p: str) -> str:
+    return p.rsplit("/", 1)[0] if "/" in p else ""
+
+
+def discover(files: list[str]) -> tuple[list[str], list[str], list[str]]:
+    """Return (files to scan, unsafe-to-pass paths, coverage gaps). "Files to scan" are lockfiles plus requirements.txt."""
+    own = [f for f in files if not any(p in VENDORED for p in f.split("/")[:-1])]
+    locks = sorted(f for f in own if _basename(f) in SUPPORTED)
+    unsafe = [f for f in locks if not _NAME_OK.match(f) or ":" in f]
+    locks = [f for f in locks if f not in unsafe]
+    present = set(own)
+    gaps = []
+    for f in own:
+        d, b = _dir(f), _basename(f)
+        sib = lambda n: (f"{d}/{n}" if d else n)  # noqa: E731
+        if b == "package.json" and not any(sib(n) in present for n in NPM_LOCKFILES):
+            gaps.append(f"{f}: no npm lockfile (package-lock.json/yarn.lock/pnpm-lock.yaml); dependencies not scanned")
+        elif b not in SUPPORTED and re.fullmatch(r"(?i).*requirements.*\.txt", b):
+            gaps.append(f"{f}: requirements-file variants are not scanned (only a file named exactly requirements.txt is)")
+        elif b == "Pipfile" and sib("Pipfile.lock") not in present:
+            gaps.append(f"{f}: no Pipfile.lock; dependencies not scanned")
+        elif b == "pyproject.toml" and not any(sib(n) in present for n in PY_LOCKFILES):
+            gaps.append(f"{f}: no Python lockfile (poetry.lock/uv.lock/pdm.lock/Pipfile.lock); dependencies declared there are not scanned"
+                        + (" (a requirements.txt beside it is scanned separately)" if sib("requirements.txt") in present else ""))
+    return locks, unsafe, sorted(gaps)
+
+
+def build_argv(lockfiles: list[str], exe: str = OSV_EXE) -> list[str]:
+    """Always offline. `env` sets the cache directory to the read-only database mount (HOME inside the container is a tmpfs)."""
+    return ["/usr/bin/env", f"XDG_CACHE_HOME={DB_MOUNT}", exe, "scan", "source", "--format", "json", "--no-resolve",
+            "--offline", "--offline-vulnerabilities", *[f"--lockfile=./{lf}" for lf in lockfiles]]
+
+
+def _severity(vuln: dict, score_by_id: dict[str, str]) -> Severity:
+    for i in [vuln.get("id"), *(vuln.get("aliases") or [])]:
+        try:
+            score = float(score_by_id.get(i, ""))
+        except (TypeError, ValueError):
+            continue
+        return (Severity.CRITICAL if score >= 9 else Severity.HIGH if score >= 7 else
+                Severity.MEDIUM if score >= 4 else Severity.LOW if score > 0 else Severity.UNKNOWN)
+    ds = vuln.get("database_specific")
+    label = str(ds.get("severity", "")).upper() if isinstance(ds, dict) else ""
+    return {"CRITICAL": Severity.CRITICAL, "HIGH": Severity.HIGH, "MODERATE": Severity.MEDIUM,
+            "MEDIUM": Severity.MEDIUM, "LOW": Severity.LOW}.get(label, Severity.UNKNOWN)
+
+
+def _fixed(vuln: dict, name: str) -> tuple[str, ...]:
+    out: list[str] = []
+    for a in vuln.get("affected") or []:
+        if not isinstance(a, dict) or (isinstance(a.get("package"), dict) and a["package"].get("name") not in (None, name)):
+            continue
+        for r in a.get("ranges") or []:
+            for ev in (r.get("events") or []) if isinstance(r, dict) else []:
+                if isinstance(ev, dict) and isinstance(ev.get("fixed"), str):
+                    out.append(clean_text(ev["fixed"], 60))
+    return tuple(dict.fromkeys(out))
+
+
+def parse_output(text: str, root: Path) -> list[Finding]:
+    """Raises ValueError on anything that is not OSV-Scanner JSON."""
+    try:
+        data = json.loads(text)
+    except (json.JSONDecodeError, RecursionError) as e:
+        raise ValueError(f"output is not valid JSON: {e}") from None
+    if not isinstance(data, dict) or not isinstance(data.get("results"), list):
+        raise ValueError("JSON has no 'results' list")
+    findings: list[Finding] = []
+    for res in data["results"]:
+        if not isinstance(res, dict) or not isinstance(res.get("packages", []), list):
+            raise ValueError("malformed entry in 'results'")
+        src = res.get("source")
+        path = rel_path(src.get("path"), root) if isinstance(src, dict) else None
+        for pk in res.get("packages") or []:
+            info = pk.get("package") if isinstance(pk, dict) else None
+            if not isinstance(info, dict) or not isinstance(info.get("name"), str) or not isinstance(pk.get("vulnerabilities", []), list):
+                raise ValueError("malformed package entry")
+            ref = PackageRef(clean_text(info["name"], 200), clean_text(info.get("version", ""), 100),
+                             clean_text(info.get("ecosystem", ""), 50))
+            score_by_id: dict[str, str] = {}
+            for g in pk.get("groups") or []:
+                if isinstance(g, dict) and g.get("max_severity") not in (None, ""):
+                    for i in g.get("ids") or []:
+                        score_by_id[str(i)] = str(g["max_severity"])
+            seen: set[str] = set()
+            for v in pk.get("vulnerabilities") or []:
+                if not isinstance(v, dict) or not isinstance(v.get("id"), str):
+                    raise ValueError("malformed vulnerability entry")
+                aliases = [a for a in (v.get("aliases") or []) if isinstance(a, str)]
+                ids = [v["id"], *aliases]
+                if seen & set(ids):      # same vulnerability under another id: report once
+                    continue
+                seen.update(ids)
+                summary = v.get("summary") or v.get("details") or ""
+                findings.append(Finding(
+                    check=NAME, rule_id=clean_text(v["id"], 100), severity=_severity(v, score_by_id),
+                    message=clean_text(f"{ref.name} {ref.version}: {summary}".strip()), category="dependency",
+                    path=path, package=ref, vulnerability_ids=tuple(clean_text(i, 100) for i in ids),
+                    fixed_versions=_fixed(v, ref.name),
+                ))
+    return findings
+
+
+def npm_unmatched(path: Path) -> int | None:
+    """Number of package-lock.json entries OSV-Scanner counts but cannot match (verified with 2.7.0: an entry with no
+    registry `resolved` URL, or one resolved from git, yields no vulnerability results and exit 0). Local `link`
+    entries are ignored. None if the file cannot be read as a lockfile (OSV-Scanner then reports it)."""
+    try:
+        if path.stat().st_size > MAX_MANIFEST_BYTES * 20:
+            return None
+        data = json.loads(path.read_text("utf-8", "replace"))
+    except (OSError, ValueError, RecursionError):
+        return None
+    if not isinstance(data, dict):
+        return None
+    n = 0
+
+    def bad(e) -> bool:
+        return (isinstance(e, dict) and isinstance(e.get("version"), str) and not e.get("link")
+                and not (isinstance(e.get("resolved"), str) and e["resolved"].startswith(("https://", "http://"))))
+
+    pk = data.get("packages")
+    if isinstance(pk, dict):
+        n += sum(1 for k, e in pk.items() if k and bad(e))
+    else:
+        stack = [data.get("dependencies")]
+        while stack:
+            d = stack.pop()
+            for e in (d.values() if isinstance(d, dict) else []):
+                n += bad(e)
+                if isinstance(e, dict):
+                    stack.append(e.get("dependencies"))
+    return n
+
+
+_PROGRESS = ("Starting filesystem walk", "Scanned ", "End status")
+
+
+def _error_lines(stderr: str) -> str:
+    """The tool's progress lines hide its error; prefer the non-progress lines."""
+    lines = [ln.strip() for ln in stderr.splitlines() if ln.strip() and not ln.startswith(_PROGRESS)]
+    return " | ".join(lines or [stderr.strip()])[-300:]
+
+
+def _unpinned_notes(ctx: CheckContext, locks: list[str]) -> list[str]:
+    notes = []
+    for lf in locks:
+        if _basename(lf) != "requirements.txt":
+            continue
+        try:
+            p = ctx.project / lf
+            if p.stat().st_size > MAX_MANIFEST_BYTES:
+                continue
+            lines = [ln.split("#")[0].strip() for ln in p.read_text("utf-8", "replace").splitlines()]
+        except OSError:
+            continue
+        loose = [ln for ln in lines if ln and not ln.startswith("-") and "==" not in ln]
+        if loose:
+            notes.append(f"{lf}: {len(loose)} requirement(s) not pinned with '==' (a manifest is not a lockfile): they are evaluated at their stated lower bound and results for them are unreliable")
+    return notes
+
+
+def run_check(ctx: CheckContext) -> CheckResult:
+    locks, unsafe, gaps = discover(ctx.files)
+    gaps = gaps + [f"{f}: unsupported file name for scanning; skipped" for f in unsafe]
+    if not locks and not gaps:
+        return CheckResult(NAME, CheckStatus.NOT_APPLICABLE, detail="no dependency files")
+    if not locks:
+        return CheckResult(NAME, CheckStatus.SKIPPED, tool="osv-scanner", notes=gaps,
+                           detail="dependency manifests found but no scannable lockfile or requirements.txt; dependencies not scanned")
+    if len(locks) > MAX_LOCKFILES:
+        gaps.append(f"{len(locks) - MAX_LOCKFILES} lockfile(s) beyond the limit of {MAX_LOCKFILES} not scanned")
+        locks = locks[:MAX_LOCKFILES]
+    try:
+        ctx.analyzer_available()
+    except AnalyzerIntegrityError as e:
+        return CheckResult(NAME, CheckStatus.ERROR, tool="osv-scanner", notes=gaps,
+                           detail=clean_text(f"analyzer image failed verification; OSV-Scanner was not run (no host fallback): {e}"))
+    except SandboxError as e:
+        return CheckResult(NAME, CheckStatus.UNAVAILABLE, tool="osv-scanner", notes=gaps,
+                           detail=clean_text(f"analyzer image unavailable; OSV-Scanner was not run (no host fallback): {e}"))
+    db_dir = Path(ctx.options.get("osv_db_dir") or osvdb.default_dir())
+    needed = sorted({"npm" if _basename(lf) in NPM_LOCKFILES else "PyPI" for lf in locks})
+    db = osvdb.validate(db_dir, needed, max_age_days=ctx.options.get("osv_db_max_age_days") or osvdb.DEFAULT_MAX_AGE_DAYS)
+    if not db.ok:
+        return CheckResult(NAME, CheckStatus.ERROR, tool="osv-scanner", notes=gaps,
+                           detail=clean_text("the offline OSV vulnerability database cannot be trusted, so dependencies were NOT checked: "
+                                             + "; ".join(db.problems) + ". Fix or refresh it with: saferun osv-db update", 700))
+    t0 = time.monotonic()
+    try:
+        run = ctx.analyzer(build_argv(locks), cwd=ctx.project, env=None, timeout_s=ctx.timeout_s, max_output=ctx.max_output,
+                           mounts=((db_dir, DB_MOUNT),))
+    except ToolUnavailable as e:
+        return CheckResult(NAME, CheckStatus.UNAVAILABLE, detail=clean_text(e), tool="osv-scanner", notes=gaps)
+    except SandboxError as e:
+        return CheckResult(NAME, CheckStatus.ERROR, tool="osv-scanner", notes=gaps, duration_s=time.monotonic() - t0,
+                           detail=clean_text(f"isolated analyzer run failed; OSV-Scanner results are not available (no host fallback): {e}"))
+    res = CheckResult(NAME, CheckStatus.ERROR, tool="osv-scanner", duration_s=time.monotonic() - t0, notes=list(gaps))
+    if run.timed_out:
+        res.status, res.detail = CheckStatus.TIMEOUT, f"osv-scanner exceeded {ctx.timeout_s:g}s and was killed"
+        return res
+    if run.output_truncated:
+        res.detail = "osv-scanner output exceeded the size limit and the run was killed"
+        return res
+    if run.exit_code == 128:
+        res.status, res.detail = CheckStatus.INCOMPLETE, "osv-scanner found no packages in the scanned file(s); nothing was scanned"
+        return res
+    if run.exit_code not in (0, 1):
+        res.detail = clean_text(f"osv-scanner exited with code {run.exit_code}: {_error_lines(run.stderr)}")
+        return res
+    try:
+        findings = parse_output(run.stdout, ctx.project)
+    except ValueError as e:
+        res.detail = clean_text(f"osv-scanner output unusable: {e}")
+        return res
+    if run.exit_code == 1 and not findings:
+        res.detail = "osv-scanner reported vulnerabilities (exit 1) but none could be read from its output"
+        return res
+    res.findings = findings[: ctx.max_findings]
+    res.truncated_findings = max(0, len(findings) - ctx.max_findings)
+    for lf in locks:
+        if _basename(lf) == "package-lock.json" and (n := npm_unmatched(ctx.project / lf)):
+            gaps.append(f"{lf}: {n} package(s) without a registry URL (or from git) cannot be matched by osv-scanner")
+            res.notes.append(gaps[-1])
+    res.notes += [f"scanned file(s): {', '.join(locks)}", *_unpinned_notes(ctx, locks), *db.notes]   # database facts last: earlier note positions are unchanged
+    if gaps:
+        res.status = CheckStatus.INCOMPLETE
+        res.detail = f"{len(gaps)} dependency file(s) not scanned: " + "; ".join(gaps)[:400]
+    else:
+        res.status = CheckStatus.FINDINGS if res.findings else CheckStatus.PASSED
+    if db.stale:
+        # valid but old data: findings are kept, but the result can never be a clean PASSED
+        res.detail = clean_text("; ".join(x for x in (res.detail, db.stale) if x), 700)
+        res.status = CheckStatus.INCOMPLETE
+    return res

saferun/engine/osvdb.py

@@ -0,0 +1,367 @@
+"""Offline OSV vulnerability databases (Phase 5B, validated since the Phase 5B review).
+
+OSV-Scanner runs inside the analyzer container with the network disabled, so it needs a local copy of the OSV data. The
+data lives in `<db_dir>/osv-scalibr/<Ecosystem>/all.zip` (the layout OSV-Scanner reads when XDG_CACHE_HOME=<db_dir>) and is
+mounted read-only. `saferun osv-db update` is the only code that downloads it: an explicit, user-run, host-side HTTPS download of
+public vulnerability data. The request contains only the ecosystem name; nothing about any project is sent.
+
+Integrity model (read this before trusting a scan result):
+* At download time: HTTPS (certificate-verified by urllib), the server-provided MD5 (`x-goog-hash`) must be present and match
+  (a missing header is refused unless the operator passes `--allow-unverified`, and the manifest then records "unverified"),
+  the file must be a readable zip whose central directory lists at least MIN_ENTRIES[ecosystem] advisory files. The host reads only
+  the zip's central directory (stdlib `zipfile`, nothing is extracted); the advisories themselves are only ever opened by OSV-Scanner
+  inside the container.
+* A `manifest.json` records, per ecosystem, the SHA-256, size, entry count and update time of what was installed.
+* Before every scan `validate()` re-hashes the files and compares them with the manifest, re-checks size/entry-count floors, refuses
+  symlinks and unreadable files, rejects timestamps from the future, and reports the data's age. Any problem makes the dependency
+  check an ERROR; data older than the age limit makes it INCOMPLETE. Neither can produce a clean result.
+What this does NOT give you: the MD5 comes from the same server over the same connection, so it detects accidental corruption, not a
+malicious or compromised server; the SHA-256 is computed locally after download, so it detects LATER modification of the files but does
+not authenticate the original download; the manifest lives next to the data, so someone who can write the directory can rewrite both
+(same trust level as the user's own files and Docker); the entry-count floor is a sanity check, not proof of completeness; and OSV
+publishes no signature that SafeRun checks.
+"""
+from __future__ import annotations
+
+import base64
+import hashlib
+import json
+import math
+import os
+import stat
+import tempfile
+import time
+import urllib.error
+import urllib.request
+import zipfile
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from pathlib import Path
+
+from .. import hostos
+
+ECOSYSTEMS = ("PyPI", "npm")
+BASE_URL = "https://osv-vulnerabilities.storage.googleapis.com"
+MAX_BYTES = 2 * 1024 ** 3
+MANIFEST_NAME = "manifest.json"
+MANIFEST_SCHEMA = 1
+# Sanity floors (about a fifth of what OSV listed in October 2026: PyPI 26 726, npm 230 914 advisories). They catch empty, stub and badly
+# truncated archives; they are not proof that an archive is complete.
+MIN_ENTRIES = {"PyPI": 5_000, "npm": 40_000}
+DEFAULT_MAX_AGE_DAYS = 30.0
+FUTURE_SKEW_S = 24 * 3600
+_HEX64 = frozenset("0123456789abcdef")
+
+
+def default_dir() -> Path:
+    env = os.environ.get("SAFERUN_OSV_DB")
+    if env:
+        return Path(env)
+    return hostos.user_data_base() / "saferun" / "osv-db"
+
+
+def available_ecosystems(db_dir: Path) -> list[str]:
+    """Ecosystems that have a data file (existence only; use `validate` for trust)."""
+    root = db_dir / "osv-scalibr"
+    try:
+        return sorted(e for e in ECOSYSTEMS if (root / e / "all.zip").is_file())
+    except OSError:
+        return []
+
+
+# ---------------------------------------------------------------------------------------------------- validation
+@dataclass
+class DbReport:
+    problems: list[str] = field(default_factory=list)   # any entry: the data cannot be trusted -> the check is an ERROR
+    stale: str = ""                                      # non-empty: valid but older than the limit -> the check is INCOMPLETE
+    notes: list[str] = field(default_factory=list)       # freshness and integrity facts, always reported
+    ecosystems: dict = field(default_factory=dict)       # manifest entries of the validated ecosystems
+    age_days: float | None = None                        # age of the OLDEST validated ecosystem
+
+    @property
+    def ok(self) -> bool:
+        return not self.problems
+
+
+def _sha256_file(path: Path) -> str:
+    h = hashlib.sha256()
+    with open(path, "rb") as f:
+        while chunk := f.read(1 << 20):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+def count_entries(path: Path) -> int:
+    """Number of advisory (`*.json`) files listed in the zip's central directory. Nothing is extracted. Raises ValueError."""
+    try:
+        with zipfile.ZipFile(path) as z:
+            return sum(1 for n in z.namelist() if n.endswith(".json"))
+    except (zipfile.BadZipFile, OSError, NotImplementedError, RuntimeError, EOFError) as e:
+        raise ValueError(f"not a readable zip archive: {e}") from None
+
+
+def _iso(epoch: float) -> str:
+    return datetime.fromtimestamp(epoch, timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
+
+
+def _load_manifest(db_dir: Path) -> dict:
+    p = db_dir / MANIFEST_NAME
+    try:
+        st = os.lstat(p)
+    except FileNotFoundError:
+        raise ValueError(f"no {MANIFEST_NAME} (the data was not installed by `saferun osv-db update`, or by an older version)") from None
+    except OSError as e:
+        raise ValueError(f"cannot read {MANIFEST_NAME}: {e}") from None
+    if not stat.S_ISREG(st.st_mode) or st.st_size > 1 << 20:
+        raise ValueError(f"{MANIFEST_NAME} is not a regular file of sane size")
+    try:
+        data = json.loads(p.read_text())
+    except (OSError, ValueError, RecursionError) as e:
+        raise ValueError(f"{MANIFEST_NAME} is not valid JSON: {e}") from None
+    if not isinstance(data, dict) or data.get("schema") != MANIFEST_SCHEMA or not isinstance(data.get("ecosystems"), dict):
+        raise ValueError(f"{MANIFEST_NAME} has an unexpected structure or schema version")
+    return data
+
+
+def _entry_problem(eco: str, e) -> str | None:
+    if not isinstance(e, dict):
+        return "manifest entry is not an object"
+    sha, size, entries, ts = e.get("sha256"), e.get("size"), e.get("entries"), e.get("updated_epoch")
+    if not (isinstance(sha, str) and len(sha) == 64 and set(sha) <= _HEX64):
+        return "manifest sha256 is malformed"
+    if not (isinstance(size, int) and not isinstance(size, bool) and size > 0):
+        return "manifest size is malformed"
+    if not (isinstance(entries, int) and not isinstance(entries, bool) and entries >= 0):
+        return "manifest entry count is malformed"
+    if not (isinstance(ts, (int, float)) and not isinstance(ts, bool) and math.isfinite(ts) and ts > 0):
+        return "manifest update time is malformed"
+    if e.get("md5") not in ("verified", "unverified"):
+        return "manifest md5 status is malformed"
+    return None
+
+
+def validate(db_dir: Path, needed=None, *, max_age_days: float = DEFAULT_MAX_AGE_DAYS, now: float | None = None) -> DbReport:
+    """Decide whether the data in `db_dir` may be used for `needed` ecosystems (default: everything in the manifest).
+    Never raises; every doubt becomes a problem."""
+    rep = DbReport()
+    now = time.time() if now is None else now
+    try:
+        st = os.lstat(db_dir)
+    except OSError:
+        rep.problems.append(f"{db_dir} does not exist")
+        return rep
+    if not stat.S_ISDIR(st.st_mode):
+        rep.problems.append(f"{db_dir} is not a directory (symlinks are refused)")
+        return rep
+    if not st.st_mode & 0o005 == 0o005:
+        rep.problems.append(f"{db_dir} is not readable by other users; the analyzer container runs as uid 65534 (chmod o+rx it)")
+    try:
+        manifest = _load_manifest(db_dir)
+    except ValueError as e:
+        rep.problems.append(str(e))
+        return rep
+    names = sorted(needed) if needed is not None else sorted(manifest["ecosystems"])
+    if not names:
+        rep.problems.append("the manifest lists no ecosystem")
+    oldest = None
+    for eco in names:
+        entry = manifest["ecosystems"].get(eco)
+        if entry is None:
+            rep.problems.append(f"no {eco} data in the manifest (run `saferun osv-db update`)")
+            continue
+        bad = _entry_problem(eco, entry)
+        if bad:
+            rep.problems.append(f"{eco}: {bad}")
+            continue
+        path = db_dir / "osv-scalibr" / eco / "all.zip"
+        try:
+            for d in (db_dir / "osv-scalibr", db_dir / "osv-scalibr" / eco):
+                dst = os.lstat(d)
+                if not stat.S_ISDIR(dst.st_mode) or not dst.st_mode & 0o005 == 0o005:
+                    raise ValueError(f"{d} is a symlink/not a directory or not readable by other users")
+            fst = os.lstat(path)
+            if not stat.S_ISREG(fst.st_mode) or not fst.st_mode & 0o004:
+                raise ValueError("data file is a symlink/not a regular file or not readable by other users")
+            if fst.st_size != entry["size"]:
+                raise ValueError(f"size {fst.st_size} differs from the recorded {entry['size']}")
+            if _sha256_file(path) != entry["sha256"]:
+                raise ValueError("SHA-256 differs from the recorded value (the file changed after installation)")
+        except (OSError, ValueError) as e:
+            rep.problems.append(f"{eco}: {e}")
+            continue
+        floor = MIN_ENTRIES.get(eco, 1)
+        if entry["entries"] < floor:
+            rep.problems.append(f"{eco}: only {entry['entries']} advisories recorded (minimum {floor}); the data is incomplete or a stub")
+            continue
+        if entry["updated_epoch"] > now + FUTURE_SKEW_S:
+            rep.problems.append(f"{eco}: update time is in the future ({_iso(entry['updated_epoch'])}); the clock or the manifest is wrong")
+            continue
+        age = max(0.0, (now - entry["updated_epoch"]) / 86400)
+        oldest = age if oldest is None else max(oldest, age)
+        rep.ecosystems[eco] = entry
+        rep.notes.append(f"OSV database {eco}: updated {_iso(entry['updated_epoch'])} ({age:.1f} days ago), {entry['entries']} advisories, "
+                         f"sha256 {entry['sha256'][:16]}…, download MD5 {entry['md5']}")
+    rep.age_days = oldest
+    if rep.ok:
+        rep.notes.append("OSV database integrity: hashes recorded locally at download time and re-checked now; this detects later changes, "
+                         "it does not authenticate the original download")
+        if any(e["md5"] == "unverified" for e in rep.ecosystems.values()):
+            rep.notes.append("OSV database was installed WITHOUT a server-provided hash (--allow-unverified); its download was not integrity-checked")
+        if oldest is not None and oldest > max_age_days:
+            rep.stale = (f"the OSV database is {oldest:.0f} days old (limit {max_age_days:g}); vulnerabilities published since are not covered. "
+                         "Refresh with: saferun osv-db update")
+    return rep
+
+
+def stale_ecosystems(db_dir: Path, *, max_age_days: float = DEFAULT_MAX_AGE_DAYS, now: float | None = None) -> list[str]:
+    """Ecosystems that must be (re)downloaded: missing, failing validation, or older than the limit. An ecosystem that is valid and
+    fresh is not listed, so it is not downloaded again. Never raises."""
+    out = []
+    for eco in ECOSYSTEMS:
+        rep = validate(Path(db_dir), [eco], max_age_days=max_age_days, now=now)
+        if not rep.ok or rep.stale:
+            out.append(eco)
+    return out
+
+
+# ---------------------------------------------------------------------------------------------------- update
+def _ensure_dir(path: Path) -> bool:
+    """Create `path` (mode 0755) if it is missing. Returns True if it was created. An EXISTING directory is never chmod'ed;
+    a symlink is refused."""
+    if path.is_symlink():
+        raise ValueError(f"refusing to use a symlink as a database directory: {path}")
+    if path.exists():
+        if not path.is_dir():
+            raise ValueError(f"not a directory: {path}")
+        return False
+    if path == default_dir():
+        hostos.ensure_private_dir(path.parent)   # <data>/saferun also holds the analyzer lock, which requires it to be owner-only
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.mkdir(mode=0o755)
+    os.chmod(path, 0o755)         # only a directory created just now; the umask may have narrowed it
+    return True
+
+
+def _write_manifest(db_dir: Path, base_url: str, eco: str, entry: dict) -> None:
+    try:
+        manifest = _load_manifest(db_dir)
+    except ValueError:
+        manifest = {"schema": MANIFEST_SCHEMA, "source": base_url, "ecosystems": {}}
+    manifest["source"] = base_url
+    manifest["ecosystems"][eco] = entry
+    fd, tmp = tempfile.mkstemp(dir=db_dir, prefix=".manifest-")
+    try:
+        with os.fdopen(fd, "w") as f:
+            json.dump(manifest, f, indent=2, sort_keys=True)
+        os.chmod(tmp, 0o644)
+        os.replace(tmp, db_dir / MANIFEST_NAME)
+    except BaseException:
+        if os.path.exists(tmp):
+            os.unlink(tmp)
+        raise
+
+
+class OsvNetworkError(OSError):
+    """The OSV server could not be reached (or kept failing). The existing data and manifest are untouched, and are NOT made fresher by this."""
+
+
+OPEN_ATTEMPTS = 3
+RETRY_STATUS = frozenset({408, 429, 500, 502, 503, 504})
+
+
+def _open(opener, url: str, *, sleep=time.sleep, attempts: int = OPEN_ATTEMPTS):
+    """Open `url`, retrying transient failures (no connection, timeout, 408/429/5xx) with a short back-off. Other HTTP errors fail at once."""
+    last = None
+    for i in range(attempts):
+        try:
+            return opener(url, timeout=120)
+        except urllib.error.HTTPError as e:
+            if e.code not in RETRY_STATUS:
+                raise OsvNetworkError(f"the OSV server answered HTTP {e.code} for {url}") from None
+            last = f"HTTP {e.code}"
+        except (urllib.error.URLError, TimeoutError, ConnectionError, OSError) as e:
+            last = str(getattr(e, "reason", e))
+        if i + 1 < attempts:
+            sleep(2 ** i)
+    raise OsvNetworkError(f"cannot reach the OSV server ({last}) after {attempts} attempts; check your internet connection or proxy. "
+                          "Existing data was left untouched and keeps its original age")
+
+
+def update(db_dir: Path, ecosystems=ECOSYSTEMS, *, base_url: str = BASE_URL, opener=None,
+           allow_unverified: bool = False, clock=time.time, progress=None, sleep=time.sleep) -> dict[str, dict]:
+    """Download `all.zip` for each ecosystem into `db_dir` and record it in the manifest. Returns {ecosystem: manifest entry}.
+    Atomic per file; raises on any problem (including an interrupted download) and leaves the previous file and manifest entry in place.
+    Never changes the permissions of a directory that already existed. `progress(ecosystem, bytes_done, total_or_None)` is called about once per MiB."""
+    if not base_url.startswith("https://"):
+        raise ValueError("OSV database downloads must use https")
+    opener = opener or urllib.request.urlopen
+    for eco in ecosystems:
+        if eco not in ECOSYSTEMS:
+            raise ValueError(f"unsupported ecosystem {eco!r}; supported: {', '.join(ECOSYSTEMS)}")
+    db_dir = Path(db_dir)
+    _ensure_dir(db_dir)
+    _ensure_dir(db_dir / "osv-scalibr")
+    result = {}
+    for eco in ecosystems:
+        target_dir = db_dir / "osv-scalibr" / eco
+        _ensure_dir(target_dir)
+        fd, tmp = tempfile.mkstemp(dir=target_dir, prefix=".dl-")
+        try:
+            md5, sha, size = hashlib.md5(usedforsecurity=False), hashlib.sha256(), 0
+            with os.fdopen(fd, "wb") as out, _open(opener, f"{base_url}/{eco}/all.zip", sleep=sleep) as resp:  # noqa: S310 - https enforced
+                expected_md5 = _goog_md5(_header_values(resp.headers, "x-goog-hash"))
+                if not expected_md5 and not allow_unverified:
+                    raise ValueError(f"{eco}: the server sent no usable MD5 header, so the download cannot be integrity-checked; "
+                                     "refusing (re-run with --allow-unverified to accept it and record it as unverified)")
+                try:
+                    total = int(resp.headers.get("Content-Length")) if resp.headers.get("Content-Length") else None
+                except (TypeError, ValueError):
+                    total = None
+                while chunk := resp.read(1 << 20):
+                    size += len(chunk)
+                    if progress:
+                        progress(eco, size, total)
+                    if size > MAX_BYTES:
+                        raise ValueError(f"{eco}: download exceeds {MAX_BYTES} bytes")
+                    md5.update(chunk)
+                    sha.update(chunk)
+                    out.write(chunk)
+            if expected_md5 and md5.hexdigest() != expected_md5:
+                raise ValueError(f"{eco}: MD5 mismatch against the server-provided hash (corrupt download)")
+            if size == 0:
+                raise ValueError(f"{eco}: empty download")
+            entries = count_entries(Path(tmp))
+            if entries < MIN_ENTRIES[eco]:
+                raise ValueError(f"{eco}: the archive lists only {entries} advisories (minimum {MIN_ENTRIES[eco]}); refusing an empty or incomplete database")
+            os.chmod(tmp, 0o644)
+            os.replace(tmp, target_dir / "all.zip")
+        except BaseException:
+            if os.path.exists(tmp):
+                os.unlink(tmp)
+            raise
+        now = clock()
+        entry = {"sha256": sha.hexdigest(), "size": size, "entries": entries, "updated_epoch": int(now), "updated_at": _iso(now),
+                 "md5": "verified" if expected_md5 else "unverified"}
+        _write_manifest(db_dir, base_url, eco, entry)
+        result[eco] = entry
+    return result
+
+
+def _header_values(headers, name: str) -> str:
+    """All values of a header joined with commas. The real server sends `x-goog-hash` as TWO separate headers (crc32c, md5);
+    `headers.get()` returns only the first, which made the original MD5 check a silent no-op."""
+    if hasattr(headers, "get_all"):
+        return ",".join(headers.get_all(name) or [])
+    return str(headers.get(name, "") or "")
+
+
+def _goog_md5(header: str) -> str:
+    """x-goog-hash value(s): crc32c=...,md5=<base64>; returns the md5 as hex, or ''."""
+    for part in (header or "").replace(" ", "").split(","):
+        if part.startswith("md5="):
+            try:
+                raw = base64.b64decode(part[4:], validate=True)
+            except ValueError:
+                return ""
+            return raw.hex() if len(raw) == 16 else ""
+    return ""