gitmole 0.19.0__tar.gz → 0.20.0__tar.gz

{gitmole-0.19.0 → gitmole-0.20.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gitmole
-Version: 0.19.0
+Version: 0.20.0
 Summary: Offline git repository analysis with a terminal report: hotspots, coupling, ownership, code age, secrets, repo health.
 License: MIT
 Project-URL: Homepage, https://github.com/antvinni/gitmole

{gitmole-0.19.0 → gitmole-0.20.0}/gitmole/__init__.py

@@ -1,3 +1,3 @@
 """gitmole: offline git repository analysis with a terminal report."""
 
-__version__ = "0.19.0"
+__version__ = "0.20.0"

{gitmole-0.19.0 → gitmole-0.20.0}/gitmole/cli.py

@@ -47,6 +47,7 @@ def parse_args(argv):
     p.add_argument("--fail-on", choices=findings.SEVERITIES, help="exit 3 if any finding is at this severity or worse")
     p.add_argument("--risk", metavar="BASE", help="score the files changed since BASE (merge base with HEAD) by their share of the repository's revisions × lines of code; needs a local path")
     p.add_argument("--risk-threshold", type=float, metavar="N", help="with --risk: exit 3 when the changed files hold more than N percent of the repository's revisions × lines of code")
+    p.add_argument("--sbom", metavar="PATH", help="write a CycloneDX 1.6 SBOM of every locked package to PATH, or - for stdout, from the osv-scanner step's package list")
     p.add_argument("--compare", metavar="BEFORE_JSON", help="add a 'Since last report' section against an earlier --json export of the same clone")
     p.add_argument("--hook", action="store_true", help="with --no-run: read an agent hook's JSON on stdin (or files after --), score the files it names like --risk, "
                                                        "print a summary the agent reads back, exit 2 when --risk-threshold is exceeded")
@@ -84,7 +85,7 @@ def main(argv=None, console: Console = None, tool_check=run.missing_tools, plann
     if args.clean:
         return _clean(args, console, ask or (lambda q: console.input(q, markup=False)))
     # When an export goes to stdout, everything else (banner, progress, report) moves to stderr.
-    quiet = "-" in (args.json, args.markdown, args.sarif)
+    quiet = "-" in (args.json, args.markdown, args.sarif, args.sbom)
     ui = Console(stderr=True) if quiet else console
 
     rc, now = _resolve_time(args, err, ui)
@@ -148,6 +149,7 @@ def _check_args(args, err, kind=None) -> int | None:
         bad = None
     else:
         bad = ("--compare needs one repository, not owner/*" if args.compare and kind == "org" else
+               "--sbom needs one repository, not owner/*" if args.sbom and kind == "org" else
                "--risk needs a local path" if args.risk else
                "--list-file-types needs a local path" if args.list_file_types else None)
     if bad:
@@ -598,7 +600,15 @@ def _render(out_dir: str, console: Console, ui: Console, args, err: Console) ->
     if args.sarif:
         from . import sarif
         _write(sarif.dumps(report, found, scope=args.sarif_scope), args.sarif, console)
-    if "-" not in (args.json, args.markdown, args.sarif):
+    if args.sbom:
+        from . import sbom
+        packages = sbom.read_packages(out_dir)
+        if packages is None:
+            err.print("[red]--sbom:[/red] no package list in the output directory; the osv-scanner step writes it, and did not "
+                      f"(dependencies: {(report.get('dependencies') or {}).get('status', 'not-run')}; run.log says why)", soft_wrap=True)
+            return 2
+        _write(sbom.dumps(report, packages), args.sbom, console)
+    if "-" not in (args.json, args.markdown, args.sarif, args.sbom):
         render.report(report, found, console, full=args.full, risk=risk, base=args.risk, compare=comparison)
     if args.fail_on and any(findings.SEVERITIES.index(f["severity"]) <= findings.SEVERITIES.index(args.fail_on) for f in found):
         return 3

{gitmole-0.19.0 → gitmole-0.20.0}/gitmole/deps.py

@@ -21,6 +21,12 @@ import subprocess
 import sys
 import tempfile
 
+try:
+    from . import imports, licences
+except ImportError:  # run as a script: the package directory is sys.path[0]
+    import imports
+    import licences
+
 ARGV = ["osv-scanner", "scan", "source", "-r", "--offline", "--format", "json", "--all-packages", "."]
 NO_SOURCES = 128           # osv-scanner: no lock file found
 NO_DATABASE = "no offline version of the OSV database"   # its message when the local copy is missing
@@ -141,8 +147,8 @@ def _label(score, vulns: list) -> str:
 
 def _relative(path: str, cwd: str) -> str:
     if os.path.isabs(path):
-        try:
-            rel = os.path.relpath(path, cwd)
+        try:   # real paths on both sides: osv-scanner may report /tmp/... for a cwd of /private/tmp/...
+            rel = os.path.relpath(os.path.realpath(path), os.path.realpath(cwd))
         except ValueError:
             return path
         return path if rel.startswith("..") else rel
@@ -174,6 +180,64 @@ def summarise(data: dict, cwd: str) -> dict:
     return {"status": "scanned", "sources": sources, "packages": packages, "vulnerable": vulnerable}
 
 
+PACKAGES = "packages.json"   # every locked package, for --sbom; not part of the report, which keeps the vulnerable ones
+
+
+def packages(data: dict, cwd: str) -> list:
+    """Every package the lock files pin, once per ecosystem, name and version, with the lock files that
+    pin it and the licence a lock file declares for it (package-lock.json, composer.lock), sorted."""
+    declared = {}
+    try:
+        for d in licences.lock_licences(cwd, [_relative(p, cwd) for p in _lock_paths(data)], every=True):
+            declared.setdefault((d["ecosystem"], d["name"], d["version"]), d["expression"])
+    except OSError:
+        pass
+    by_key = {}
+    for r in data.get("results") or []:
+        path = _relative((r.get("source") or {}).get("path") or "", cwd)
+        for p in r.get("packages") or []:
+            info = p.get("package") or {}
+            key = (info.get("ecosystem", ""), info.get("name", ""), info.get("version", ""))
+            row = by_key.setdefault(key, {"ecosystem": key[0], "name": key[1], "version": key[2], "sources": []})
+            if path not in row["sources"]:
+                row["sources"].append(path)
+    out = []
+    for key in sorted(by_key):
+        row = by_key[key]
+        row["sources"].sort()
+        if key in declared:
+            row["license"] = declared[key]
+        out.append(row)
+    return out
+
+
+def _lock_paths(data: dict) -> list:
+    return [(r.get("source") or {}).get("path") or "" for r in data.get("results") or []]
+
+
+# The same scan with the vulnerability matcher switched off: it reads every lock file and matches nothing,
+# so it needs no database. The flag is osv-scanner's experimental plugin switch (2.x); when it fails, there
+# is no package list and --sbom says so.
+LIST_ARGV = ARGV[:-1] + ["--experimental-disable-plugins", "vulnmatch/osvlocal", "."]
+
+
+def list_packages():
+    proc = subprocess.run(LIST_ARGV, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    if proc.returncode not in (0, 1):
+        print(f"deps.py: listing the packages without the database: osv-scanner exited {proc.returncode}", file=sys.stderr)
+        return None
+    try:
+        data = json.loads(proc.stdout.decode("utf-8", "replace") or "{}")
+    except json.JSONDecodeError:
+        return None
+    return data if isinstance(data, dict) else None
+
+
+def write_packages(rows: list, target: str) -> None:
+    with open(target, "w", encoding="utf-8") as fh:
+        json.dump({"packages": rows}, fh, indent=0, sort_keys=True)
+
+
 def write(result: dict, target: str) -> None:
     fd, tmp = tempfile.mkstemp(dir=os.path.dirname(os.path.abspath(target)), prefix=".dependencies-", suffix=".json")
     try:
@@ -198,6 +262,9 @@ def main(argv=None) -> int:
         result = {"status": "no-sources"}
     elif NO_DATABASE in err:
         result = {"status": "no-database", "download": DOWNLOAD}
+        listed = list_packages()   # the package list for --sbom needs no database
+        if listed is not None:
+            write_packages(packages(listed, os.getcwd()), os.path.join(os.path.dirname(os.path.abspath(target)), PACKAGES))
     elif proc.returncode in (0, 1):   # 1: packages with vulnerabilities were found
         text = proc.stdout.decode("utf-8", "replace").strip()
         try:
@@ -208,6 +275,11 @@ def main(argv=None) -> int:
         result = summarise(data, os.getcwd())
         result["database_date"] = database_date()
         result["database_digest"] = database_digest()
+        try:
+            imports.annotate(result["vulnerable"], os.getcwd())
+        except OSError as e:   # the rows stand without it
+            print(f"deps.py: imports: {e}", file=sys.stderr)
+        write_packages(packages(data, os.getcwd()), os.path.join(os.path.dirname(os.path.abspath(target)), PACKAGES))
     else:
         print(f"deps.py: osv-scanner exited {proc.returncode}; no report written", file=sys.stderr)
         return proc.returncode

{gitmole-0.19.0 → gitmole-0.20.0}/gitmole/findings.py

@@ -4,7 +4,7 @@ from __future__ import annotations
 
 import re
 
-from . import classify, coupling, filetypes, hotspots, knowledge, leaks, loss, maat, textfmt, trend
+from . import classify, coupling, filetypes, hotspots, knowledge, leaks, licences, loss, maat, osps, textfmt, trend
 
 SEVERITIES = ["critical", "warning", "info"]
 
@@ -29,6 +29,8 @@ def _f(severity: str, title: str, statement: str, advice: str, rule: dict, evide
     between them a reader of the JSON can check the finding without reading this file."""
     if rule.get("id") in REFS and "ref" not in rule:
         rule = {**rule, "ref": REFS[rule["id"]]}
+    if rule.get("id") in osps.RULE_CONTROLS and "osps" not in rule:   # the OSPS Baseline controls it gives evidence for
+        rule = {**rule, "osps": list(osps.RULE_CONTROLS[rule["id"]])}
     return {"severity": severity, "title": title, "detail": f"{statement.rstrip()} {advice}", "advice": advice,
             "rule": rule, "evidence": evidence}
 
@@ -698,7 +700,8 @@ def _vuln_statement(rows: list, sources: int) -> str:
         if r.get("malicious"):
             score = ", malicious"
         fixed = f", fixed in {r['fixed']}" if r.get("fixed") else ", no fix yet"
-        return f"{r['name']} {r['version']} ({ref}{score}{fixed}) in {r['source']}"
+        loaded = ", imported by no tracked source" if r.get("imported") is False else ""
+        return f"{r['name']} {r['version']} ({ref}{score}{fixed}{loaded}) in {r['source']}"
     listed = "; ".join(one(r) for r in rows[:3])
     more = f" and {len(rows) - 3} more" if len(rows) > 3 else ""
     return f"{_plural(len(rows), 'vulnerable package')} in {_plural(sources, 'lock file')}: {listed}{more}."
@@ -740,7 +743,8 @@ def vulnerable_dependencies(report: dict) -> list:
                       evidence={"lock_files": sources,
                                 "packages": [{"name": r["name"], "version": r["version"], "source": r["source"], "score": r.get("score"),
                                               "fixed": r.get("fixed") or None, "ids": list(r.get("ids") or []),
-                                              "aliases": list(r.get("aliases") or []), "malicious": bool(r.get("malicious"))} for r in group[:10]]}))
+                                              "aliases": list(r.get("aliases") or []), "malicious": bool(r.get("malicious")),
+                                              "imported": r.get("imported", "unknown")} for r in group[:10]]}))
     return out
 
 
@@ -753,7 +757,8 @@ def hygiene_findings(report: dict) -> list:
     stands in for without the GitHub API. Nothing for an output directory from before the step."""
     h = report.get("hygiene") or {}
     out = []
-    for check in (_hygiene_actions, _hygiene_lockfiles, _hygiene_updates, _hygiene_presence, _hygiene_confusion, _hygiene_install, _hygiene_binaries, _hygiene_submodules, _hygiene_symlinks, _hygiene_trojan):
+    for check in (_hygiene_actions, _hygiene_lockfiles, _hygiene_updates, _hygiene_presence, _hygiene_confusion, _hygiene_install, _hygiene_binaries, _hygiene_submodules, _hygiene_symlinks, _hygiene_trojan,
+                  _hygiene_unused, _hygiene_licence, _hygiene_copyleft):
         check(h, out)
     return out
 
@@ -908,6 +913,59 @@ def _hygiene_trojan(h: dict, out: list) -> None:
                       evidence={"bidi": (tj.get("bidi") or [])[:10], "mixed_script": (tj.get("mixed_script") or [])[:10]}))
 
 
+def _hygiene_unused(h: dict, out: list) -> None:
+    im = h.get("imports") or {}
+    rows = im.get("unused") or []
+    if not rows:
+        return
+    by_manifest = {}
+    for r in rows:
+        by_manifest.setdefault(r["manifest"], []).append(r["package"])
+    listed = "; ".join(f"{_files_list(v)} in {k}" for k, v in list(by_manifest.items())[:3])
+    n = im.get("count", len(rows))
+    first = rows[0]
+    out.append(_f("info", "Declared dependencies nothing imports",
+                  f"{n} runtime {'dependency is' if n == 1 else 'dependencies are'} declared and never imported by a tracked file, nor named in a script or configuration: {listed}.",
+                  f"Remove {first['package']} from {first['manifest']} if nothing loads it at run time; an unused dependency is still installed, scanned and updated.",
+                  rule={"id": "unused_dependencies", "reads": "package.json dependencies, go.mod direct requirements, Cargo.toml [dependencies]"},
+                  evidence={"count": n, "manifests": im.get("manifests", 0), "unused": rows[:10]}))
+
+
+def _hygiene_licence(h: dict, out: list) -> None:
+    lic = h.get("licences") or {}
+    declared = lic.get("declared") or []
+    if lic.get("approved") is False or lic.get("mismatch"):
+        named = "; ".join(f"{d['source']} declares {d['expression']}" for d in declared)
+        parts = []
+        if lic.get("mismatch"):
+            parts.append(f"{named}, but {lic['files'][0]} is the {lic['file_licence']} text")
+        if lic.get("approved") is False:
+            parts.append(f"the declared licence ({textfmt.join_and([d['expression'] for d in declared] or [lic.get('file_licence') or ''])}) is not OSI- or FSF-approved")
+        out.append(_f("warning" if lic.get("approved") is False else "info", "Project licence as declared", "; ".join(parts) + ".",
+                      "Make the manifests and the licence file name the same licence; a packager reads the manifest, a lawyer the file." if lic.get("mismatch")
+                      else "Say so plainly in the README if the project is source-available rather than open source.",
+                      rule={"id": "project_licence", "reads": "root manifests and the licence file"},
+                      evidence={"declared": declared, "file": (lic.get("files") or [None])[0], "file_licence": lic.get("file_licence"),
+                                "approved": lic.get("approved"), "mismatch": bool(lic.get("mismatch"))}))
+
+
+def _hygiene_copyleft(h: dict, out: list) -> None:
+    lic = h.get("licences") or {}
+    strong = lic.get("strong") or []
+    if not strong or lic.get("project") != licences.PERMISSIVE:
+        return
+    n = lic.get("strong_count", len(strong))
+    listed = _files_list([f"{d['name']} {d['version']} ({d['expression']})" for d in strong])
+    own = textfmt.join_and(sorted({d["expression"] for d in lic.get("declared") or []} | ({lic["file_licence"]} if lic.get("file_licence") else set())))
+    weak = f" {lic['weak_count']} more declare{'s' if lic.get('weak_count') == 1 else ''} weak copyleft (LGPL, MPL, EPL), which a dependency usually may." if lic.get("weak_count") else ""
+    out.append(_f("warning", "Copyleft dependencies in a permissive project",
+                  f"The project declares {own}, and {n} runtime {'dependency' if n == 1 else 'dependencies'} in {textfmt.join_and(sorted({d['lockfile'] for d in strong}))} "
+                  f"{'declares' if n == 1 else 'declare'} a strong copyleft licence: {listed}.{weak} Declared, as the lock file records it, not read from the package's files.",
+                  f"Check whether {strong[0]['name']} is distributed with the project; if it is, its licence terms reach the whole work.",
+                  rule={"id": "copyleft_dependencies", "reads": "package-lock.json and composer.lock licence fields, runtime packages only"},
+                  evidence={"project": own, "count": n, "weak": lic.get("weak_count", 0), "dependencies": lic.get("dependencies", 0), "strong": strong[:10]}))
+
+
 def _aside_path(path: str) -> bool:
     return filetypes.is_test_path(path) or filetypes.is_sample_path(path) or filetypes.is_vendor_path(path)
 

{gitmole-0.19.0 → gitmole-0.20.0}/gitmole/hygiene.py

@@ -10,7 +10,8 @@ workflow's `uses:`, a lock file's name, `.gitmodules`, a symlink's mode) or on t
 - actions: `uses: owner/repo@ref` in .github/workflows where the ref is not a full commit SHA;
 - lockfiles: a manifest whose last commit is newer than its lock file's, or a manifest with none;
 - updates: the ecosystems whose lock files are tracked that dependabot.yml does not cover;
-- presence: a licence, a security policy, CODEOWNERS and the CODEOWNERS paths that match nothing;
+- presence: a licence, a security policy, a contribution guide, CODEOWNERS and the CODEOWNERS paths
+  that match nothing;
 - confusion: a scoped npm package resolved from a registry other than the one .npmrc declares for
   its scope, several registries in one lock file, a pip `extra-index-url`;
 - install: packages with install scripts in package-lock.json, lifecycle scripts in a tracked
@@ -20,7 +21,9 @@ workflow's `uses:`, a lock file's name, `.gitmodules`, a symlink's mode) or on t
 - submodules: plain http:// or git:// URLs, credentials in a URL, relative URLs, `branch =`;
 - symlinks: links that resolve outside the tree or into .git/;
 - trojan: bidirectional control characters (CVE-2021-42574) and identifiers that mix Latin with
-  Cyrillic, Greek or another confusable script, in source files."""
+  Cyrillic, Greek or another confusable script, in source files;
+- licences: the licences the project and its locked dependencies declare (licences.py);
+- imports: declared dependencies that nothing tracked imports (imports.py)."""
 from __future__ import annotations
 
 import ast
@@ -34,9 +37,11 @@ from collections import defaultdict
 from urllib.parse import urlsplit
 
 try:
-    from . import filetypes
+    from . import filetypes, imports, licences
 except ImportError:  # run as a script: the package directory is sys.path[0]
     import filetypes
+    import imports
+    import licences
 
 CAP = 50   # rows kept per list: the count says how many there were
 
@@ -197,8 +202,10 @@ def presence(repo: str) -> dict:
                 if p.startswith(prefix) and "/" not in p[len(prefix):] and re.match(names, p[len(prefix):], re.I):
                     return p
         return None
-    licence = next((p for p in tracked if "/" not in p and re.match(r"^(licen[cs]e|copying)(\.|-|$)", p, re.I)), None)
+    licence = next((p for p in tracked if "/" not in p and re.match(r"^(licen[cs]e|copying)(\.|-|$)", p, re.I)), None) \
+        or next(("LICENSES/" for p in tracked if p.startswith("LICENSES/")), None)   # the REUSE layout
     policy = first(r"^security(\.md|\.txt|\.rst)?$")
+    contributing = first(r"^contributing(\.md|\.txt|\.rst|\.adoc)?$")
     owners = first(r"^codeowners$")
     missing = []
     if owners:
@@ -209,7 +216,7 @@ def presence(repo: str) -> dict:
             pattern = line.split()[0]
             if not _codeowners_matches(pattern, tracked):
                 missing.append(pattern)
-    return {"license": licence, "security_policy": policy, "codeowners": owners, "codeowners_missing": missing[:CAP]}
+    return {"license": licence, "security_policy": policy, "contributing": contributing, "codeowners": owners, "codeowners_missing": missing[:CAP]}
 
 
 # --- dependency confusion -----------------------------------------------------------------------
@@ -483,7 +490,7 @@ def trojan_source(repo: str) -> dict:
 
 CHECKS = {"actions": actions_pinning, "lockfiles": lockfiles, "updates": dependency_updates, "presence": presence,
           "confusion": dependency_confusion, "install": install_scripts, "binaries": binaries, "submodules": submodules,
-          "symlinks": symlinks, "trojan": trojan_source}
+          "symlinks": symlinks, "trojan": trojan_source, "licences": licences.check, "imports": imports.unused}
 
 
 def main(argv=None) -> int:

gitmole-0.20.0/gitmole/imports.py

@@ -0,0 +1,267 @@
+"""Which packages the tracked source imports: for a vulnerable package, whether anything loads it at
+all, and the declared dependencies nothing loads. Not reachability (osv-scanner's call analysis needs a
+buildable tree and a toolchain); a textual read of import statements, so it says `imported` and never
+`reachable`, and nothing is suppressed on it.
+
+Each ecosystem keys on its own import syntax: `import`/`require` specifiers in JavaScript and
+TypeScript, import paths in Go, `crate::` paths and `extern crate` in Rust, `import`/`from` in Python,
+`require` in Ruby. Where the import name is the package name by the ecosystem's rules (npm, Go
+modules, Rust crates with `-` read as `_`), an absent import is `false`. Where it need not be (a Python
+distribution's modules, a gem's files), a match is `true` and no match is `unknown`: gitmole keeps no
+table of distribution names, so it cannot say PyYAML is imported as yaml. Declared-but-never-imported
+is read for package.json `dependencies`, go.mod direct requirements and Cargo.toml `[dependencies]`
+only, for the same reason."""
+from __future__ import annotations
+
+import json
+import os
+import re
+
+try:
+    from . import filetypes
+except ImportError:  # run inside a script: the package directory is sys.path[0]
+    import filetypes
+
+LIMIT = 1_000_000   # bytes read per file: a larger file is generated or data, not code that imports
+JS = (".js", ".jsx", ".mjs", ".cjs", ".ts", ".tsx", ".mts", ".cts", ".vue", ".svelte", ".astro")
+EXTENSIONS = {"npm": JS, "PyPI": (".py", ".pyi"), "Go": (".go",), "crates.io": (".rs",), "RubyGems": (".rb", ".rake", ".gemspec")}
+EXACT = {"npm", "Go", "crates.io"}   # the import name is the package name: an absent import is false
+
+_JS_SPEC = re.compile(r"""(?:\bfrom\s*|\bimport\s*\(\s*|\bimport\s+|\brequire\s*\(\s*|\brequire\.resolve\s*\(\s*|\bexport\s*\*\s*from\s*)['"]([^'"\s]+)['"]""")
+_PY_IMPORT = re.compile(r"^[ \t]*import[ \t]+([\w., \t]+)", re.M)
+_PY_FROM = re.compile(r"^[ \t]*from[ \t]+(\w[\w.]*)[ \t]+import\b", re.M)
+_GO_BLOCK = re.compile(r"^import\s*\((.*?)^\)", re.M | re.S)
+_GO_ONE = re.compile(r"""^import\s+(?:[\w.]+\s+)?"([^"]+)\"""", re.M)
+_GO_QUOTED = re.compile(r'"([^"]+)"')
+_GO_GENERATE = re.compile(r"^//go:generate\s+(.*)$", re.M)
+_RS_PATH = re.compile(r"\b([A-Za-z_][A-Za-z0-9_]*)::")
+_RS_CRATE = re.compile(r"\b(?:extern\s+crate|use)\s+(?:::)?([A-Za-z_][A-Za-z0-9_]*)")
+_RB_REQUIRE = re.compile(r"""\brequire(?:_relative)?\s*\(?\s*['"]([^'"]+)['"]""")
+
+
+def js_package(spec: str) -> str | None:
+    """The npm package a specifier loads: `@scope/name` or `name`, without the subpath; None for a
+    relative path, an absolute one, a URL or a node: builtin."""
+    if not spec or spec[0] in "./#~" or ":" in spec.split("/", 1)[0]:
+        return None
+    parts = spec.split("/")
+    if spec.startswith("@"):
+        return "/".join(parts[:2]) if len(parts) > 1 else None
+    return parts[0]
+
+
+def python_name(name: str) -> str:
+    """PEP 503's normal form, with `_` as the separator so a module name compares with it."""
+    return re.sub(r"[-_.]+", "_", name).lower()
+
+
+def rust_name(name: str) -> str:
+    return name.replace("-", "_")
+
+
+def _read(repo: str, path: str) -> str:
+    try:
+        with open(os.path.join(repo, path), "rb") as fh:
+            data = fh.read(LIMIT + 1)
+    except OSError:
+        return ""
+    if len(data) > LIMIT or b"\0" in data[:8000]:
+        return ""
+    return data.decode("utf-8", "replace")
+
+
+def scan_text(ecosystem: str, text: str) -> set:
+    """The names one file imports, in the form the ecosystem's packages are compared in."""
+    found = set()
+    if ecosystem == "npm":
+        found.update(p for p in (js_package(s) for s in _JS_SPEC.findall(text)) if p)
+    elif ecosystem == "PyPI":
+        for group in _PY_IMPORT.findall(text):
+            for item in group.split(","):
+                word = item.strip().split(" ")[0].split(".")[0]
+                if word:
+                    found.add(python_name(word))
+        found.update(python_name(m.split(".")[0]) for m in _PY_FROM.findall(text))
+    elif ecosystem == "Go":
+        for block in _GO_BLOCK.findall(text):
+            found.update(_GO_QUOTED.findall(block))
+        found.update(_GO_ONE.findall(text))
+        for line in _GO_GENERATE.findall(text):   # a tool run by go:generate is a use of its module
+            found.update(w for w in line.split() if "." in w.split("/")[0] and "/" in w)
+    elif ecosystem == "crates.io":
+        found.update(_RS_PATH.findall(text))
+        found.update(_RS_CRATE.findall(text))
+    elif ecosystem == "RubyGems":
+        found.update(_RB_REQUIRE.findall(text))
+    return found
+
+
+def scan(repo: str, ecosystems, paths: list = None) -> dict:
+    """{ecosystem: the names imported anywhere in the tracked files of that ecosystem's extensions}.
+    node_modules is left out: somebody else's imports."""
+    paths = filetypes.git_paths(repo, "ls-files") if paths is None else paths
+    wanted = {e: EXTENSIONS[e] for e in ecosystems if e in EXTENSIONS}
+    out = {e: set() for e in wanted}
+    for path in paths:
+        if "node_modules/" in path:
+            continue
+        low = path.lower()
+        for eco, exts in wanted.items():
+            if low.endswith(exts):
+                out[eco] |= scan_text(eco, _read(repo, path))
+    return out
+
+
+def _go_match(module: str, imported: set) -> bool:
+    return any(p == module or p.startswith(module + "/") for p in imported)
+
+
+def is_imported(ecosystem: str, name: str, imported: dict):
+    """True, False or None (unknown) for one package against scan()'s result."""
+    names = imported.get(ecosystem)
+    if names is None:
+        return None
+    if ecosystem == "npm":
+        hit = name in names
+    elif ecosystem == "Go":
+        if name in ("stdlib", "toolchain"):
+            return None
+        hit = _go_match(name, names)
+    elif ecosystem == "crates.io":
+        hit = rust_name(name) in names
+    elif ecosystem == "PyPI":
+        hit = python_name(name) in names
+    elif ecosystem == "RubyGems":
+        hit = any(r == name or r.split("/")[0] == name or r == name.replace("-", "/") for r in names)
+    else:
+        return None
+    if hit:
+        return True
+    return False if ecosystem in EXACT else None
+
+
+def annotate(rows: list, repo: str, paths: list = None) -> None:
+    """Set `imported` to true, false or "unknown" on each vulnerable-package row, in place."""
+    if not rows:
+        return
+    imported = scan(repo, {r.get("ecosystem") for r in rows}, paths)
+    for r in rows:
+        value = is_imported(r.get("ecosystem"), r.get("name", ""), imported)
+        r["imported"] = "unknown" if value is None else value
+
+
+# --- declared, never imported -------------------------------------------------------------------
+
+def _aside(path: str) -> bool:
+    return ("node_modules/" in path or filetypes.is_test_path(path) or filetypes.is_sample_path(path)
+            or filetypes.is_vendor_path(path) or filetypes.is_doc_path(path))
+
+
+def npm_declared(text: str) -> tuple:
+    """(the runtime `dependencies` of a package.json, its scripts' text). Type-only packages
+    (`@types/`) are left out: nothing imports them by name."""
+    try:
+        data = json.loads(text)
+    except ValueError:
+        return [], ""
+    if not isinstance(data, dict):
+        return [], ""
+    deps = data.get("dependencies") or {}
+    scripts = data.get("scripts") or {}
+    names = sorted(n for n in deps if isinstance(n, str) and not n.startswith("@types/")) if isinstance(deps, dict) else []
+    return names, " ".join(str(v) for v in scripts.values()) if isinstance(scripts, dict) else ""
+
+
+def go_declared(text: str) -> list:
+    """go.mod's direct requirements: the `require` lines without `// indirect`."""
+    out, block = [], False
+    for raw in text.split("\n"):
+        line = raw.strip()
+        if line.startswith("require ("):
+            block = True
+            continue
+        if block and line.startswith(")"):
+            block = False
+            continue
+        body = line[len("require "):] if line.startswith("require ") else line if block else None
+        if body is None or not body or body.startswith("//") or "// indirect" in body:
+            continue
+        parts = body.split()
+        if len(parts) >= 2:
+            out.append(parts[0])
+    return sorted(set(out))
+
+
+_TOML_HEADER = re.compile(r"^\s*\[+\s*([^\]]+?)\s*\]+\s*(?:#.*)?$")
+_TOML_KEY = re.compile(r"""^\s*("?)([A-Za-z0-9_.-]+)\1\s*=""")
+
+
+def cargo_declared(text: str) -> list:
+    """Cargo.toml's `[dependencies]`, target-specific ones included, as the names the code uses: the
+    table key, which a `package =` rename makes the import name. Build and dev dependencies are left
+    out, and so are `-sys` crates, which are linked for their native library and not named in code."""
+    out, table = [], ""
+    for line in text.split("\n"):
+        h = _TOML_HEADER.match(line)
+        if h:
+            table = h.group(1).replace('"', "").replace("'", "")
+            if re.match(r"^(target\..+\.)?dependencies\.[A-Za-z0-9_-]+$", table):   # [dependencies.name]
+                out.append(table.rsplit(".", 1)[1])
+            continue
+        if re.match(r"^(target\..+\.)?dependencies$", table):
+            k = _TOML_KEY.match(line)
+            if k:
+                out.append(k.group(2).split(".")[0])   # `name.workspace = true` is a dotted key
+    return sorted({n for n in out if not n.endswith(("-sys", "_sys"))})
+
+
+def _quoted_heads(text: str) -> set:
+    """Every quoted string's package head, for configuration files that name a package without
+    importing it (a Babel preset, an ESLint plugin, a Jest environment)."""
+    out = set()
+    for s in re.findall(r"""['"`]([@\w][\w@./-]*)['"`]""", text):
+        p = js_package(s)
+        if p:
+            out.add(p)
+    return out
+
+
+def _manifest_or_lock(path: str) -> bool:
+    """A manifest or lock file names every dependency in quotes, so it cannot count as a use of one."""
+    base = os.path.basename(path).lower()
+    return base in ("package.json", "composer.json", "deno.json") or "lock" in base or base == "npm-shrinkwrap.json"
+
+
+def unused(repo: str, paths: list = None) -> dict:
+    """Declared runtime dependencies that no tracked file imports: for npm, not imported, not named in
+    a quoted string of any JavaScript, TypeScript, JSON or YAML file other than a manifest or lock file,
+    and not a word in the manifest's scripts; for Go, no import path or go:generate line under the module; for Rust, no `name::` path,
+    `use name` or `extern crate name`. Manifests under tests, examples, documentation and vendored
+    code are left out."""
+    paths = filetypes.git_paths(repo, "ls-files") if paths is None else paths
+    manifests = [p for p in paths if os.path.basename(p) in ("package.json", "go.mod", "Cargo.toml") and not _aside(p)]
+    if not manifests:
+        return {"manifests": 0, "unused": [], "count": 0}
+    kinds = {os.path.basename(p) for p in manifests}
+    wanted = ({"npm"} if "package.json" in kinds else set()) | ({"Go"} if "go.mod" in kinds else set()) | ({"crates.io"} if "Cargo.toml" in kinds else set())
+    imported = scan(repo, wanted, paths)
+    if "npm" in wanted:
+        named = set()
+        for p in paths:
+            if "node_modules/" not in p and p.lower().endswith(JS + (".json", ".jsonc", ".json5", ".yml", ".yaml")) and not _manifest_or_lock(p):
+                named |= _quoted_heads(_read(repo, p))
+        imported["npm"] = imported["npm"] | named
+    out = []
+    for m in sorted(manifests):
+        text = _read(repo, m)
+        base = os.path.basename(m)
+        if base == "package.json":
+            names, scripts = npm_declared(text)
+            words = set(re.findall(r"[\w@./-]+", scripts))
+            out += [{"manifest": m, "ecosystem": "npm", "package": n} for n in names
+                    if n not in imported["npm"] and n not in words and n.rsplit("/", 1)[-1] not in words]
+        elif base == "go.mod":
+            out += [{"manifest": m, "ecosystem": "Go", "package": n} for n in go_declared(text) if not _go_match(n, imported["Go"])]
+        else:
+            out += [{"manifest": m, "ecosystem": "crates.io", "package": n} for n in cargo_declared(text) if rust_name(n) not in imported["crates.io"]]
+    return {"manifests": len(manifests), "unused": out[:50], "count": len(out)}