vexpath 0.1.2__tar.gz → 0.1.4__tar.gz

{vexpath-0.1.2 → vexpath-0.1.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: vexpath
-Version: 0.1.2
+Version: 0.1.4
 Summary: Command-line tool: generate SBOMs, match advisories, analyse reachability and emit signed VEX.
 Project-URL: Homepage, https://github.com/Unknown122-coder/vexpath
 Project-URL: Documentation, https://github.com/Unknown122-coder/vexpath/tree/main/docs
@@ -27,7 +27,7 @@ Classifier: Typing :: Typed
 Requires-Python: >=3.12
 Requires-Dist: rich>=13.9
 Requires-Dist: typer>=0.15
-Requires-Dist: vexpath-core==0.1.2
+Requires-Dist: vexpath-core==0.1.4
 Description-Content-Type: text/markdown
 
 # vexpath

{vexpath-0.1.2 → vexpath-0.1.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "vexpath"
-version = "0.1.2"
+version = "0.1.4"
 description = "Command-line tool: generate SBOMs, match advisories, analyse reachability and emit signed VEX."
 readme = "README.md"
 requires-python = ">=3.12"
@@ -45,7 +45,7 @@ classifiers = [
 ]
 
 dependencies = [
-    "vexpath-core==0.1.2",
+    "vexpath-core==0.1.4",
     "typer>=0.15",
     "rich>=13.9",
 ]

{vexpath-0.1.2 → vexpath-0.1.4}/src/vexpath/commands/advisory.py

@@ -8,6 +8,7 @@ from typing import Annotated
 
 import typer
 
+from vexpath import corpus
 from vexpath.output import OutputFormat, emit, emit_json, fail, note
 from vexpath.sources import SNAPSHOT_ENV
 from vexpath_core.advisory.coverage import FixReferenceTier, measure
@@ -19,6 +20,47 @@ from vexpath_core.snapshot import SnapshotIntegrityError, read_snapshot
 app = typer.Typer(help="Download and inspect the advisory corpus.", no_args_is_help=True)
 
 
+@app.command("pull")
+def pull_corpus(
+    directory: Annotated[
+        Path | None,
+        typer.Argument(
+            help="Where to put it. Defaults to the per-user cache, where every command finds it."
+        ),
+    ] = None,
+    url: Annotated[
+        str | None,
+        typer.Option("--url", help="Fetch from somewhere else: a fork, a mirror, your own build."),
+    ] = None,
+) -> None:
+    """Download the published advisory corpus, with its vulnerable symbols.
+
+    The corpus is what makes a finding clearable. OSV says a package is
+    affected; it does not say which functions the fix changed, and without
+    those there is nothing for a reachability search to look for -- so a scan
+    with no corpus reports almost everything as ``symbols_unknown``, which is
+    honest and useless.
+
+    Building one means diffing the fix commit of every advisory, which is
+    hours of cloning upstream repositories and produces the same answer for
+    everybody. So it is built once and published, and this collects it.
+
+    Verified on arrival: the content hash is recomputed over the records and
+    checked against what the snapshot claims. A corpus that fails never
+    replaces one that works.
+    """
+    try:
+        destination, advisories, symbols = corpus.pull(directory, base_url=url)
+    except corpus.CorpusError as exc:
+        fail(str(exc))
+
+    note(
+        f"{advisories} advisories, {symbols} vulnerable symbols -> {destination}\n"
+        f"Every command uses it from now on; override with --snapshot or "
+        f"${SNAPSHOT_ENV}."
+    )
+
+
 @app.command("sync")
 def sync(
     directory: Annotated[

vexpath-0.1.4/src/vexpath/corpus.py

@@ -0,0 +1,180 @@
+"""Fetching the published advisory corpus.
+
+Building the corpus is the expensive half of this tool and nobody should have
+to repeat it. Downloading advisories from OSV takes twenty seconds; extracting
+which functions each fix changed takes hours of cloning and diffing upstream
+repositories, and the result is identical for everyone. So it is built once and
+published, and this is the half that collects it.
+
+Without it a new user gets the honest but useless answer: almost every finding
+comes back ``symbols_unknown`` -- the advisory never said which functions its
+fix changed, so nothing can be searched for -- and the tool that promises to
+tell you what is reachable tells you it could not look.
+
+**Published to a GitHub release rather than served from the platform.** Release
+assets cost nothing to serve and are already covered by the release's
+provenance; the platform is one small machine that sleeps when idle, and
+sending every user ten megabytes from it would be the first thing to fall over.
+
+The download is verified rather than trusted: ``read_snapshot`` recomputes the
+content hash over the normalised records and refuses a corpus whose contents do
+not match what the snapshot claims. A corrupted or substituted corpus changes
+verdicts, so loading one is not an acceptable degraded mode.
+"""
+
+from __future__ import annotations
+
+import os
+import shutil
+import tempfile
+from pathlib import Path
+
+import httpx
+
+from vexpath_core.snapshot import (
+    ADVISORIES_FILE_GZ,
+    SNAPSHOT_METADATA_FILE,
+    SnapshotIntegrityError,
+    read_snapshot,
+)
+
+#: Where a published corpus is fetched from when nothing says otherwise.
+#:
+#: ``releases/latest/download`` is GitHub's redirect to whatever the newest
+#: release is, so this URL does not have to name a version and does not go
+#: stale.
+DEFAULT_CORPUS_URL = "https://github.com/Unknown122-coder/vexpath/releases/latest/download/corpus"
+
+#: Override, for a fork, a mirror, or a corpus built in-house.
+CORPUS_URL_ENVIRONMENT_VARIABLE = "VEXPATH_CORPUS_URL"
+
+#: How long to wait for the corpus. It is around ten megabytes compressed, and
+#: a slow connection is not a failure.
+DOWNLOAD_TIMEOUT = 300.0
+
+#: Refuse anything larger. The published corpus is about ten megabytes; a
+#: response far above that is a redirect to something else, a captive portal,
+#: or a mistake, and streaming it to disk unchecked would fill the disk.
+MAX_CORPUS_BYTES = 200 * 1024 * 1024
+
+
+class CorpusError(Exception):
+    """Raised when a corpus cannot be fetched or cannot be trusted."""
+
+
+def default_corpus_directory() -> Path:
+    """Where a downloaded corpus lives.
+
+    Beside the call-graph cache and following the same platform convention,
+    because both are rebuildable data rather than configuration: losing either
+    costs time, not information.
+    """
+    override = os.environ.get("VEXPATH_CACHE_DIR")
+    if override:
+        return Path(override) / "corpus"
+    if os.name == "nt":
+        base = os.environ.get("LOCALAPPDATA") or str(Path.home() / "AppData" / "Local")
+        return Path(base) / "vexpath" / "corpus"
+    base = os.environ.get("XDG_CACHE_HOME") or str(Path.home() / ".cache")
+    return Path(base) / "vexpath" / "corpus"
+
+
+def corpus_url() -> str:
+    """The base URL to fetch from, honouring the override."""
+    return os.environ.get(CORPUS_URL_ENVIRONMENT_VARIABLE, "").strip() or DEFAULT_CORPUS_URL
+
+
+def pull(
+    destination: Path | None = None,
+    *,
+    base_url: str | None = None,
+    client: httpx.Client | None = None,
+) -> tuple[Path, int, int]:
+    """Download and verify a published corpus.
+
+    Returns the directory, the number of advisories and the number of symbols.
+
+    Downloaded into a temporary directory and moved into place only once it
+    verifies, so an interrupted or corrupted download cannot replace a corpus
+    that was working. The client is injectable so the tests exercise the real
+    code against recorded bytes rather than the network.
+
+    Raises:
+        CorpusError: if the corpus cannot be fetched, is implausibly large, or
+            does not match its own content hash.
+    """
+    target = destination or default_corpus_directory()
+    root = (base_url or corpus_url()).rstrip("/")
+
+    owns_client = client is None
+    http = client or httpx.Client(timeout=DOWNLOAD_TIMEOUT, follow_redirects=True)
+    try:
+        with tempfile.TemporaryDirectory() as directory:
+            staging = Path(directory)
+            _download(http, f"{root}.{SNAPSHOT_METADATA_FILE}", staging / SNAPSHOT_METADATA_FILE)
+            _download(http, f"{root}.{ADVISORIES_FILE_GZ}", staging / ADVISORIES_FILE_GZ)
+
+            # Verifies the content hash, which is the point of doing it here
+            # rather than after the move: a corpus that fails this never
+            # replaces one that works.
+            metadata, records = read_snapshot(staging)
+            symbols = sum(len(record.symbols) for record in records)
+
+            target.parent.mkdir(parents=True, exist_ok=True)
+            if target.exists():
+                shutil.rmtree(target)
+            shutil.move(str(staging), str(target))
+    except httpx.HTTPError as exc:
+        msg = f"the corpus could not be downloaded from {root}: {exc}"
+        raise CorpusError(msg) from exc
+    except SnapshotIntegrityError as exc:
+        # The check that makes downloading a corpus acceptable at all. A
+        # substituted one changes verdicts -- drop an advisory's symbols and
+        # every project using that package is quietly cleared -- so this is
+        # fatal, and the message says to try again rather than how to skip it.
+        msg = f"the corpus from {root} does not match its own content hash and was discarded: {exc}"
+        raise CorpusError(msg) from exc
+    except (OSError, ValueError) as exc:
+        msg = f"the corpus from {root} could not be read: {exc}"
+        raise CorpusError(msg) from exc
+    finally:
+        if owns_client:
+            http.close()
+
+    del metadata
+    return target, len(records), symbols
+
+
+def _download(client: httpx.Client, url: str, destination: Path) -> None:
+    """Stream one file to disk, refusing an implausible one."""
+    with client.stream("GET", url) as response:
+        if response.status_code == 404:
+            msg = (
+                f"{url} does not exist. No corpus has been published for this "
+                f"release yet; 'vexpath advisory sync' builds one locally instead."
+            )
+            raise CorpusError(msg)
+        response.raise_for_status()
+
+        written = 0
+        with destination.open("wb") as handle:
+            for chunk in response.iter_bytes():
+                written += len(chunk)
+                if written > MAX_CORPUS_BYTES:
+                    msg = (
+                        f"{url} is larger than the {MAX_CORPUS_BYTES} byte limit, "
+                        f"which a published corpus is not"
+                    )
+                    raise CorpusError(msg)
+                handle.write(chunk)
+
+
+__all__ = [
+    "CORPUS_URL_ENVIRONMENT_VARIABLE",
+    "DEFAULT_CORPUS_URL",
+    "MAX_CORPUS_BYTES",
+    "CorpusError",
+    "corpus_url",
+    "default_corpus_directory",
+    "pull",
+]

{vexpath-0.1.2 → vexpath-0.1.4}/src/vexpath/sources.py

@@ -11,10 +11,11 @@ from __future__ import annotations
 import os
 from pathlib import Path
 
+from vexpath.corpus import default_corpus_directory
 from vexpath.output import fail, note, warn
 from vexpath_core.advisory import OsvApiSource, SnapshotSource
 from vexpath_core.protocols import AdvisorySource
-from vexpath_core.snapshot import SnapshotIntegrityError
+from vexpath_core.snapshot import SNAPSHOT_METADATA_FILE, SnapshotIntegrityError
 
 #: Environment variable naming a local advisory snapshot. Set it and commands
 #: become reproducible and offline by default.
@@ -24,17 +25,34 @@ OSV_BASE_ENV = "VEXPATH_OSV_API_BASE"
 
 
 def snapshot_directory(explicit: Path | None = None) -> Path | None:
-    """Resolve the snapshot to read, from the flag or the environment."""
+    """Resolve the snapshot to read: the flag, the environment, or the cache.
+
+    The last of those is what makes this tool useful to somebody who has not
+    built a corpus. ``vexpath advisory pull`` downloads the published one, and
+    every command finds it from then on without being told -- which matters
+    because the alternative is a live OSV query, and OSV does not publish
+    which functions a fix changed. Without a corpus nearly every finding comes
+    back ``symbols_unknown``: an honest answer, and not a useful one.
+
+    Order is deliberate. An explicit flag beats the environment, the
+    environment beats the cache, and a cache that is merely present never
+    overrides something the caller asked for.
+    """
     if explicit is not None:
         return explicit
+
     raw = os.environ.get(SNAPSHOT_ENV, "").strip()
-    if not raw:
-        return None
-    candidate = Path(raw)
-    if not candidate.is_dir():
+    if raw:
+        candidate = Path(raw)
+        if candidate.is_dir():
+            return candidate
         warn(f"${SNAPSHOT_ENV} points at {candidate}, which is not a directory; ignoring it.")
         return None
-    return candidate
+
+    downloaded = default_corpus_directory()
+    if (downloaded / SNAPSHOT_METADATA_FILE).is_file():
+        return downloaded
+    return None
 
 
 def advisory_source(snapshot: Path | None, *, result_noun: str = "scan") -> AdvisorySource:

vexpath-0.1.4/tests/test_corpus.py

@@ -0,0 +1,191 @@
+"""Fetching the published corpus.
+
+The corpus is what makes a finding clearable: OSV says a package is affected,
+it does not say which functions the fix changed. Without one, nearly every
+finding comes back ``symbols_unknown`` -- honest, and useless. Building one is
+hours of cloning upstream repositories and produces the same answer for
+everybody, so it is published and collected rather than rebuilt.
+
+These exercise the real download code against recorded bytes. The client is
+injectable for exactly that reason: a test that mocks the function under test
+proves nothing about it.
+"""
+
+from __future__ import annotations
+
+import gzip
+import json
+from pathlib import Path
+
+import httpx
+import pytest
+
+from vexpath import corpus
+from vexpath.sources import snapshot_directory
+from vexpath_core.snapshot import ADVISORIES_FILE, SNAPSHOT_METADATA_FILE
+
+
+def _built_corpus(tmp_path: Path) -> tuple[bytes, bytes]:
+    """A real snapshot, built by the real writer, as published bytes."""
+    from datetime import UTC, datetime
+
+    from vexpath_core.enums import (
+        Confidence,
+        Severity,
+        SymbolKind,
+        SymbolSource,
+        SymbolsStatus,
+    )
+    from vexpath_core.models.advisory import Advisory, AdvisorySnapshot, VulnerableSymbol
+    from vexpath_core.snapshot import write_snapshot
+
+    advisory = Advisory(
+        id="CVE-2026-0001",
+        summary="a problem",
+        severity=Severity.HIGH,
+        symbols_status=SymbolsStatus.EXTRACTED,
+        affected=(),
+        symbols=(
+            VulnerableSymbol(
+                advisory_id="CVE-2026-0001",
+                ecosystem="PyPI",
+                package_name="lib",
+                symbol="lib.danger",
+                kind=SymbolKind.FUNCTION,
+                source=SymbolSource.FIX_COMMIT,
+                confidence=Confidence.HIGH,
+            ),
+        ),
+    )
+    source = tmp_path / "built"
+    # The writer fills in the content hash and the counts, which is what makes
+    # this a real snapshot rather than a file shaped like one.
+    write_snapshot(
+        source,
+        AdvisorySnapshot(
+            id="published-1",
+            created_at=datetime(2026, 11, 13, tzinfo=UTC),
+            content_sha256="0" * 64,
+            core_version="0.0.0",
+            ecosystems=["PyPI"],
+            counts={},
+        ),
+        [advisory],
+    )
+    metadata = (source / SNAPSHOT_METADATA_FILE).read_bytes()
+    advisories = gzip.compress((source / ADVISORIES_FILE).read_bytes())
+    return metadata, advisories
+
+
+def _client(metadata: bytes, advisories: bytes, *, status: int = 200) -> httpx.Client:
+    def handle(request: httpx.Request) -> httpx.Response:
+        body = metadata if request.url.path.endswith("snapshot.json") else advisories
+        return httpx.Response(status, content=body)
+
+    return httpx.Client(transport=httpx.MockTransport(handle))
+
+
+def test_a_published_corpus_is_downloaded_and_verified(tmp_path: Path) -> None:
+    metadata, advisories = _built_corpus(tmp_path)
+    destination = tmp_path / "cache"
+
+    with _client(metadata, advisories) as client:
+        where, count, symbols = corpus.pull(
+            destination, base_url="https://example.test/corpus", client=client
+        )
+
+    assert where == destination
+    assert count == 1
+    assert symbols == 1
+    assert (destination / SNAPSHOT_METADATA_FILE).is_file()
+
+
+def test_a_corpus_that_does_not_match_its_hash_is_refused(tmp_path: Path) -> None:
+    """The check that makes downloading one acceptable at all.
+
+    A substituted corpus changes verdicts: drop the symbols for an advisory
+    and every project using that package is quietly cleared. So the hash is
+    recomputed over the records and compared with what the snapshot claims,
+    and a mismatch is fatal rather than a degraded mode.
+    """
+    metadata, advisories = _built_corpus(tmp_path)
+    tampered = json.loads(metadata)
+    tampered["content_sha256"] = "0" * 64
+
+    with (
+        _client(json.dumps(tampered).encode(), advisories) as client,
+        pytest.raises(corpus.CorpusError),
+    ):
+        corpus.pull(tmp_path / "cache", base_url="https://example.test/corpus", client=client)
+
+
+def test_a_failed_download_does_not_replace_a_working_corpus(tmp_path: Path) -> None:
+    """Downloaded to a temporary directory and moved only once it verifies.
+
+    Somebody running `advisory pull` on a flaky connection must not end up
+    worse off than before they ran it.
+    """
+    metadata, advisories = _built_corpus(tmp_path)
+    destination = tmp_path / "cache"
+
+    with _client(metadata, advisories) as client:
+        corpus.pull(destination, base_url="https://example.test/corpus", client=client)
+    existing = (destination / SNAPSHOT_METADATA_FILE).read_bytes()
+
+    tampered = json.loads(metadata)
+    tampered["content_sha256"] = "0" * 64
+    with (
+        _client(json.dumps(tampered).encode(), advisories) as client,
+        pytest.raises(corpus.CorpusError),
+    ):
+        corpus.pull(destination, base_url="https://example.test/corpus", client=client)
+
+    assert (destination / SNAPSHOT_METADATA_FILE).read_bytes() == existing
+
+
+def test_a_missing_corpus_says_what_to_do_instead(tmp_path: Path) -> None:
+    metadata, advisories = _built_corpus(tmp_path)
+    with (
+        _client(metadata, advisories, status=404) as client,
+        pytest.raises(corpus.CorpusError, match="advisory sync"),
+    ):
+        corpus.pull(tmp_path / "cache", base_url="https://example.test/corpus", client=client)
+
+
+def test_a_downloaded_corpus_is_found_without_being_named(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    """The point of the whole feature: it works with no flag and no variable.
+
+    A new user runs `advisory pull` once and every command afterwards is
+    reproducible and offline, without being told where anything is.
+    """
+    metadata, advisories = _built_corpus(tmp_path)
+    cache = tmp_path / "home"
+    monkeypatch.setenv("VEXPATH_CACHE_DIR", str(cache))
+    monkeypatch.delenv("VEXPATH_ADVISORY_SNAPSHOT_DIR", raising=False)
+
+    assert snapshot_directory() is None
+
+    with _client(metadata, advisories) as client:
+        corpus.pull(base_url="https://example.test/corpus", client=client)
+
+    assert snapshot_directory() == cache / "corpus"
+
+
+def test_what_the_caller_asked_for_beats_the_cache(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    """A cache that is merely present never overrides an explicit choice."""
+    metadata, advisories = _built_corpus(tmp_path)
+    cache = tmp_path / "home"
+    monkeypatch.setenv("VEXPATH_CACHE_DIR", str(cache))
+    monkeypatch.delenv("VEXPATH_ADVISORY_SNAPSHOT_DIR", raising=False)
+    with _client(metadata, advisories) as client:
+        corpus.pull(base_url="https://example.test/corpus", client=client)
+
+    explicit = tmp_path / "elsewhere"
+    assert snapshot_directory(explicit) == explicit
+
+    monkeypatch.setenv("VEXPATH_ADVISORY_SNAPSHOT_DIR", str(tmp_path / "built"))
+    assert snapshot_directory() == tmp_path / "built"