phantom-scan 0.1.0__py3-none-any.whl

phantom/__init__.py

@@ -0,0 +1,12 @@
+"""phantom — detect divergence between a package's source and its published artifact.
+
+Core entry point for library use::
+
+    from phantom.core import scan
+    from phantom.registry import build_default_registry
+
+    registry = build_default_registry()
+    result = scan("some-pkg", "1.2.3", registry.get("pypi"))
+"""
+
+__version__ = "0.1.0"

phantom/cache.py

@@ -0,0 +1,63 @@
+"""Disk cache keyed by URL digest and the single HTTP helper. A cached
+scan of the same ``pkg==version`` is deterministic and works offline."""
+
+from __future__ import annotations
+
+import hashlib
+import urllib.error
+import urllib.request
+from pathlib import Path
+
+from phantom.errors import FetchError, NotFoundError
+
+USER_AGENT = "phantom/0.1 (+https://github.com/pipebreach/phantom)"
+
+
+def default_cache_dir() -> Path:
+    return Path.home() / ".cache" / "phantom"
+
+
+class DiskCache:
+    """Blob store keyed by the SHA-256 of an arbitrary string key."""
+
+    def __init__(self, root: Path) -> None:
+        self.root = root
+
+    def _path(self, key: str) -> Path:
+        digest = hashlib.sha256(key.encode("utf-8")).hexdigest()
+        return self.root / digest[:2] / digest
+
+    def get(self, key: str) -> bytes | None:
+        path = self._path(key)
+        if path.is_file():
+            return path.read_bytes()
+        return None
+
+    def put(self, key: str, data: bytes) -> None:
+        path = self._path(key)
+        path.parent.mkdir(parents=True, exist_ok=True)
+        tmp = path.with_suffix(".tmp")
+        tmp.write_bytes(data)
+        tmp.replace(path)
+
+
+def http_get(url: str, cache: DiskCache | None = None, timeout: float = 60.0) -> bytes:
+    """GET a URL through the cache. Raises ``NotFoundError`` on 404,
+    ``FetchError`` on any other failure."""
+    if cache is not None:
+        cached = cache.get(url)
+        if cached is not None:
+            return cached
+    request = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
+    try:
+        with urllib.request.urlopen(request, timeout=timeout) as response:
+            data = response.read()
+    except urllib.error.HTTPError as exc:
+        if exc.code == 404:
+            raise NotFoundError(f"404 Not Found: {url}") from exc
+        raise FetchError(f"HTTP {exc.code} fetching {url}") from exc
+    except urllib.error.URLError as exc:
+        raise FetchError(f"network error fetching {url}: {exc.reason}") from exc
+    if cache is not None:
+        cache.put(url, data)
+    return data

phantom/cli.py

@@ -0,0 +1,133 @@
+"""Thin CLI layer over ``phantom.core.scan``. Exit codes: 0 clean, 1 findings
+of high/critical severity, 2 execution error, 3 out of scope."""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+
+from phantom import __version__, core
+from phantom.cache import DiskCache, default_cache_dir
+from phantom.errors import OutOfScopeError, PhantomError
+from phantom.registry import Registry, build_default_registry
+from phantom.report import json_report, sarif_report, table_report
+
+EXIT_OK = 0
+EXIT_FINDINGS = 1
+EXIT_ERROR = 2
+EXIT_OUT_OF_SCOPE = 3
+
+_EXIT_CODES_HELP = """\
+exit codes:
+  0  no findings (or only medium/low severity)
+  1  at least one high/critical finding
+  2  execution error
+  3  package out of scope (no pure-Python wheel available)
+"""
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="phantom",
+        description=(
+            "Detect divergence between a package's declared source and its "
+            "published artifact."
+        ),
+        epilog=_EXIT_CODES_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    parser.add_argument("--version", action="version", version=f"phantom {__version__}")
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    scan = subparsers.add_parser(
+        "scan",
+        help="scan a single package version (e.g. phantom scan requests==2.31.0)",
+        epilog=_EXIT_CODES_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    scan.add_argument("spec", help="package spec in the form <pkg>==<version>")
+    scan.add_argument(
+        "--ecosystem",
+        default="pypi",
+        help="package ecosystem (default: pypi)",
+    )
+    output = scan.add_mutually_exclusive_group()
+    output.add_argument(
+        "--json", action="store_true", help="emit the versioned JSON schema"
+    )
+    output.add_argument(
+        "--sarif", action="store_true", help="emit SARIF 2.1.0 for code scanning"
+    )
+    scan.add_argument(
+        "--cache-dir",
+        type=Path,
+        default=None,
+        help=f"download cache directory (default: {default_cache_dir()})",
+    )
+
+    audit = subparsers.add_parser(
+        "audit",
+        help="scan every package in a lockfile (requirements.txt / poetry.lock) [M2]",
+    )
+    audit.add_argument("lockfile", type=Path, help="path to the lockfile")
+
+    return parser
+
+
+def run(argv: list[str] | None = None, registry: Registry | None = None) -> int:
+    """Parse arguments and execute; ``registry`` is injectable for tests."""
+    args = build_parser().parse_args(argv)
+
+    if args.command == "audit":
+        print(
+            "phantom audit is not implemented yet (planned for M2).",
+            file=sys.stderr,
+        )
+        return EXIT_ERROR
+
+    try:
+        package, version = _parse_spec(args.spec)
+    except ValueError as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        return EXIT_ERROR
+
+    if registry is None:
+        cache = DiskCache(args.cache_dir or default_cache_dir())
+        registry = build_default_registry(cache)
+
+    try:
+        ecosystem = registry.get(args.ecosystem)
+        result = core.scan(package, version, ecosystem)
+    except OutOfScopeError as exc:
+        print(f"out of scope: {exc}", file=sys.stderr)
+        return EXIT_OUT_OF_SCOPE
+    except PhantomError as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        return EXIT_ERROR
+
+    if args.json:
+        print(json_report.render(result))
+    elif args.sarif:
+        print(sarif_report.render(result))
+    else:
+        print(table_report.render(result))
+
+    return core.exit_code_for(result)
+
+
+def _parse_spec(spec: str) -> tuple[str, str]:
+    package, sep, version = spec.partition("==")
+    if not sep or not package or not version:
+        raise ValueError(
+            f"invalid spec {spec!r}: expected exact pin <pkg>==<version>"
+        )
+    return package.strip(), version.strip()
+
+
+def main() -> None:
+    sys.exit(run())
+
+
+if __name__ == "__main__":
+    main()

phantom/core.py

@@ -0,0 +1,81 @@
+"""Orchestrator and library entry point: drives the ``Ecosystem`` interfaces
+only, holds no state."""
+
+from __future__ import annotations
+
+from phantom import differ
+from phantom.ecosystems.base import Ecosystem
+from phantom.models import (
+    Confidence,
+    Finding,
+    FindingType,
+    NoSource,
+    ScanResult,
+    Severity,
+    SourceStatus,
+)
+
+
+def scan(package: str, version: str, ecosystem: Ecosystem) -> ScanResult:
+    """Scan one package version for source/artifact divergence.
+
+    Propagates fetcher errors; unresolvable source is a finding, not an error.
+    """
+    artifact = ecosystem.fetcher.fetch_artifact(package, version)
+    source = ecosystem.source_resolver.resolve_source(
+        package, version, artifact.metadata
+    )
+
+    if isinstance(source, NoSource):
+        return _no_source_result(package, version, ecosystem.name, source)
+
+    outcome = differ.diff(artifact, source, ecosystem.normalizers)
+    return ScanResult(
+        package=package,
+        version=version,
+        ecosystem=ecosystem.name,
+        source_status=SourceStatus.RESOLVED,
+        source_repo=source.repo_url,
+        source_ref=source.ref,
+        findings=outcome.findings,
+        files_scanned=outcome.files_scanned,
+    )
+
+
+def exit_code_for(result: ScanResult) -> int:
+    """Return 1 if any finding is high or critical, else 0."""
+    if any(f.severity in (Severity.HIGH, Severity.CRITICAL) for f in result.findings):
+        return 1
+    return 0
+
+
+def _no_source_result(
+    package: str, version: str, ecosystem: str, source: NoSource
+) -> ScanResult:
+    if source.finding_type == FindingType.NO_SOURCE_DECLARED:
+        # Unverifiable by construction: a risk in itself.
+        status = SourceStatus.NO_SOURCE_DECLARED
+        severity = Severity.HIGH
+        confidence = Confidence.HIGH
+    else:
+        # A repo exists but no tag convention matched; flag softly.
+        status = SourceStatus.REF_NOT_FOUND
+        severity = Severity.MEDIUM
+        confidence = Confidence.MEDIUM
+    finding = Finding(
+        type=source.finding_type,
+        path=None,
+        severity=severity,
+        confidence=confidence,
+        reason=source.detail,
+    )
+    return ScanResult(
+        package=package,
+        version=version,
+        ecosystem=ecosystem,
+        source_status=status,
+        source_repo=source.repo_url,
+        source_ref=None,
+        findings=[finding],
+        files_scanned=0,
+    )

phantom/differ.py

@@ -0,0 +1,129 @@
+"""Compares artifact files against a source hash index.
+
+Content-based and path-agnostic: a wheel file matches if its normalized hash
+exists anywhere in the source tree, so build-time moves and renames do not
+cause false positives.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from phantom import risk
+from phantom.models import (
+    Artifact,
+    Confidence,
+    FileEntry,
+    Finding,
+    FindingType,
+    Severity,
+    SourceTree,
+)
+from phantom.normalizers.base import Normalizer
+
+# Commonly generated at build time (setuptools-scm, hatch-vcs); a phantom
+# hit on these gets low confidence.
+_LIKELY_GENERATED_BASENAMES = {"_version.py", "version.py", "_version_meta.py"}
+
+
+@dataclass
+class DiffOutcome:
+    findings: list[Finding]
+    files_scanned: int
+
+
+def diff(
+    artifact: Artifact, source: SourceTree, normalizers: list[Normalizer]
+) -> DiffOutcome:
+    source_hashes = _build_index(source.files, normalizers)
+    findings: list[Finding] = []
+    files_scanned = 0
+
+    for file in artifact.files:
+        if _is_packaging_metadata(file.path):
+            continue
+        if file.path.endswith(".pth"):
+            files_scanned += 1
+            findings.append(_pth_finding(file))
+            continue
+        normalizer = _normalizer_for(file, normalizers)
+        if normalizer is None:
+            continue
+        files_scanned += 1
+        if normalizer.normalized_hash(file) in source_hashes:
+            continue
+        findings.append(_phantom_finding(file, source))
+
+    findings.sort(key=lambda f: (-f.severity.rank, f.path or ""))
+    return DiffOutcome(findings=findings, files_scanned=files_scanned)
+
+
+def _build_index(files: list[FileEntry], normalizers: list[Normalizer]) -> set[str]:
+    index: set[str] = set()
+    for file in files:
+        normalizer = _normalizer_for(file, normalizers)
+        if normalizer is not None:
+            index.add(normalizer.normalized_hash(file))
+    return index
+
+
+def _normalizer_for(
+    file: FileEntry, normalizers: list[Normalizer]
+) -> Normalizer | None:
+    for normalizer in normalizers:
+        if normalizer.applies_to(file):
+            return normalizer
+    return None
+
+
+def _is_packaging_metadata(path: str) -> bool:
+    first = path.split("/", 1)[0]
+    return first.endswith(".dist-info") or first.endswith(".data")
+
+
+def _pth_finding(file: FileEntry) -> Finding:
+    # .pth import lines execute at every interpreter startup.
+    has_import = any(
+        line.lstrip().startswith("import ")
+        for line in file.data.decode("utf-8", errors="replace").splitlines()
+    )
+    return Finding(
+        type=FindingType.SUSPICIOUS_PTH,
+        path=file.path,
+        severity=Severity.CRITICAL if has_import else Severity.HIGH,
+        confidence=Confidence.HIGH,
+        reason=(
+            ".pth file shipped inside a wheel; "
+            + (
+                "it contains import lines that execute code at every "
+                "interpreter startup"
+                if has_import
+                else "legitimate wheels almost never include .pth files"
+            )
+        ),
+        execution_vectors=["startup:.pth-import"] if has_import else [],
+    )
+
+
+def _phantom_finding(file: FileEntry, source: SourceTree) -> Finding:
+    assessment = risk.assess(file)
+    basename = file.path.rsplit("/", 1)[-1]
+    confidence = Confidence.HIGH
+    reason = (
+        f"content of {file.path} has no normalized-hash match anywhere in "
+        f"{source.repo_url}@{source.ref}; {assessment.detail}"
+    )
+    if basename in _LIKELY_GENERATED_BASENAMES:
+        confidence = Confidence.LOW
+        reason += (
+            "; note: this filename is commonly generated at build time "
+            "(e.g. setuptools-scm), so this may be legitimate generated code"
+        )
+    return Finding(
+        type=FindingType.PHANTOM_FILE,
+        path=file.path,
+        severity=assessment.severity,
+        confidence=confidence,
+        reason=reason,
+        execution_vectors=assessment.vectors,
+    )

phantom/ecosystems/__init__.py

@@ -0,0 +1,3 @@
+from phantom.ecosystems.base import Ecosystem, Fetcher, SourceResolver
+
+__all__ = ["Ecosystem", "Fetcher", "SourceResolver"]

phantom/ecosystems/base.py

@@ -0,0 +1,54 @@
+"""Ecosystem plugin contracts. ``phantom.core.scan()`` only speaks these
+interfaces; adding an ecosystem never touches the core or the differ."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+
+from phantom.models import Artifact, NoSource, SourceTree
+from phantom.normalizers.base import Normalizer
+
+
+class Fetcher(ABC):
+    """Obtains the published artifact for a package version."""
+
+    @abstractmethod
+    def fetch_artifact(self, pkg: str, version: str) -> Artifact:
+        """Download and unpack the artifact.
+
+        Raises ``OutOfScopeError`` if the release cannot be analyzed,
+        ``NotFoundError`` if it does not exist, ``FetchError`` on network
+        failure.
+        """
+
+
+class SourceResolver(ABC):
+    """Locates the source tree that should match the published version."""
+
+    @abstractmethod
+    def resolve_source(
+        self, pkg: str, version: str, metadata: dict
+    ) -> SourceTree | NoSource:
+        """Resolve repo + ref from registry metadata and fetch the tree.
+
+        Returns ``NoSource`` instead of raising: unresolvable source is a
+        finding, not an error.
+        """
+
+
+class Ecosystem(ABC):
+    """Bundle of everything needed to scan one packaging ecosystem."""
+
+    name: str
+
+    @property
+    @abstractmethod
+    def fetcher(self) -> Fetcher: ...
+
+    @property
+    @abstractmethod
+    def source_resolver(self) -> SourceResolver: ...
+
+    @property
+    @abstractmethod
+    def normalizers(self) -> list[Normalizer]: ...

phantom/ecosystems/npm.py

@@ -0,0 +1,9 @@
+"""npm ecosystem — stub, planned for M2 (FR-10).
+
+Will implement a ``Fetcher`` for registry tarballs (``.tgz``), a
+``SourceResolver`` for the ``repository`` field of ``package.json``, and
+normalizers for JS minification/transpilation (M3). Registering it in
+``phantom.registry`` is all the core needs.
+"""
+
+from __future__ import annotations

phantom/ecosystems/pypi.py

@@ -0,0 +1,203 @@
+"""PyPI ecosystem: wheels from the JSON API, sources from GitHub tag
+tarballs. Only pure-Python wheels (``*-none-any.whl``) are in scope."""
+
+from __future__ import annotations
+
+import io
+import json
+import re
+import tarfile
+import zipfile
+
+from phantom.cache import DiskCache, http_get
+from phantom.ecosystems.base import Ecosystem, Fetcher, SourceResolver
+from phantom.errors import FetchError, NotFoundError, OutOfScopeError
+from phantom.models import Artifact, FileEntry, FindingType, NoSource, SourceTree
+from phantom.normalizers.base import Normalizer
+from phantom.normalizers.python_ast import PythonASTNormalizer
+
+PYPI_JSON_URL = "https://pypi.org/pypi/{pkg}/{version}/json"
+# codeload serves public tag tarballs without the API rate limit.
+GITHUB_TARBALL_URL = "https://codeload.github.com/{owner}/{repo}/tar.gz/refs/tags/{tag}"
+
+# Tag conventions tried in order.
+TAG_PATTERNS = ("v{version}", "{version}", "release-{version}")
+
+# project_urls keys checked in priority order.
+_SOURCE_URL_KEYS = ("source", "source code", "repository", "code", "github", "homepage")
+
+_GITHUB_RE = re.compile(
+    r"https?://(?:www\.)?github\.com/(?P<owner>[\w.-]+)/(?P<repo>[\w.-]+)"
+)
+
+
+def _parse_github_repo(url: str) -> tuple[str, str] | None:
+    match = _GITHUB_RE.match(url.strip())
+    if not match:
+        return None
+    owner = match.group("owner")
+    repo = match.group("repo").removesuffix(".git")
+    return owner, repo
+
+
+class PyPIFetcher(Fetcher):
+    def __init__(self, cache: DiskCache | None = None) -> None:
+        self.cache = cache
+
+    def fetch_artifact(self, pkg: str, version: str) -> Artifact:
+        url = PYPI_JSON_URL.format(pkg=pkg, version=version)
+        try:
+            metadata = json.loads(http_get(url, self.cache))
+        except NotFoundError as exc:
+            raise NotFoundError(f"{pkg}=={version} not found on PyPI") from exc
+
+        releases = metadata.get("urls", [])
+        wheel = self._pick_pure_wheel(releases)
+        if wheel is None:
+            raise OutOfScopeError(self._out_of_scope_reason(pkg, version, releases))
+
+        data = http_get(wheel["url"], self.cache)
+        files = self._unpack_wheel(data)
+        return Artifact(
+            package=pkg,
+            version=version,
+            filename=wheel["filename"],
+            files=files,
+            metadata=metadata.get("info", {}),
+        )
+
+    @staticmethod
+    def _pick_pure_wheel(releases: list[dict]) -> dict | None:
+        for release in releases:
+            if release.get("packagetype") == "bdist_wheel" and release.get(
+                "filename", ""
+            ).endswith("-none-any.whl"):
+                return release
+        return None
+
+    @staticmethod
+    def _out_of_scope_reason(pkg: str, version: str, releases: list[dict]) -> str:
+        kinds = sorted({r.get("packagetype", "?") for r in releases})
+        if "bdist_wheel" in kinds:
+            detail = "only platform-specific (compiled) wheels are published"
+        elif "sdist" in kinds:
+            detail = "only an sdist is published"
+        else:
+            detail = "no distribution files found"
+        return (
+            f"{pkg}=={version}: {detail}. M1 only supports pure-Python wheels "
+            f"(*-none-any.whl)."
+        )
+
+    @staticmethod
+    def _unpack_wheel(data: bytes) -> list[FileEntry]:
+        try:
+            archive = zipfile.ZipFile(io.BytesIO(data))
+        except zipfile.BadZipFile as exc:
+            raise FetchError(f"downloaded wheel is not a valid zip: {exc}") from exc
+        entries = []
+        for info in archive.infolist():
+            if info.is_dir():
+                continue
+            entries.append(FileEntry(path=info.filename, data=archive.read(info)))
+        return entries
+
+
+class PyPISourceResolver(SourceResolver):
+    def __init__(self, cache: DiskCache | None = None) -> None:
+        self.cache = cache
+
+    def resolve_source(
+        self, pkg: str, version: str, metadata: dict
+    ) -> SourceTree | NoSource:
+        repo = self._find_github_repo(metadata)
+        if repo is None:
+            return NoSource(
+                finding_type=FindingType.NO_SOURCE_DECLARED,
+                detail=(
+                    f"{pkg} declares no GitHub source repository in its PyPI "
+                    f"metadata (project_urls/home_page). The published artifact "
+                    f"cannot be verified against any source."
+                ),
+            )
+        owner, name = repo
+        repo_url = f"https://github.com/{owner}/{name}"
+        tried = []
+        for pattern in TAG_PATTERNS:
+            tag = pattern.format(version=version)
+            tried.append(tag)
+            try:
+                data = http_get(
+                    GITHUB_TARBALL_URL.format(owner=owner, repo=name, tag=tag),
+                    self.cache,
+                )
+            except NotFoundError:
+                continue
+            return SourceTree(repo_url=repo_url, ref=tag, files=_untar(data))
+        return NoSource(
+            finding_type=FindingType.SOURCE_REF_NOT_FOUND,
+            detail=(
+                f"no tag matching version {version} found in {repo_url} "
+                f"(tried: {', '.join(tried)})"
+            ),
+            repo_url=repo_url,
+        )
+
+    @staticmethod
+    def _find_github_repo(metadata: dict) -> tuple[str, str] | None:
+        project_urls = {
+            key.lower(): value
+            for key, value in (metadata.get("project_urls") or {}).items()
+            if value
+        }
+        candidates = [
+            project_urls[key] for key in _SOURCE_URL_KEYS if key in project_urls
+        ]
+        candidates += [v for v in project_urls.values() if v not in candidates]
+        if metadata.get("home_page"):
+            candidates.append(metadata["home_page"])
+        for url in candidates:
+            repo = _parse_github_repo(url)
+            if repo is not None:
+                return repo
+        return None
+
+
+def _untar(data: bytes) -> list[FileEntry]:
+    """Extract a tarball in memory, stripping GitHub's top-level directory."""
+    try:
+        archive = tarfile.open(fileobj=io.BytesIO(data), mode="r:gz")
+    except tarfile.TarError as exc:
+        raise FetchError(f"downloaded source tarball is invalid: {exc}") from exc
+    entries = []
+    with archive:
+        for member in archive.getmembers():
+            if not member.isfile():
+                continue
+            path = member.name.split("/", 1)[1] if "/" in member.name else member.name
+            handle = archive.extractfile(member)
+            if handle is None:
+                continue
+            entries.append(FileEntry(path=path, data=handle.read()))
+    return entries
+
+
+class PyPIEcosystem(Ecosystem):
+    name = "pypi"
+
+    def __init__(self, cache: DiskCache | None = None) -> None:
+        self._fetcher = PyPIFetcher(cache)
+        self._source_resolver = PyPISourceResolver(cache)
+        self._normalizers: list[Normalizer] = [PythonASTNormalizer()]
+
+    @property
+    def fetcher(self) -> Fetcher:
+        return self._fetcher
+
+    @property
+    def source_resolver(self) -> SourceResolver:
+        return self._source_resolver
+
+    @property
+    def normalizers(self) -> list[Normalizer]:
+        return self._normalizers