python-sbom 0.1.1__py3-none-any.whl

python_sbom/_collect/_facts.py

@@ -0,0 +1,155 @@
+from __future__ import annotations
+
+import json
+import os
+import platform
+import sys
+import time
+import warnings
+from functools import cache
+from importlib.metadata import distributions
+from itertools import takewhile
+from typing import TYPE_CHECKING, Final
+
+from installer.records import InvalidRecordEntry, parse_record_file
+from packaging.direct_url import DirectUrl, DirectUrlValidationError
+from packaging.requirements import InvalidRequirement, Requirement
+from packaging.utils import canonicalize_name
+
+from python_sbom._errors import SbomError
+from python_sbom._model._inputs import BuildFacts, InstalledDistribution, Interpreter, RecordRow
+from python_sbom._model._metadata import parse_metadata
+from python_sbom._wheel._record import record_rows
+
+if TYPE_CHECKING:
+    from collections.abc import Container, Sequence
+    from importlib.metadata import Distribution
+    from pathlib import Path
+
+    from python_sbom._model._metadata import CoreMetadata
+
+    from ._revision import RevisionReader
+
+
+def collect_facts(
+    root: Path, revision: RevisionReader, *, path: Sequence[str] | None = None, default_epoch: int | None = None
+) -> BuildFacts:
+    if from_environment := bool(raw := os.environ.get("SOURCE_DATE_EPOCH", "")):
+        try:
+            epoch = int(raw)
+        except ValueError as error:
+            msg = f"SOURCE_DATE_EPOCH must be a Unix timestamp, got {raw!r}"
+            raise SbomError(msg) from error
+    else:
+        epoch = int(time.time()) if default_epoch is None else default_epoch
+    return BuildFacts(
+        epoch=epoch,
+        epoch_from_environment=from_environment,
+        revision=revision(root),
+        interpreter=Interpreter(
+            name=sys.implementation.name,
+            version=platform.python_version(),
+            implementation=platform.python_implementation(),
+        ),
+        distributions=_installed(path),
+    )
+
+
+def _installed(path: Sequence[str] | None) -> list[InstalledDistribution]:
+    found: Final[dict[str, tuple[Distribution, CoreMetadata]]] = {}
+    for distribution in distributions(path=list(path)) if path is not None else distributions():
+        # where Distribution.metadata looks, a .dist-info's METADATA, then an .egg-info's PKG-INFO or the file itself
+        text = distribution.read_text("METADATA") or distribution.read_text("PKG-INFO") or distribution.read_text("")
+        # the first on the path is the one imports load, so a shadowed copy further down is left out
+        if (metadata := parse_metadata(text or "")).name and metadata.version:
+            found.setdefault(canonicalize_name(metadata.name), (distribution, metadata))
+    return [
+        InstalledDistribution(
+            name=name,
+            metadata=metadata,
+            files=_recorded_files(distribution),
+            depends_on=_depends_on(name, distribution, metadata, found),
+        )
+        for name, (distribution, metadata) in found.items()
+    ]
+
+
+def _recorded_files(distribution: Distribution) -> list[RecordRow]:
+    # a platform wheel's files are built for this machine's OS and architecture, and an editable install's point at a
+    # checkout, so listing either would tie the document to one machine
+    # Root-Is-Purelib as the email parser takes a field, its first occurrence among the header lines, as written after
+    # the colon, without parsing the rest of WHEEL
+    headers = takewhile(bool, (distribution.read_text("WHEEL") or "").splitlines())
+    fields = (line.partition(":") for line in headers)
+    purelib = next(
+        (
+            value.lstrip(" \t") == "true"
+            for field, colon, value in fields
+            if colon and field.lower() == "root-is-purelib"
+        ),
+        False,
+    )
+    if not purelib or _editable(distribution):
+        return []
+    # RECORD read once and as written: Distribution.files would stat every row to drop files missing on disk, and build
+    # a path object for each, while the SBOM lists what the install recorded
+    rows: Final = parse_record_file((distribution.read_text("RECORD") or "").splitlines())
+    try:
+        return record_rows((row for row in rows if _describes_distribution(row[0])), distribution.name)
+    except InvalidRecordEntry as error:
+        warnings.warn(f"listed no files of {distribution.name}, its RECORD is malformed: {error}", stacklevel=2)
+        return []
+
+
+def _editable(distribution: Distribution) -> bool:
+    if (text := distribution.read_text("direct_url.json")) is None:
+        return False
+    try:
+        dir_info = DirectUrl.from_dict(json.loads(text)).dir_info
+    # an installer that wrote the file wrong breaks no build: its files get listed, as for any other install
+    except (ValueError, DirectUrlValidationError) as error:
+        warnings.warn(f"read the direct_url.json of {distribution.name} as not editable: {error}", stacklevel=2)
+        return False
+    return dir_info is not None and dir_info.editable is True
+
+
+def _describes_distribution(path: str) -> bool:
+    # console-script launchers live outside site-packages and embed the environment's interpreter path; the installer
+    # metadata names the frontend; uv's cache files hold paths and times; none of them says anything about the package
+    directory, _, name = path.rpartition("/")
+    return not (
+        path.startswith("../")
+        or (directory.endswith(".dist-info") and name in {"INSTALLER", "REQUESTED", "direct_url.json"})
+        or (name.startswith("uv_") and name.endswith(".json"))
+    )
+
+
+def _depends_on(name: str, distribution: Distribution, metadata: CoreMetadata, installed: Container[str]) -> list[str]:
+    names: Final[set[str]] = set()
+    # an .egg-info states its requirements in requires.txt, which Distribution.requires reads
+    for line in metadata.values("requires_dist") or distribution.requires or []:
+        match _required(line):
+            case InvalidRequirement() as error:
+                # old distributions declare requirements, such as "pytz (>dev)", that today's syntax rejects
+                warnings.warn(f"left out the requirement {line!r} of {name}: {error}", stacklevel=2)
+            case str() as required if required in installed:
+                names.add(required)
+    return sorted(names)
+
+
+@cache
+def _required(line: str) -> str | InvalidRequirement | None:
+    # the normalized name a requirement needs here, None when its marker rules it out, or why it does not parse; the
+    # same requirements recur across a build environment's distributions, and the verdict holds for the process
+    try:
+        requirement = Requirement(line)
+    except InvalidRequirement as error:
+        return error
+    if requirement.marker is not None and not requirement.marker.evaluate():
+        return None
+    return canonicalize_name(requirement.name)
+
+
+__all__ = [
+    "collect_facts",
+]

python_sbom/_collect/_revision.py

@@ -0,0 +1,93 @@
+from __future__ import annotations
+
+import os
+import re
+import shutil
+import subprocess  # ruff: ignore[suspicious-subprocess-import] # runs git to read the source revision
+import warnings
+from typing import TYPE_CHECKING, Final, Protocol
+
+from python_sbom._errors import SbomError
+from python_sbom._model._inputs import Revision
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+#: the file an sdist carries its source revision in, as ``node: <commit>``, for the wheel built from it
+REVISION_FILE: Final[str] = ".python-sbom-revision"
+_COMMIT: Final[re.Pattern[str]] = re.compile(r"[0-9a-f]{40}|[0-9a-f]{64}")
+
+
+class RevisionReader(Protocol):
+    def __call__(self, root: Path, /) -> Revision | None: ...
+
+
+def revision_reader(commit: str | None) -> RevisionReader:
+    # a commit given on the command line stands for the checkout, so git and the files go unread
+    if commit is None:
+        return default_revision
+    revision = _commit_revision(commit, "--commit")
+    return lambda _root: revision
+
+
+def default_revision(root: Path, /) -> Revision | None:
+    # the variable exists for builds from an sdist, so it wins over whatever checkout the sdist was unpacked into
+    if commit := os.environ.get("PYTHON_SBOM_COMMIT"):
+        return _commit_revision(commit, "PYTHON_SBOM_COMMIT")
+    if (found := _git_revision(root)) is not None:
+        return found
+    # the sdist build wrote it from the revision it resolved, so it is at least as exact as an archival file
+    if (recorded := root / REVISION_FILE).is_file():
+        return _commit_revision(recorded.read_text(encoding="utf-8").strip().removeprefix("node: "), REVISION_FILE)
+    archival = root / ".git_archival.txt"
+    if archival.is_file() and (
+        match := re.search(rf"^node: ({_COMMIT.pattern})$", archival.read_text(encoding="utf-8"), re.MULTILINE)
+    ):
+        return Revision(commit=match[1], dirty=False)
+    return None
+
+
+def _git_revision(root: Path, /) -> Revision | None:
+    resolved = root.resolve()
+    if (git := shutil.which("git")) is None or not any(
+        (directory / ".git").exists() for directory in (resolved, *resolved.parents)
+    ):
+        return None
+    try:
+        # a project unpacked into another checkout, such as an sdist under its build directory, is not its source
+        _git(git, root, "ls-files", "--error-unmatch", "--", "pyproject.toml")
+        commit = _git(git, root, "rev-parse", "--verify", "--quiet", "HEAD")
+        # untracked files count, since the build packages every file .gitignore lets through
+        changes = _git(git, root, "status", "--porcelain", "--", ".")
+    except subprocess.CalledProcessError as error:
+        if error.returncode != 1:  # 1: the project is untracked, or the checkout has no commit yet
+            warnings.warn(f"git cannot read the source revision in {root}: {error.stderr.strip()}", stacklevel=2)
+        return None
+    return Revision(commit=commit, dirty=bool(changes))
+
+
+def _git(git: str, root: Path, *arguments: str) -> str:
+    # git first runs in whichever repository encloses the project, whose fsmonitor hook would run a command from its
+    # config; no optional locks, so status never rewrites the index under another git process
+    return subprocess.run(  # ruff: ignore[subprocess-without-shell-equals-true] # git from PATH with fixed arguments
+        [git, "-c", "core.fsmonitor=false", "--no-optional-locks", "-C", str(root), *arguments],
+        check=True,
+        capture_output=True,
+        text=True,
+    ).stdout.strip()
+
+
+def _commit_revision(commit: str, origin: str) -> Revision:
+    # the commit lands in the source link, so anything but a full hash could point the link elsewhere
+    if not _COMMIT.fullmatch(commit):
+        msg = f"{origin} must be a full lowercase hex commit hash of 40 or 64 characters, got {commit!r}"
+        raise SbomError(msg)
+    return Revision(commit=commit, dirty=False)
+
+
+__all__ = [
+    "REVISION_FILE",
+    "RevisionReader",
+    "default_revision",
+    "revision_reader",
+]

python_sbom/_collect/_settings.py

@@ -0,0 +1,82 @@
+from __future__ import annotations
+
+import tomllib
+from typing import TYPE_CHECKING, TypeAlias
+
+from python_sbom._errors import SbomError
+from python_sbom._model._inputs import Settings
+
+if TYPE_CHECKING:
+    from collections.abc import Mapping
+    from datetime import date, datetime, time
+    from pathlib import Path
+    from typing import TypeIs  # new in 3.13; annotations here are never evaluated
+
+    from python_sbom._model._types import OrganizationalEntity
+
+#: a value of a TOML table, as :mod:`tomllib` or hatchling hand it over
+Toml: TypeAlias = "str | int | float | bool | datetime | date | time | list[Toml] | dict[str, Toml]"
+
+
+def settings_from_pyproject(root: Path) -> Settings:
+    pyproject = root / "pyproject.toml"
+    try:
+        loaded = tomllib.loads(pyproject.read_text(encoding="utf-8"))
+    except FileNotFoundError as error:
+        msg = f"{root} has no pyproject.toml to read [tool.python-sbom] from"
+        raise SbomError(msg) from error
+    except tomllib.TOMLDecodeError as error:
+        msg = f"{pyproject} is not valid TOML: {error}"
+        raise SbomError(msg) from error
+    return settings_from_table(loaded.get("tool", {}).get("python-sbom", {}), f"[tool.python-sbom] of {pyproject}")
+
+
+def settings_from_table(table: Mapping[str, Toml], where: str) -> Settings:
+    known = {"bundled-wheels", "content-provider", "default-branch", "repository", "supplier"}
+    if unknown := sorted(table.keys() - known):
+        msg = f"{where} has unknown option(s) {', '.join(unknown)}; known: {', '.join(sorted(known))}"
+        raise SbomError(msg)
+    if not isinstance(repository := table.get("repository"), str | None):
+        msg = f"`repository` in {where} must be a string"
+        raise SbomError(msg)
+    if not isinstance(default_branch := table.get("default-branch", "main"), str):
+        msg = f"`default-branch` in {where} must be a string"
+        raise SbomError(msg)
+    if not _strings(patterns := table.get("bundled-wheels", [])):
+        msg = f"`bundled-wheels` in {where} must be an array of strings"
+        raise SbomError(msg)
+    if not isinstance(content_provider := table.get("content-provider"), str | None):
+        msg = f"`content-provider` in {where} must be a string, `module:attribute`"
+        raise SbomError(msg)
+    return Settings(
+        supplier=_supplier(table.get("supplier"), where),
+        repository=None if repository is None else repository.rstrip("/"),
+        default_branch=default_branch,
+        bundled_wheels=tuple(patterns),
+        content_provider=content_provider,
+    )
+
+
+def _strings(values: object) -> TypeIs[list[str]]:
+    return isinstance(values, list) and all(isinstance(value, str) for value in values)
+
+
+def _supplier(value: Toml | None, where: str) -> OrganizationalEntity | None:
+    if value is None:
+        return None
+    if (
+        isinstance(value, dict)
+        and value.keys() <= {"name", "url"}
+        and isinstance(name := value.get("name"), str)
+        and _strings(urls := value.get("url", []))
+    ):
+        return {"name": name, "url": urls}
+    msg = f"`supplier` in {where} must be a table with a string `name` and an optional `url` string array"
+    raise SbomError(msg)
+
+
+__all__ = [
+    "Toml",
+    "settings_from_pyproject",
+    "settings_from_table",
+]

python_sbom/_collect/_source.py

@@ -0,0 +1,54 @@
+from __future__ import annotations
+
+from pathlib import PurePath
+from typing import TYPE_CHECKING, Final
+
+from python_sbom._errors import SbomError
+from python_sbom._model._inputs import BuildScript
+from python_sbom._wheel._zip import sha256_hex
+
+if TYPE_CHECKING:
+    from collections.abc import Iterable
+    from pathlib import Path, PurePosixPath
+
+    from python_sbom._model._inputs import Settings
+
+
+def copyright_notice(root: Path, license_files: list[str]) -> str | None:
+    # the first Copyright line of the license files the project names, as they sit in the project
+    for license_file in license_files:
+        if (path := root / license_file).is_file():
+            for line in path.read_text(encoding="utf-8").splitlines():
+                if line.startswith("Copyright"):
+                    return line
+    return None
+
+
+def selected_wheels(root: Path, settings: Settings) -> list[str]:
+    wheels: Final[set[str]] = set()
+    for pattern in settings.bundled_wheels:
+        # a wheel outside the project is none the project ships, and pathlib refuses an absolute pattern outright
+        if (path := PurePath(pattern)).anchor or ".." in path.parts:
+            msg = f"`bundled-wheels` pattern {pattern!r} must be relative to the project, without `..`"
+            raise SbomError(msg)
+        wheels.update(wheel.relative_to(root).as_posix() for wheel in root.glob(pattern))
+    return sorted(wheels)
+
+
+def hash_build_scripts(root: Path, paths: Iterable[PurePosixPath]) -> list[BuildScript]:
+    # str, so a Windows drive in an absolute path survives the join with the native root; a script named twice, by the
+    # caller and by the provider, is listed once
+    return [BuildScript(path=path, sha256=file_sha256(root / str(path))) for path in dict.fromkeys(paths)]
+
+
+def file_sha256(path: Path) -> str:
+    with path.open("rb") as stream:
+        return sha256_hex(stream)
+
+
+__all__ = [
+    "copyright_notice",
+    "file_sha256",
+    "hash_build_scripts",
+    "selected_wheels",
+]

python_sbom/_collect/_wheel.py

@@ -0,0 +1,55 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Final
+
+from python_sbom._errors import SbomError
+from python_sbom._model._inputs import WheelTree
+from python_sbom._wheel._zip import read_wheel_metadata
+
+from ._source import copyright_notice, file_sha256, hash_build_scripts, selected_wheels
+
+if TYPE_CHECKING:
+    from collections.abc import Iterable, Iterator
+    from pathlib import Path, PurePosixPath
+
+    from python_sbom._model._inputs import Settings
+    from python_sbom._wheel._reader import ArtifactReader
+
+
+def read_wheel(
+    artifact: ArtifactReader, root: Path, settings: Settings, build_scripts: Iterable[PurePosixPath]
+) -> WheelTree:
+    _, metadata = read_wheel_metadata(artifact.archive, str(artifact.path))
+    return WheelTree(
+        name=artifact.path.name,
+        metadata=metadata,
+        copyright=copyright_notice(root, metadata.values("license_files")),
+        bundled=artifact.bundled_wheels(_shipped_copies(root, settings, artifact)),
+        build_scripts=hash_build_scripts(root, build_scripts),
+    )
+
+
+def _shipped_copies(root: Path, settings: Settings, artifact: ArtifactReader) -> list[str]:
+    # the bundled-wheels option selects wheels in the source tree, but the SBOM describes the bytes the artifact ships:
+    # each selected wheel is found in the artifact by its digest and read from there, so a wheel changed or left out
+    # since the build is refused rather than described; identical copies take the artifact's matching members in turn
+    shipped: Final[dict[str, list[str]]] = {}
+    for name in sorted(artifact.members):
+        if name.endswith(".whl"):
+            shipped.setdefault(artifact.digest(name), []).append(name)
+    copies: Final[dict[str, Iterator[str]]] = {digest: iter(names) for digest, names in shipped.items()}
+    found: Final[list[str]] = []
+    for source in selected_wheels(root, settings):
+        if (member := next(copies.get(file_sha256(root / source), iter(())), None)) is None:
+            msg = (
+                f"bundled wheel {source} is not in {artifact.path.name} byte for byte: it changed since the build, or"
+                " the build left it out"
+            )
+            raise SbomError(msg)
+        found.append(member)
+    return found
+
+
+__all__ = [
+    "read_wheel",
+]

python_sbom/_errors.py

@@ -0,0 +1,15 @@
+from __future__ import annotations
+
+
+class SbomError(Exception):
+    """Base of the errors this package raises; the command line reports them without a traceback."""
+
+
+# validation reports every other error per document, and lets this one through, since it fails every document alike
+class MissingExtraError(SbomError): ...  # simplint: ignore[SMP704] # see above
+
+
+__all__ = [
+    "MissingExtraError",
+    "SbomError",
+]

python_sbom/_formats/__init__.py

@@ -0,0 +1 @@
+"""Serialization of documents: CycloneDX JSON and the SPDX 2.3 rendering."""

python_sbom/_formats/_json.py

@@ -0,0 +1,18 @@
+from __future__ import annotations
+
+import json
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from python_sbom._model._types import Document
+
+    from ._spdx import Spdx
+
+
+def dumps(document: Document | Spdx) -> bytes:
+    return f"{json.dumps(document, indent=2)}\n".encode()
+
+
+__all__ = [
+    "dumps",
+]