python-sbom 0.1.1__py3-none-any.whl

python_sbom/_model/_types.py

@@ -0,0 +1,281 @@
+from __future__ import annotations
+
+from typing import NotRequired, TypedDict
+
+# the document model is our own: cyclonedx-python-lib models neither formulation nor compositions (its model/bom.py
+# holds only commented-out stubs, tracked upstream in CycloneDX/cyclonedx-python-lib#578 and #581), and its SortedSet
+# collections re-order lists whose order the document keeps on purpose
+
+
+class OrganizationalEntity(TypedDict):
+    """
+    An organization that supplies or makes a component.
+
+    :ivar name: the organization's name
+    :ivar url: the organization's web sites
+
+    """
+
+    name: str
+    url: list[str]
+
+
+class Contact(TypedDict):
+    """
+    A person who wrote or maintains a component.
+
+    :ivar name: the person's name
+    :ivar email: the person's email address
+
+    """
+
+    name: NotRequired[str]
+    email: NotRequired[str]
+
+
+class Hash(TypedDict):
+    """
+    A digest of a component's bytes.
+
+    :ivar alg: the algorithm as CycloneDX names it, such as ``SHA-256``
+    :ivar content: the digest as lowercase hex
+
+    """
+
+    alg: str
+    content: str
+
+
+class Property(TypedDict):
+    """
+    A name and value pair attached to a component or the document.
+
+    :ivar name: the property's name; names this package writes start with the project's prefix or ``python:``
+    :ivar value: the property's value
+
+    """
+
+    name: str
+    value: str
+
+
+class ExternalReference(TypedDict):
+    """
+    A link from a component to something about it.
+
+    :ivar type: the CycloneDX reference type, such as ``vcs`` or ``website``
+    :ivar url: the link
+    :ivar comment: what the link is, such as the Project-URL label it came from
+
+    """
+
+    type: str
+    url: str
+    comment: NotRequired[str]
+
+
+class LicenseInfo(TypedDict):
+    """
+    One license, by SPDX id or by name.
+
+    :ivar id: the SPDX license id
+    :ivar name: the license's name, when it has no SPDX id
+    :ivar acknowledgement: ``declared`` when the metadata names it, ``concluded`` when python-sbom resolved it
+
+    """
+
+    id: NotRequired[str]
+    name: NotRequired[str]
+    acknowledgement: str
+
+
+class License(TypedDict):
+    """
+    One license entry of a component: a license or an SPDX expression.
+
+    :ivar license: the license, by SPDX id or by name
+    :ivar expression: an SPDX license expression
+    :ivar acknowledgement: ``declared`` or ``concluded``, for an expression
+
+    """
+
+    license: NotRequired[LicenseInfo]
+    expression: NotRequired[str]
+    acknowledgement: NotRequired[str]
+
+
+class IdentityMethod(TypedDict):
+    """
+    How a component's identity was established.
+
+    :ivar technique: the CycloneDX technique, such as ``manifest-analysis`` or ``hash-comparison``
+    :ivar confidence: from 0 to 1
+    :ivar value: what the technique read, such as the path of a ``METADATA`` file or a digest
+
+    """
+
+    technique: str
+    confidence: int
+    value: str
+
+
+class Identity(TypedDict):
+    """
+    The evidence for one field of a component's identity.
+
+    :ivar field: the field it supports, such as ``purl`` or ``hash``
+    :ivar confidence: from 0 to 1
+    :ivar methods: how it was established
+
+    """
+
+    field: str
+    confidence: int
+    methods: list[IdentityMethod]
+
+
+class Evidence(TypedDict):
+    """
+    Why a component is what it says it is.
+
+    :ivar identity: the evidence per field
+
+    """
+
+    identity: list[Identity]
+
+
+#: A CycloneDX 1.6 component: a distribution, an application, the interpreter or a file, as a dictionary with the keys
+#: the CycloneDX component object defines, of which python-sbom uses ``type``, ``bom-ref``, ``name``, ``version``,
+#: ``purl``, ``supplier``, ``authors``, ``description``, ``licenses``, ``copyright``, ``hashes``,
+#: ``externalReferences``, ``properties``, ``evidence`` and the nested ``components``.
+Component = TypedDict(
+    "Component",
+    {
+        "type": str,
+        "bom-ref": str,
+        "name": str,
+        "version": NotRequired[str],
+        "purl": NotRequired[str],
+        "supplier": NotRequired[OrganizationalEntity],
+        "authors": NotRequired[list[Contact]],
+        "description": NotRequired[str],
+        "licenses": NotRequired[list[License]],
+        "copyright": NotRequired[str],
+        "hashes": NotRequired[list[Hash]],
+        "externalReferences": NotRequired[list[ExternalReference]],
+        "properties": NotRequired[list[Property]],
+        "evidence": NotRequired[Evidence],
+        "components": NotRequired[list["Component"]],
+    },
+)
+
+
+class Dependency(TypedDict):
+    """
+    The components one component depends on.
+
+    :ivar ref: the ``bom-ref`` of the dependent component
+    :ivar dependsOn: the ``bom-ref`` values of its dependencies
+
+    """
+
+    ref: str
+    dependsOn: NotRequired[list[str]]
+
+
+class _Reference(TypedDict):
+    ref: str
+
+
+class _Output(TypedDict):
+    type: str
+    resource: _Reference
+
+
+class _Input(TypedDict):
+    environmentVars: list[Property]
+
+
+_Workflow = TypedDict(
+    "_Workflow",
+    {
+        "bom-ref": str,
+        "uid": str,
+        "name": str,
+        "taskTypes": list[str],
+        "resourceReferences": list[_Reference],
+        "outputs": list[_Output],
+        "inputs": NotRequired[list[_Input]],
+    },
+)
+
+
+#: How an artifact was built: one or more workflows, each with its tools, inputs and outputs.
+Formula = TypedDict("Formula", {"bom-ref": str, "workflows": list[_Workflow]})
+
+
+class Lifecycle(TypedDict):
+    phase: str
+
+
+class Tools(TypedDict):
+    components: list[Component]
+
+
+class Metadata(TypedDict):
+    timestamp: str
+    lifecycles: list[Lifecycle]
+    tools: Tools
+    manufacturer: NotRequired[OrganizationalEntity]
+    authors: NotRequired[list[Contact]]
+    supplier: NotRequired[OrganizationalEntity]
+    component: Component
+    licenses: NotRequired[list[License]]
+    properties: NotRequired[list[Property]]
+
+
+class Composition(TypedDict):
+    aggregate: str
+    dependencies: list[str]
+
+
+class Body(TypedDict):
+    metadata: Metadata
+    components: list[Component]
+    dependencies: list[Dependency]
+    compositions: NotRequired[list[Composition]]
+    formulation: NotRequired[list[Formula]]
+
+
+_Header = TypedDict(
+    "_Header",
+    {"$schema": str, "bomFormat": str, "specVersion": str, "serialNumber": str, "version": int},
+)
+
+
+class Document(Body, _Header):
+    pass
+
+
+__all__ = [
+    "Body",
+    "Component",
+    "Composition",
+    "Contact",
+    "Dependency",
+    "Document",
+    "Evidence",
+    "ExternalReference",
+    "Formula",
+    "Hash",
+    "Identity",
+    "IdentityMethod",
+    "License",
+    "LicenseInfo",
+    "Lifecycle",
+    "Metadata",
+    "OrganizationalEntity",
+    "Property",
+    "Tools",
+]

python_sbom/_model/_wheel.py

@@ -0,0 +1,89 @@
+from __future__ import annotations
+
+from pathlib import PurePosixPath
+from typing import TYPE_CHECKING, Final
+
+from packaging.requirements import Requirement
+
+from ._components import bundled_wheel_component, component_from_metadata, vendored_dependencies
+from ._inputs import Artifact
+from ._markers import applies_without_extras
+from ._names import purl, pypi_url, sbom_file_name, sbom_property
+
+if TYPE_CHECKING:
+    from collections.abc import Mapping
+
+    from ._content import BundledWheelInfo
+    from ._identity import ProjectIdentity
+    from ._inputs import Settings, WheelTree
+    from ._types import Component
+
+
+def wheel_artifact(
+    tree: WheelTree, *, project: ProjectIdentity, settings: Settings, bundled_info: Mapping[str, BundledWheelInfo]
+) -> Artifact:
+    metadata = tree.metadata
+    bundled = [
+        bundled_wheel_component(wheel, project.property_prefix, bundled_info[wheel.path]) for wheel in tree.bundled
+    ]
+    root = component_from_metadata(metadata)
+    root["type"] = "application"
+    name, version = metadata.name, metadata.version
+    root["externalReferences"] = [
+        *root["externalReferences"],
+        {"type": "distribution", "url": pypi_url(name, version)},
+        {"type": "attestation", "url": f"https://pypi.org/integrity/{name}/{version}/{tree.name}/provenance"},
+    ]
+    return Artifact(
+        root=root,
+        components=[
+            *bundled,
+            # hatchling repeats a Requires-Dist when a hook re-adds a dependency, and each bom-ref must be unique
+            *(
+                _declared_dependency(text, requirement, project.property_prefix)
+                for text in dict.fromkeys(metadata.values("requires_dist"))
+                if applies_without_extras(requirement := Requirement(text))
+            ),
+        ],
+        dependencies=vendored_dependencies(bundled),
+        build_scripts=tree.build_scripts,
+        version=version,
+        sbom_file=sbom_file_name(name),
+        scope=_scope(settings),
+        workflow=("wheel-build", "build the wheel and this SBOM"),
+        license_file=next(iter(metadata.values("license_files")), None),
+        copyright=tree.copyright,
+    )
+
+
+def _declared_dependency(text: str, requirement: Requirement, property_prefix: str) -> Component:
+    component: Final[Component] = {
+        "type": "library",
+        # the same distribution can be declared more than once with different markers, so the purl is not unique
+        "bom-ref": f"requires-dist:{text}",
+        "name": requirement.name,
+        "purl": purl(requirement.name),
+        "externalReferences": [{"type": "distribution", "url": pypi_url(requirement.name)}],
+        "properties": [{"name": sbom_property(property_prefix, "requires-dist"), "value": text}],
+    }
+    if requirement.marker is not None:
+        component["properties"].append({
+            "name": sbom_property(property_prefix, "environment-marker"),
+            "value": str(requirement.marker),
+        })
+    return component
+
+
+def _scope(settings: Settings) -> str:
+    declared = (
+        "the direct runtime dependencies declared in the wheel's core metadata, which carry no version because the"
+        " installer resolves them"
+    )
+    if directories := " and ".join(sorted({str(PurePosixPath(pattern).parent) for pattern in settings.bundled_wheels})):
+        return f"Components are the wheels bundled as data files under {directories} and {declared}."
+    return f"Components are {declared}."
+
+
+__all__ = [
+    "wheel_artifact",
+]

python_sbom/_validate/__init__.py

@@ -0,0 +1 @@
+"""Checks of a document against the schema, its own references, its SPDX rendering and a license policy."""

python_sbom/_validate/_artifact.py

@@ -0,0 +1,55 @@
+from __future__ import annotations
+
+import zipfile
+from typing import IO, TYPE_CHECKING, Final
+
+from python_sbom._wheel._zip import CHUNK, open_archive
+
+from ._document import validate_document
+from ._limits import MAX_COMPONENTS, byte_limit
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+
+def validate(path: Path, allowed_licenses: frozenset[str] | None, max_components: int = MAX_COMPONENTS) -> list[str]:
+    limit: Final[int] = byte_limit(max_components)
+    if not zipfile.is_zipfile(path):
+        with path.open("rb") as file:
+            return validate_document(_read(file, limit), allowed_licenses, max_components)
+    problems: Final[list[str]] = []
+    with open_archive(path, path.name) as archive:
+        if not (names := sorted(name for name in archive.namelist() if _is_sbom(name))):
+            return [f"no SBOM found in {path.name}"]
+        # one at a time, so memory holds a single SBOM however many the archive packs
+        for name in names:
+            with archive.open(name) as member:
+                raw = _read(member, limit)
+            problems.extend(
+                f"{name}: {problem}" for problem in validate_document(raw, allowed_licenses, max_components)
+            )
+    return problems
+
+
+def _read(source: IO[bytes], limit: int) -> bytes:
+    # stops within a chunk past the limit: enough for validate_document to refuse the SBOM without reading all of it
+    content: Final[bytearray] = bytearray()
+    while len(content) <= limit and (chunk := source.read(CHUNK)):
+        content.extend(chunk)
+    return bytes(content)
+
+
+def _is_sbom(name: str) -> bool:
+    # every top-level .dist-info counts, so a decoy one sorting first cannot hide the SBOM of another
+    match name.split("/"):
+        case [file]:
+            return file.endswith(".cdx.json")
+        case [dist_info, "sboms", file]:
+            return dist_info.endswith(".dist-info") and file.endswith(".cdx.json")
+        case _:
+            return False
+
+
+__all__ = [
+    "validate",
+]

python_sbom/_validate/_document.py

@@ -0,0 +1,221 @@
+from __future__ import annotations
+
+import json
+import math
+import re
+from importlib.util import find_spec
+from typing import TYPE_CHECKING, Any, Final
+
+from python_sbom._errors import MissingExtraError, SbomError
+from python_sbom._formats._spdx import render_spdx
+from python_sbom._model._graph import artifact_contents, reference_problems, repeated, walk_components
+from python_sbom._model._names import SBOM_FILE
+
+from ._limits import MAX_COMPONENTS, MAX_TEXT, Json, byte_limit
+
+if TYPE_CHECKING:
+    from collections.abc import Iterable
+    from typing import TypeIs  # new in 3.13; annotations here are never evaluated
+
+    from python_sbom._model._types import Component, Document
+
+# the schema leaves these optional, yet every document python-sbom writes carries them and the checks below read them
+_REQUIRED: Final[tuple[tuple[str, ...], ...]] = (
+    ("metadata", "timestamp"),
+    ("metadata", "tools", "components"),
+    ("metadata", "component", "bom-ref"),
+    ("metadata", "component", "name"),
+    ("components",),
+    ("dependencies",),
+)
+_SERIAL: Final[re.Pattern[str]] = re.compile(r"urn:uuid:[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}")
+# the modules of the validate extra
+_EXTRA: Final[tuple[str, ...]] = ("jsonschema_rs", "license_expression", "spdx_tools")
+# the schema validator recurses once per level and crashes the process past a few thousand levels on a small thread
+# stack; the SBOMs python-sbom and other tools write nest about a dozen levels deep
+_MAX_DEPTH: Final[int] = 128
+
+
+def validate_document(
+    raw: bytes, allowed_licenses: frozenset[str] | None, max_components: int = MAX_COMPONENTS
+) -> list[str]:
+    if len(raw) > (limit := byte_limit(max_components)):
+        problem = (
+            f"is over {limit} bytes, the size limit for {max_components} components;"
+            " pass a higher --max-components to check it"
+        )
+        return [problem]
+    # one document's problem never stops the others from being checked, so whatever the checks raise is reported
+    try:
+        return _problems(raw, allowed_licenses, max_components)
+    except MissingExtraError:
+        raise
+    except SbomError as error:
+        return [str(error)]
+    except Exception as error:  # ruff: ignore[blind-except] # a hostile document may trip any check, see above
+        return [f"could not be checked, {type(error).__name__}: {str(error)[:MAX_TEXT]}"]
+
+
+def _problems(raw: bytes, allowed_licenses: frozenset[str] | None, max_components: int) -> list[str]:
+    # every document must conform to the CycloneDX 1.6 schema; one python-sbom wrote must also carry the urn:uuid serial
+    # actions/attest needs, resolvable and unique bom-refs and a valid SPDX rendering, and meet the license policy. A
+    # missing extra fails every document alike, while importing it, half a second, waits until the cheap census passes
+    missing_extra: Final[str] = "validation needs the validate extra: pip install 'python-sbom[validate]'"
+    if any(find_spec(name) is None for name in _EXTRA):
+        raise MissingExtraError(missing_extra)
+    loaded: Final = load_json(raw)
+    if census := _census_problems(loaded, max_components):
+        return census
+    # find_spec sees only the top-level modules, not a dependency of theirs that is missing or broken
+    try:
+        from ._schemas import schema_error, spdx_problems  # ruff: ignore[import-outside-top-level] # the optional extra, see above
+    except ImportError as error:
+        raise MissingExtraError(missing_extra) from error
+
+    if error := schema_error(loaded):
+        return [f"does not conform to the CycloneDX 1.6 schema: {error}"]
+    # python-sbom names its file in a metadata property, so documents other tools wrote get the schema check alone; the
+    # schema already guarantees the shape of what this reads
+    if not any(
+        prop.get("name", "").endswith(f":{SBOM_FILE}") for prop in loaded.get("metadata", {}).get("properties", [])
+    ):
+        return []
+    missing: Final[list[str]] = [".".join(path) for path in _REQUIRED if not _has(loaded, path)]
+    # is_document holds once nothing is missing; asking it narrows the loaded JSON to a document
+    if missing or not is_document(loaded):
+        return [f"lacks {', '.join(missing)}, which every document python-sbom writes carries"]
+    # optional in the CycloneDX spec itself, but actions/attest's format sniffer requires it to recognize the document
+    # as CycloneDX at all, and silently rejects anything missing it as an unknown format
+    if not _SERIAL.fullmatch(serial := loaded.get("serialNumber", "")):
+        return [f"serialNumber must be a urn:uuid: RFC 4122 UUID, got {serial!r}"]
+    # the SPDX rendering of a document whose references are broken only repeats those problems
+    problems: Final[list[str]] = _reference_problems(loaded) or spdx_problems(render_spdx(loaded))
+    if allowed_licenses is not None:
+        # the license parser loads only for a policy to judge against, as most checks have none
+        try:
+            from ._licenses import license_problems  # ruff: ignore[import-outside-top-level] # see above
+        except ImportError as error:
+            raise MissingExtraError(missing_extra) from error
+        problems.extend(
+            license_problems(artifact_contents(loaded["metadata"]["component"], loaded["components"]), allowed_licenses)
+        )
+    return problems
+
+
+def load_json(raw: bytes) -> Any:  # ruff: ignore[any-type] # json.loads' own type: any shape until checked
+    # why bytes are no JSON value, worded once for validate and for spdx, which renders a document without validating it
+    try:
+        return json.loads(raw.decode("utf-8"))
+    except UnicodeDecodeError as error:
+        msg = f"is not UTF-8: {error.reason} at byte {error.start}"
+        raise SbomError(msg) from error
+    except json.JSONDecodeError as error:
+        msg = f"is not JSON: {error}"
+        raise SbomError(msg) from error
+    except RecursionError as error:
+        # the JSON parser recurses once per level and gives up near the recursion limit, long before the depth check
+        # sees the document
+        msg = "nests too deep for the JSON parser to read"
+        raise SbomError(msg) from error
+
+
+def _census_problems(document: Json, max_components: int) -> list[str]:
+    # counts the components and bom-refs and measures the depth in one linear pass before the schema vouches for any
+    # shape; iterative, so deep nesting cannot exhaust the stack. Nested components count too: the schema checks every
+    # components array, and one array may hold them all, so only the total bounds its work
+    components = 0
+    refs: Final[list[str]] = []
+    # only lists and objects go on the stack, every other value is judged where its container is read, so a scalar
+    # nests too deep when its container already sits at the limit; the document starts out as the one item of a list
+    pending: Final[list[tuple[list[Json] | dict[str, Json], int]]] = [([document], 0)]
+    while pending:
+        node, depth = pending.pop()
+        if isinstance(node, dict):
+            if isinstance(ref := node.get("bom-ref"), str):
+                refs.append(ref)
+            components += len(listed) if isinstance(listed := node.get("components"), list) else 0
+        if components > max_components:
+            return [
+                (
+                    f"lists more than {max_components} components, nested ones included;"
+                    " pass a higher --max-components to check it"
+                )
+            ]
+        children = node.values() if isinstance(node, dict) else node
+        if children and depth >= _MAX_DEPTH:
+            return [f"nests more than {_MAX_DEPTH} levels deep, too deep to check"]
+        if problem := _push_containers(children, depth + 1, pending):
+            return problem
+    # every bom-ref in the document, wherever it sits, as CycloneDX requires them unique and documents other tools wrote
+    # get no reference check beyond this
+    if twice := repeated(refs):
+        return [f"bom-ref values repeat: {', '.join(twice)}"]
+    return []
+
+
+def _push_containers(
+    children: Iterable[Json], depth: int, pending: list[tuple[list[Json] | dict[str, Json], int]]
+) -> list[str]:
+    for child in children:
+        if isinstance(child, (list, dict)):
+            pending.append((child, depth))
+        elif isinstance(child, float) and not math.isfinite(child):
+            # json.loads takes NaN and Infinity, which RFC 8259 leaves out of JSON and the schema reads as null
+            return [f"holds {child}, which is not a JSON number"]
+    return []
+
+
+def _has(value: Json, path: tuple[str, ...]) -> bool:
+    for key in path:
+        if not isinstance(value, dict) or key not in value:
+            return False
+        value = value[key]
+    return True
+
+
+def is_document(value: object) -> TypeIs[Document]:
+    # what the SPDX rendering reads without a fallback; the schema leaves even these optional
+    return isinstance(value, dict) and all(_has(value, ("metadata", "component", key)) for key in ("bom-ref", "name"))
+
+
+def _reference_problems(document: Document) -> list[str]:
+    root: Final[Component] = document["metadata"]["component"]
+    tools: Final[list[Component]] = document["metadata"]["tools"]["components"]
+    components: Final[list[Component]] = list(walk_components([root, *document["components"], *tools]))
+    if nameless := sorted(component["name"] for component in components if "bom-ref" not in component):
+        return [f"components lack a bom-ref: {nameless}"]
+    formulas: Final[list[str]] = [
+        ref
+        for formula in document.get("formulation", [])
+        for ref in (formula["bom-ref"], *(workflow["bom-ref"] for workflow in formula.get("workflows", [])))
+    ]
+    if problems := reference_problems(
+        [*(component["bom-ref"] for component in components), *formulas], document["dependencies"]
+    ):
+        return problems
+    problems: Final[list[str]] = []
+    dependencies: Final[dict[str, set[str]]] = {
+        entry["ref"]: set(entry.get("dependsOn", [])) for entry in document["dependencies"]
+    }
+    direct: Final[set[str]] = {component["bom-ref"] for component in document["components"]}
+    if dependencies.get(root["bom-ref"]) != direct:
+        problems.append(
+            f"root dependsOn {sorted(dependencies.get(root['bom-ref'], []))} != components {sorted(direct)}"
+        )
+    tool_refs: Final[set[str]] = {tool["bom-ref"] for tool in tools}
+    for formula in document.get("formulation", []):
+        for workflow in formula.get("workflows", []):
+            # a workflow may also name an external resource, which no tool can be
+            referenced = {
+                reference["ref"] for reference in workflow.get("resourceReferences", []) if "ref" in reference
+            }
+            if unknown := referenced - tool_refs:
+                problems.append(f"workflow {workflow['uid']} references unknown tools: {sorted(unknown)}")
+    return problems
+
+
+__all__ = [
+    "is_document",
+    "load_json",
+    "validate_document",
+]