datalab-capsule 0.1.0__py3-none-any.whl

datalab_capsule/__init__.py

@@ -0,0 +1,10 @@
+# Copyright (c) DataLab Platform Developers, BSD 3-Clause license, see LICENSE file.
+
+"""Neutral provenance model and open capsules for DataLab workspaces.
+
+This package never imports DataLab, Sigima, SigimaX or Qt. Its core only needs
+the standard library and ``jsonschema``; HDF5 helpers import NumPy and h5py
+lazily.
+"""
+
+__version__ = "0.1.0"

datalab_capsule/_schema.py

@@ -0,0 +1,31 @@
+# Copyright (c) DataLab Platform Developers, BSD 3-Clause license, see LICENSE file.
+
+"""Packaged JSON Schemas and validation helper."""
+
+from __future__ import annotations
+
+import json
+from functools import lru_cache
+from importlib import resources
+from typing import Any
+
+__all__ = ["load_schema", "schema_errors"]
+
+
+@lru_cache(maxsize=None)
+def load_schema(name: str) -> dict[str, Any]:
+    """Return a packaged JSON Schema (e.g. ``"ledger-1"``)."""
+    path = resources.files("datalab_capsule").joinpath(f"schemas/{name}.schema.json")
+    return json.loads(path.read_text(encoding="utf-8"))
+
+
+def schema_errors(name: str, data: Any) -> list[str]:
+    """Return the JSON Schema errors of *data*, sorted by location."""
+    import jsonschema  # pylint: disable=import-outside-toplevel
+
+    validator = jsonschema.Draft202012Validator(load_schema(name))
+    errors = sorted(validator.iter_errors(data), key=lambda e: list(e.absolute_path))
+    return [
+        f"{'/'.join(str(p) for p in error.absolute_path) or '<root>'}: {error.message}"
+        for error in errors
+    ]

datalab_capsule/archive.py

@@ -0,0 +1,420 @@
+# Copyright (c) DataLab Platform Developers, BSD 3-Clause license, see LICENSE file.
+
+"""Workspace capsules: ``.dlcapsule`` ZIP archives holding a workspace and its manifest.
+
+A capsule holds ``ro-crate-metadata.json`` (see :mod:`datalab_capsule.manifest`)
+and, when the workspace is included, ``workspace.h5`` (the DataLab HDF5 workspace,
+provenance block included), both stored without compression. A capsule may
+instead reference its workspace by an absolute IRI: the archive then holds the
+manifest only, and the workspace is obtained through a resolver passed
+explicitly by the caller; its SHA-256 and size are checked before use.
+
+The reader never executes code, never accesses the network and refuses unsafe
+archives before extracting anything: absolute paths, ``..``, drive letters,
+backslashes, duplicate or unexpected entries, links, encrypted or compressed
+entries (so no decompression bomb can be read), and entries over the limits
+below.
+
+Size limits form a :class:`SizePolicy`. The archive size is checked before the
+archive is read; the size of each entry is checked against its header before
+extraction and against the bytes actually extracted. Each edition can pass its
+own policy to :func:`read_capsule`. :data:`DEFAULT_SIZE_POLICY` is an accepted
+upper bound, not a tested capacity: the whole archive is held in memory.
+"""
+
+from __future__ import annotations
+
+import dataclasses
+import io
+import json
+import os
+import stat
+import zipfile
+from collections.abc import Callable, Mapping, Sequence
+from typing import Any
+
+from datalab_capsule.hdf5 import ProvenanceFormatError, read_block
+from datalab_capsule.integrity import ProvenanceError, load_json, sha256_digest
+from datalab_capsule.manifest import (
+    ATTACHMENTS_DIR,
+    MANIFEST_NAME,
+    REFERENCED_KEY,
+    WORKSPACE_NAME,
+    ManifestError,
+    attachment_entities,
+    attachment_name,
+    build_manifest,
+    check_attachment_name,
+    validate_manifest,
+    workspace_entity,
+)
+
+__all__ = [
+    "CAPSULE_SUFFIX",
+    "DEFAULT_SIZE_POLICY",
+    "MAX_ARCHIVE_BYTES",
+    "MAX_ATTACHMENTS",
+    "MAX_ATTACHMENT_BYTES",
+    "MAX_ENTRIES",
+    "MAX_MANIFEST_BYTES",
+    "MAX_WORKSPACE_BYTES",
+    "Capsule",
+    "CapsuleError",
+    "SizePolicy",
+    "create_from_hdf5",
+    "read_capsule",
+]
+
+CAPSULE_SUFFIX = ".dlcapsule"
+#: Largest number of entries accepted in an archive.
+MAX_ENTRIES = 16
+#: Largest manifest accepted by default (bytes, uncompressed).
+MAX_MANIFEST_BYTES = 64 * 1024 * 1024
+#: Largest workspace accepted by default (bytes, uncompressed).
+MAX_WORKSPACE_BYTES = 4 * 1024**3
+#: Largest number of light attachments in a capsule.
+MAX_ATTACHMENTS = 8
+#: Largest attachment accepted by default (bytes): attachments are light files.
+MAX_ATTACHMENT_BYTES = 16 * 1024 * 1024
+#: Largest archive accepted by default (bytes): every entry plus ZIP headers.
+MAX_ARCHIVE_BYTES = (
+    MAX_MANIFEST_BYTES
+    + MAX_WORKSPACE_BYTES
+    + MAX_ATTACHMENTS * MAX_ATTACHMENT_BYTES
+    + 1024 * 1024
+)
+_ENTRY_NAMES = (MANIFEST_NAME, WORKSPACE_NAME)
+
+
+class CapsuleError(ProvenanceError, ValueError):
+    """Raised when a capsule cannot be created or is not safe or valid."""
+
+
+@dataclasses.dataclass(frozen=True)
+class SizePolicy:
+    """Size limits applied when reading a capsule (bytes).
+
+    Attributes:
+        max_archive_bytes: Largest archive, checked before it is read.
+        max_manifest_bytes: Largest manifest (uncompressed).
+        max_workspace_bytes: Largest workspace (uncompressed).
+        max_attachment_bytes: Largest attachment (uncompressed).
+    """
+
+    max_archive_bytes: int
+    max_manifest_bytes: int
+    max_workspace_bytes: int
+    max_attachment_bytes: int = MAX_ATTACHMENT_BYTES
+
+    def entry_limit(self, name: str) -> int:
+        """Return the size limit of entry *name*."""
+        if name == MANIFEST_NAME:
+            return self.max_manifest_bytes
+        if name.startswith(ATTACHMENTS_DIR):
+            return self.max_attachment_bytes
+        return self.max_workspace_bytes
+
+
+#: Policy used when an edition does not pass its own.
+DEFAULT_SIZE_POLICY = SizePolicy(
+    max_archive_bytes=MAX_ARCHIVE_BYTES,
+    max_manifest_bytes=MAX_MANIFEST_BYTES,
+    max_workspace_bytes=MAX_WORKSPACE_BYTES,
+)
+
+
+@dataclasses.dataclass(frozen=True)
+class Capsule:
+    """A validated capsule.
+
+    Attributes:
+        manifest: RO-Crate manifest.
+        workspace: Bytes of the workspace, to pass to the editions' open path, or
+         None when it is referenced and no resolver was given.
+        workspace_ref: IRI of a referenced workspace, or None when included.
+        attachments: ``name -> bytes`` of the declared attachments, checked
+         against the manifest. They are data for the user: nothing in this
+         package opens, imports or runs them.
+    """
+
+    manifest: dict[str, Any]
+    workspace: bytes | None
+    workspace_ref: str | None = None
+    attachments: dict[str, bytes] = dataclasses.field(default_factory=dict)
+
+
+def _read_source(source: str | os.PathLike | bytes, limit: int | None = None) -> bytes:
+    """Return the bytes of *source*, refusing more than *limit* before reading."""
+    if isinstance(source, (bytes, bytearray, memoryview)):
+        if limit is not None and len(source) > limit:
+            raise CapsuleError(f"Archive too large: {len(source)} > {limit} bytes")
+        return bytes(source)
+    with open(source, "rb") as file:
+        if limit is None:
+            return file.read()
+        size = os.fstat(file.fileno()).st_size
+        if size > limit:
+            raise CapsuleError(f"Archive too large: {size} > {limit} bytes")
+        data = file.read(size + 1)
+    if len(data) > limit:
+        raise CapsuleError(f"Archive too large: more than {limit} bytes")
+    return data
+
+
+def create_from_hdf5(
+    source: str | os.PathLike | bytes,
+    *,
+    name: str | None = None,
+    description: str | None = None,
+    license_: str | None = None,
+    date_published: str | None = None,
+    revision_of: str | None = None,
+    workspace_ref: str | None = None,
+    attachments: Sequence[Mapping[str, Any]] = (),
+) -> bytes:
+    """Create a capsule from a workspace file holding a provenance block.
+
+    Args:
+        source: Path or bytes of the HDF5 workspace.
+        name: Capsule name.
+        description: Capsule description.
+        license_: Licence statement; a neutral text is used when None.
+        date_published: ISO 8601 date (defaults to now).
+        revision_of: Manifest fingerprint of the parent revision, if any.
+        workspace_ref: Absolute IRI where the workspace will be published. When
+         given, the workspace is referenced, not included: the archive holds
+         the manifest only, with the workspace's SHA-256 and size.
+        attachments: Light files selected by the author, stored as
+         ``attachments/<name>`` and declared in the manifest with their size and
+         SHA-256: ``{"name", "data", "media_type", "description"}``
+         (``description`` optional).
+
+    Returns:
+        The capsule bytes.
+
+    Raises:
+        CapsuleError: If the workspace has no (valid) provenance block, or the
+         attachments are too many or too large.
+        ManifestError: If a manifest field is invalid (parent revision,
+         workspace IRI, attachment name, media type).
+    """
+    import h5py  # pylint: disable=import-outside-toplevel
+
+    if len(attachments) > MAX_ATTACHMENTS:
+        raise CapsuleError(f"Too many attachments (at most {MAX_ATTACHMENTS})")
+    declared = []
+    for item in attachments:
+        data = bytes(item["data"])
+        if len(data) > MAX_ATTACHMENT_BYTES:
+            raise CapsuleError(f"Attachment too large: {item['name']!r}")
+        declared.append(
+            {
+                "name": item["name"],
+                "size": len(data),
+                "sha256": sha256_digest(data),
+                "media_type": item.get("media_type") or "application/octet-stream",
+                "description": item.get("description"),
+                "data": data,
+            }
+        )
+
+    workspace = _read_source(source)
+    try:
+        with h5py.File(io.BytesIO(workspace), "r") as h5file:
+            block = read_block(h5file)
+    except (OSError, ProvenanceFormatError) as exc:
+        raise CapsuleError(f"Invalid workspace: {exc}") from exc
+    if block is None:
+        raise CapsuleError("The workspace has no provenance block")
+    ledger, locators = block
+    options = {
+        key: value
+        for key, value in (
+            ("name", name),
+            ("description", description),
+            ("license_", license_),
+            ("date_published", date_published),
+            ("revision_of", revision_of),
+            ("workspace_ref", workspace_ref),
+        )
+        if value is not None
+    }
+    manifest = build_manifest(
+        ledger,
+        locators,
+        workspace_sha256=sha256_digest(workspace),
+        workspace_size=len(workspace),
+        attachments=declared,
+        **options,
+    )
+    entries = [(MANIFEST_NAME, json.dumps(manifest, ensure_ascii=False, indent=1))]
+    if workspace_ref is None:
+        entries.append((WORKSPACE_NAME, workspace))
+    entries.extend((ATTACHMENTS_DIR + item["name"], item["data"]) for item in declared)
+    buffer = io.BytesIO()
+    with zipfile.ZipFile(buffer, "w", compression=zipfile.ZIP_STORED) as archive:
+        for entry, data in entries:
+            info = zipfile.ZipInfo(entry, date_time=(1980, 1, 1, 0, 0, 0))
+            info.compress_type = zipfile.ZIP_STORED
+            # Unix permissions, whatever the host: same bytes on every platform.
+            info.create_system = 3
+            info.external_attr = (stat.S_IFREG | 0o644) << 16
+            archive.writestr(info, data)
+    return buffer.getvalue()
+
+
+def _check_entry(info: zipfile.ZipInfo, seen: set[str], policy: SizePolicy) -> None:
+    name = info.filename
+    if (
+        not name
+        or name.startswith("/")
+        or "\\" in name
+        or ".." in name.split("/")
+        or (len(name) > 1 and name[1] == ":")
+    ):
+        raise CapsuleError(f"Unsafe entry name: {name!r}")
+    if name in seen:
+        raise CapsuleError(f"Duplicate entry: {name}")
+    seen.add(name)
+    if name.startswith(ATTACHMENTS_DIR):
+        try:
+            check_attachment_name(name[len(ATTACHMENTS_DIR) :])
+        except ManifestError as exc:
+            raise CapsuleError(f"Unsafe entry name: {name!r}") from exc
+    elif name not in _ENTRY_NAMES:
+        raise CapsuleError(f"Unexpected entry: {name}")
+    if info.flag_bits & 0x1:
+        raise CapsuleError(f"Encrypted entry: {name}")
+    if stat.S_ISLNK(info.external_attr >> 16):
+        raise CapsuleError(f"Link entry: {name}")
+    if info.compress_type != zipfile.ZIP_STORED:
+        raise CapsuleError(f"Compressed entry: {name}")
+    if info.file_size > policy.entry_limit(name):
+        raise CapsuleError(f"Entry too large: {name}")
+
+
+def _read_entry(archive: zipfile.ZipFile, info: zipfile.ZipInfo, limit: int) -> bytes:
+    try:
+        with archive.open(info) as stream:
+            # The header size was checked against *limit*: never read past it.
+            data = stream.read(info.file_size + 1)
+    except (zipfile.BadZipFile, EOFError, ValueError) as exc:
+        # Raised while reading, e.g. on a CRC mismatch, a truncated entry or a
+        # local header name that cannot be decoded
+        raise CapsuleError(f"Damaged entry {info.filename}: {exc}") from exc
+    if len(data) > limit or len(data) != info.file_size:
+        raise CapsuleError(f"Entry size does not match its header: {info.filename}")
+    return data
+
+
+def read_capsule(
+    source: str | os.PathLike | bytes,
+    policy: SizePolicy = DEFAULT_SIZE_POLICY,
+    resolve_workspace: Callable[[str], bytes] | None = None,
+) -> Capsule:
+    """Read, check and validate a capsule without extracting it to disk.
+
+    Args:
+        source: Path or bytes of the capsule.
+        policy: Size limits of the calling edition.
+        resolve_workspace: ``IRI -> bytes`` used for a referenced workspace. It
+         is the only way a referenced workspace is obtained: nothing is fetched
+         by default. The bytes are checked against the manifest.
+
+    Returns:
+        The capsule; ``workspace`` is None for a referenced workspace without
+        resolver.
+
+    Raises:
+        CapsuleError: If the archive is unsafe, too large, incomplete or
+         inconsistent, or the workspace does not match the manifest.
+        ManifestError: If the manifest is invalid.
+    """
+    data = _read_source(source, policy.max_archive_bytes)
+    try:
+        archive = zipfile.ZipFile(io.BytesIO(data))
+    except (zipfile.BadZipFile, ValueError) as exc:
+        # ValueError: e.g. an entry name flagged UTF-8 that is not UTF-8
+        raise CapsuleError(f"Not a ZIP archive: {exc}") from exc
+    with archive:
+        infos = archive.infolist()
+        if len(infos) > MAX_ENTRIES:
+            raise CapsuleError("Too many entries")
+        seen: set[str] = set()
+        for info in infos:
+            _check_entry(info, seen, policy)
+        missing = {MANIFEST_NAME} - seen
+        if missing:
+            raise CapsuleError(f"Missing entries: {', '.join(sorted(missing))}")
+        contents = {
+            info.filename: _read_entry(archive, info, policy.entry_limit(info.filename))
+            for info in infos
+        }
+    try:
+        manifest = load_json(contents[MANIFEST_NAME].decode("utf-8"))
+    except (UnicodeDecodeError, ValueError) as exc:
+        raise ManifestError(f"Invalid manifest JSON: {exc}") from exc
+    validate_manifest(manifest)
+    attachments = _checked_attachments(manifest, contents)
+    entity = workspace_entity(manifest)
+    if not entity.get(REFERENCED_KEY):
+        if WORKSPACE_NAME not in contents:
+            raise CapsuleError(f"Missing entries: {WORKSPACE_NAME}")
+        return Capsule(
+            manifest,
+            _checked(entity, contents[WORKSPACE_NAME]),
+            attachments=attachments,
+        )
+    if WORKSPACE_NAME in contents:
+        raise CapsuleError(
+            f"Unexpected entry for a referenced workspace: {WORKSPACE_NAME}"
+        )
+    ref = entity["@id"]
+    if resolve_workspace is None:
+        return Capsule(manifest, None, ref, attachments)
+    workspace = resolve_workspace(ref)
+    if len(workspace) > policy.max_workspace_bytes:
+        raise CapsuleError(f"Referenced workspace too large: {ref}")
+    return Capsule(manifest, _checked(entity, bytes(workspace)), ref, attachments)
+
+
+def _checked_attachments(
+    manifest: dict[str, Any], contents: Mapping[str, bytes]
+) -> dict[str, bytes]:
+    """Return the declared attachments, each matching its size and SHA-256.
+
+    Raises:
+        CapsuleError: If an attachment is missing, undeclared or altered.
+    """
+    declared = {entity["@id"]: entity for entity in attachment_entities(manifest)}
+    if len(declared) > MAX_ATTACHMENTS:
+        raise CapsuleError(f"Too many attachments (at most {MAX_ATTACHMENTS})")
+    present = {name for name in contents if name.startswith(ATTACHMENTS_DIR)}
+    if present - set(declared):
+        raise CapsuleError(
+            f"Undeclared attachments: {', '.join(sorted(present - set(declared)))}"
+        )
+    if set(declared) - present:
+        raise CapsuleError(
+            f"Missing attachments: {', '.join(sorted(set(declared) - present))}"
+        )
+    result = {}
+    for entry, entity in declared.items():
+        data = contents[entry]
+        if entity["sha256"] != sha256_digest(data).split(":", 1)[1] or entity[
+            "contentSize"
+        ] != str(len(data)):
+            raise CapsuleError(f"The attachment does not match the manifest: {entry}")
+        result[attachment_name(entity)] = data
+    return result
+
+
+def _checked(entity: dict[str, Any], workspace: bytes) -> bytes:
+    """Return *workspace* if its SHA-256 and size match the manifest."""
+    if entity["sha256"] != sha256_digest(workspace).split(":", 1)[1] or entity[
+        "contentSize"
+    ] != str(len(workspace)):
+        raise CapsuleError(
+            f"The workspace does not match the manifest: {entity['@id']}"
+        )
+    return workspace

datalab_capsule/calls.py

@@ -0,0 +1,155 @@
+# Copyright (c) DataLab Platform Developers, BSD 3-Clause license, see LICENSE file.
+
+"""Operation calls: the neutral description of one function call.
+
+A call names an operation (identifier and contract version), its parameters as
+plain JSON values and one binding per input role. Repeated roles appear once per
+input, in call order. A call whose ``operation`` is ``None`` is *opaque*: it
+names no operation (a function unknown to the computation library, such as a
+plugin, or a record written without identity by an older version) and it is
+never replayed. A call that names an operation keeps it whether or not the
+contract is qualified for replay: a replay checks the qualification of that
+contract version against the local library.
+
+Binding kinds:
+
+- ``{"state_id": ...}``: a concrete state (used by activities);
+- ``{"object_uuids": [...]}``: objects whose state at execution time is unknown
+  (only in ``legacy_metadata`` activities, see :mod:`datalab_capsule.legacy`);
+- ``{"step": ..., "role": ...}``: the output of an upstream step (reserved).
+"""
+
+from __future__ import annotations
+
+import copy
+import re
+from typing import Any
+
+from datalab_capsule.integrity import ProvenanceError
+
+__all__ = [
+    "BINDING_KINDS",
+    "OPERATION_ID_PATTERN",
+    "CallError",
+    "ReservedBindingError",
+    "is_opaque",
+    "make_call",
+    "validate_call",
+]
+
+OPERATION_ID_PATTERN = re.compile(r"^[a-z0-9]+(?:[._-][a-z0-9]+)*$")
+BINDING_KINDS = ("state_id", "object_uuids", "step")
+_IMPLEMENTED_BINDINGS = ("state_id",)
+_UUID = re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$")
+
+
+class CallError(ProvenanceError, ValueError):
+    """Raised when an operation call is malformed."""
+
+
+class ReservedBindingError(CallError):
+    """Raised when a call uses a binding kind that is not implemented yet."""
+
+
+def make_call(
+    operation_id: str | None,
+    contract_version: int | None,
+    parameters: dict[str, Any] | None,
+    inputs: list[tuple[str, str]],
+) -> dict[str, Any]:
+    """Build and validate an operation call bound to states.
+
+    Args:
+        operation_id: Operation identifier, or None for an opaque call.
+        contract_version: Contract version (required with *operation_id*).
+        parameters: Encoded parameter values, or None if they could not be
+         encoded.
+        inputs: Ordered ``(role, state_id)`` pairs.
+
+    Returns:
+        JSON-compatible call.
+    """
+    operation = None
+    if operation_id is not None:
+        operation = {"id": operation_id, "contract_version": contract_version}
+    call = {
+        "operation": operation,
+        "parameters": copy.deepcopy(parameters),
+        "inputs": [
+            {"role": role, "binding": {"state_id": state_id}}
+            for role, state_id in inputs
+        ],
+    }
+    validate_call(call)
+    return call
+
+
+def is_opaque(call: dict[str, Any]) -> bool:
+    """Return True if *call* names no operation."""
+    return call.get("operation") is None
+
+
+def _check_binding(binding: Any, where: str, allow_objects: bool) -> None:
+    if not isinstance(binding, dict) or len(binding) == 0:
+        raise CallError(f"{where}: binding must be a non-empty object")
+    kinds = [kind for kind in BINDING_KINDS if kind in binding]
+    if len(kinds) != 1:
+        raise CallError(f"{where}: binding must hold exactly one of {BINDING_KINDS}")
+    kind = kinds[0]
+    if kind == "object_uuids" and allow_objects:
+        uuids = binding["object_uuids"]
+        if (
+            set(binding) != {"object_uuids"}
+            or not isinstance(uuids, list)
+            or not uuids
+            or not all(isinstance(u, str) and _UUID.match(u) for u in uuids)
+        ):
+            raise CallError(f"{where}: object binding must list object UUIDs")
+        return
+    if kind not in _IMPLEMENTED_BINDINGS:
+        raise ReservedBindingError(f"{where}: binding kind {kind!r} is reserved")
+    if set(binding) != {"state_id"} or not isinstance(binding["state_id"], str):
+        raise CallError(f"{where}: state binding must be {{'state_id': <str>}}")
+
+
+def validate_call(call: Any, *, allow_object_bindings: bool = False) -> None:
+    """Validate the structure of an operation call.
+
+    Args:
+        call: Call to validate.
+        allow_object_bindings: Accept ``object_uuids`` bindings (legacy
+         activities only).
+
+    Raises:
+        ReservedBindingError: If a reserved binding kind is used.
+        CallError: If the call is malformed.
+    """
+    if not isinstance(call, dict) or set(call) != {"operation", "parameters", "inputs"}:
+        raise CallError("A call holds exactly: operation, parameters, inputs")
+    operation = call["operation"]
+    if operation is not None:
+        if not isinstance(operation, dict) or set(operation) != {
+            "id",
+            "contract_version",
+        }:
+            raise CallError("operation must be null or {id, contract_version}")
+        if not isinstance(operation["id"], str) or not OPERATION_ID_PATTERN.match(
+            operation["id"]
+        ):
+            raise CallError(f"Invalid operation identifier: {operation['id']!r}")
+        version = operation["contract_version"]
+        if not isinstance(version, int) or isinstance(version, bool) or version < 1:
+            raise CallError("contract_version must be an integer >= 1")
+    parameters = call["parameters"]
+    if parameters is not None and not isinstance(parameters, dict):
+        raise CallError("parameters must be an object or null")
+    inputs = call["inputs"]
+    if not isinstance(inputs, list):
+        raise CallError("inputs must be a list")
+    for index, item in enumerate(inputs):
+        where = f"inputs[{index}]"
+        if not isinstance(item, dict) or set(item) != {"role", "binding"}:
+            raise CallError(f"{where}: an input holds exactly role and binding")
+        if not isinstance(item["role"], str) or not item["role"]:
+            raise CallError(f"{where}: role must be a non-empty string")
+        _check_binding(item["binding"], where, allow_object_bindings)