reader-workbench 1.0.0__py3-none-any.whl

reader_workbench/workbench/records/evidence.py

@@ -0,0 +1,247 @@
+from __future__ import annotations
+
+import hashlib
+from collections.abc import Callable
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from reader_workbench.errors import RecordError
+from reader_workbench.workbench.experiments import ExperimentCatalog, ExperimentLocation
+from reader_workbench.workbench.graph import FileRef, InputRef, RecordRef, ResourceRef, SourceRecordRef
+
+from .identity import is_sha256_digest
+
+_DISCOVERY_POLICIES = frozenset(
+    {
+        "declared_file",
+        "declared_resource",
+        "plugin_discovery",
+        "record",
+        "source_record",
+    }
+)
+SourceExperimentResolver = Callable[[str], ExperimentLocation]
+
+
+@dataclass(frozen=True)
+class ArtifactEvidence:
+    relative_path: Path
+    size_bytes: int
+    content_digest: str
+
+    def __post_init__(self) -> None:
+        path = Path(self.relative_path)
+        if path.is_absolute() or not path.parts or any(part == ".." for part in path.parts):
+            raise RecordError("artifact evidence path must be relative and confined")
+        if not isinstance(self.size_bytes, int) or self.size_bytes < 0:
+            raise RecordError("artifact evidence size_bytes must be a non-negative integer")
+        if not is_sha256_digest(self.content_digest):
+            raise RecordError("artifact evidence content_digest must be a sha256 digest")
+        object.__setattr__(self, "relative_path", path)
+
+    def to_dict(self) -> dict[str, object]:
+        return {
+            "path": self.relative_path.as_posix(),
+            "size_bytes": self.size_bytes,
+            "content_digest": self.content_digest,
+        }
+
+    @classmethod
+    def from_dict(cls, payload: Any) -> ArtifactEvidence:
+        if not isinstance(payload, dict) or set(payload) != {"path", "size_bytes", "content_digest"}:
+            raise RecordError("artifact evidence must contain only path, size_bytes, and content_digest")
+        raw_path = payload.get("path")
+        if not isinstance(raw_path, str) or not raw_path.strip():
+            raise RecordError("artifact evidence path must be a non-empty string")
+        return cls(
+            relative_path=Path(raw_path),
+            size_bytes=payload.get("size_bytes"),
+            content_digest=payload.get("content_digest"),
+        )
+
+
+@dataclass(frozen=True)
+class RecordInputEvidence:
+    """Immutable evidence for one input consumed by a persisted record."""
+
+    label: str
+    ref: InputRef
+    discovery_policy: str
+    artifact: ArtifactEvidence | None = None
+    record_revision_digest: str | None = None
+
+    def __post_init__(self) -> None:
+        if not isinstance(self.label, str) or not self.label.strip():
+            raise RecordError("record input evidence label must be a non-empty string")
+        if self.discovery_policy not in _DISCOVERY_POLICIES:
+            raise RecordError(
+                "record input evidence discovery_policy must be one of " + ", ".join(sorted(_DISCOVERY_POLICIES))
+            )
+        if isinstance(self.ref, RecordRef):
+            if self.discovery_policy != "record":
+                raise RecordError("record references must use discovery_policy 'record'")
+            if self.artifact is not None:
+                raise RecordError("record references must not include artifact evidence")
+            if not is_sha256_digest(self.record_revision_digest):
+                raise RecordError("record references must include a sha256 record_revision_digest")
+            return
+        if isinstance(self.ref, SourceRecordRef):
+            if self.discovery_policy != "source_record":
+                raise RecordError("source record references must use discovery_policy 'source_record'")
+            if self.artifact is not None:
+                raise RecordError("source record references must not include artifact evidence")
+            if not is_sha256_digest(self.record_revision_digest):
+                raise RecordError("source record references must include a sha256 record_revision_digest")
+            return
+        if not isinstance(self.ref, (FileRef, ResourceRef)):
+            raise RecordError("record input evidence contains an unsupported reference")
+        if self.discovery_policy in {"record", "source_record"}:
+            raise RecordError("file and resource inputs must not use record discovery policies")
+        if self.artifact is None:
+            raise RecordError("file and resource inputs must include artifact evidence")
+        if self.record_revision_digest is not None:
+            raise RecordError("file and resource inputs must not include record_revision_digest")
+
+    def to_dict(self) -> dict[str, object]:
+        base: dict[str, object] = {
+            "label": self.label,
+            "discovery_policy": self.discovery_policy,
+        }
+        if isinstance(self.ref, RecordRef):
+            return {
+                **base,
+                "kind": "record",
+                "record": self.ref.record_id,
+                "record_revision_digest": self.record_revision_digest,
+            }
+        if isinstance(self.ref, SourceRecordRef):
+            return {
+                **base,
+                "kind": "source_record",
+                "resource": self.ref.resource_id,
+                "experiment": self.ref.experiment_id,
+                "record": self.ref.record_id,
+                "record_revision_digest": self.record_revision_digest,
+            }
+        if isinstance(self.ref, ResourceRef):
+            base.update({"kind": "resource", "resource": self.ref.resource_id})
+        else:
+            base["kind"] = "file"
+        base["artifact"] = self.artifact.to_dict() if self.artifact else None
+        return base
+
+    @classmethod
+    def from_dict(
+        cls,
+        payload: Any,
+        *,
+        experiment_root: Path,
+        source_experiment_resolver: SourceExperimentResolver | None = None,
+    ) -> RecordInputEvidence:
+        if not isinstance(payload, dict):
+            raise RecordError("record input evidence must be a JSON object")
+        kind = payload.get("kind")
+        label = payload.get("label")
+        discovery_policy = payload.get("discovery_policy")
+        if kind == "record":
+            expected = {
+                "label",
+                "kind",
+                "record",
+                "discovery_policy",
+                "record_revision_digest",
+            }
+            if set(payload) != expected:
+                raise RecordError("record input evidence has unknown or missing fields")
+            record_id = payload.get("record")
+            if not isinstance(record_id, str) or not record_id:
+                raise RecordError("record input evidence must include a non-empty record id")
+            return cls(
+                label=label,
+                ref=RecordRef(record_id=record_id),
+                discovery_policy=discovery_policy,
+                record_revision_digest=payload.get("record_revision_digest"),
+            )
+        if kind == "source_record":
+            expected = {
+                "label",
+                "kind",
+                "resource",
+                "experiment",
+                "record",
+                "discovery_policy",
+                "record_revision_digest",
+            }
+            if set(payload) != expected:
+                raise RecordError("source record input evidence has unknown or missing fields")
+            resource_id = payload.get("resource")
+            experiment_id = payload.get("experiment")
+            record_id = payload.get("record")
+            if any(not isinstance(value, str) or not value for value in (resource_id, experiment_id, record_id)):
+                raise RecordError("source record input evidence requires resource, experiment, and record identities")
+            try:
+                resolver = source_experiment_resolver
+                if resolver is None:
+                    resolver = ExperimentCatalog.from_experiment_root(experiment_root).resolve
+                location = resolver(experiment_id)
+            except Exception as exc:
+                raise RecordError(f"Could not resolve source experiment {experiment_id!r}: {exc}") from exc
+            return cls(
+                label=label,
+                ref=SourceRecordRef(
+                    resource_id=resource_id,
+                    experiment_id=location.id,
+                    record_id=record_id,
+                    experiment_root=location.root,
+                    outputs_dir=location.outputs_dir,
+                ),
+                discovery_policy=discovery_policy,
+                record_revision_digest=payload.get("record_revision_digest"),
+            )
+        if kind not in {"file", "resource"}:
+            raise RecordError(f"record input evidence has unknown kind {kind!r}")
+        expected = {"label", "kind", "discovery_policy", "artifact"}
+        if kind == "resource":
+            expected.add("resource")
+        if set(payload) != expected:
+            raise RecordError("record input evidence has unknown or missing fields")
+        artifact = ArtifactEvidence.from_dict(payload.get("artifact"))
+        resolved = experiment_root.resolve(strict=False) / artifact.relative_path
+        if kind == "resource":
+            resource_id = payload.get("resource")
+            if not isinstance(resource_id, str) or not resource_id:
+                raise RecordError("resource input evidence must include a non-empty resource id")
+            ref: InputRef = ResourceRef(resource_id=resource_id, path=resolved)
+        else:
+            ref = FileRef(path=resolved)
+        return cls(
+            label=label,
+            ref=ref,
+            discovery_policy=discovery_policy,
+            artifact=artifact,
+        )
+
+
+def capture_artifact_evidence(path: Path, *, root: Path) -> ArtifactEvidence:
+    root_resolved = root.resolve(strict=False)
+    resolved = path.resolve(strict=True)
+    try:
+        relative = resolved.relative_to(root_resolved)
+    except ValueError as exc:
+        raise RecordError(f"artifact {path} must resolve within the outputs directory {root}") from exc
+    if not resolved.is_file():
+        raise RecordError(f"artifact {path} must be a regular file")
+    return ArtifactEvidence(
+        relative_path=relative,
+        size_bytes=resolved.stat().st_size,
+        content_digest=_sha256_file(resolved),
+    )
+
+
+def _sha256_file(path: Path, *, chunk_size: int = 1024 * 1024) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as artifact:
+        for chunk in iter(lambda: artifact.read(chunk_size), b""):
+            digest.update(chunk)
+    return "sha256:" + digest.hexdigest()

reader_workbench/workbench/records/identity.py

@@ -0,0 +1,87 @@
+from __future__ import annotations
+
+import hashlib
+import json
+from dataclasses import dataclass
+from functools import cache
+from pathlib import Path
+from typing import Any
+
+from reader_workbench._version import package_version
+from reader_workbench.errors import RecordError
+
+_PACKAGED_SOURCE_SUFFIXES = frozenset({".py", ".pyi"})
+_PACKAGED_DATA_GLOBS = ("workbench/notebooks/*.marimo.py.txt",)
+_PACKAGED_DATA_NAMES = frozenset({"py.typed"})
+
+
+def digest_json(payload: Any) -> str:
+    raw = json.dumps(payload, ensure_ascii=True, sort_keys=True, separators=(",", ":"), default=str).encode()
+    return "sha256:" + hashlib.sha256(raw).hexdigest()
+
+
+def is_sha256_digest(value: object) -> bool:
+    if not isinstance(value, str) or not value.startswith("sha256:"):
+        return False
+    digest = value.removeprefix("sha256:")
+    return len(digest) == 64 and all(character in "0123456789abcdef" for character in digest)
+
+
+@dataclass(frozen=True)
+class BuildIdentity:
+    reader_version: str
+    source_digest: str
+
+    def __post_init__(self) -> None:
+        if not isinstance(self.reader_version, str) or not self.reader_version.strip():
+            raise RecordError("build_identity.reader_version must be a non-empty string")
+        if not is_sha256_digest(self.source_digest):
+            raise RecordError("build_identity.source_digest must be a sha256 digest")
+
+    def to_dict(self) -> dict[str, str]:
+        return {
+            "reader_version": self.reader_version,
+            "source_digest": self.source_digest,
+        }
+
+    @classmethod
+    def from_dict(cls, payload: Any) -> BuildIdentity:
+        if not isinstance(payload, dict) or set(payload) != {"reader_version", "source_digest"}:
+            raise RecordError("build_identity must contain only reader_version and source_digest")
+        return cls(
+            reader_version=payload.get("reader_version"),
+            source_digest=payload.get("source_digest"),
+        )
+
+
+def _packaged_runtime_files(package_root: Path) -> tuple[Path, ...]:
+    files: set[Path] = set()
+    for path in package_root.rglob("*"):
+        if not path.is_file():
+            continue
+        relative = path.relative_to(package_root)
+        if relative.parts[0] == "tests":
+            continue
+        if path.suffix in _PACKAGED_SOURCE_SUFFIXES or path.name in _PACKAGED_DATA_NAMES:
+            files.add(path)
+    for pattern in _PACKAGED_DATA_GLOBS:
+        files.update(path for path in package_root.glob(pattern) if path.is_file())
+    return tuple(sorted(files, key=lambda item: item.relative_to(package_root).as_posix()))
+
+
+def _source_digest(package_root: Path) -> str:
+    digest = hashlib.sha256()
+    for path in _packaged_runtime_files(package_root):
+        relative = path.relative_to(package_root).as_posix().encode("utf-8")
+        digest.update(len(relative).to_bytes(8, "big"))
+        digest.update(relative)
+        data = path.read_bytes()
+        digest.update(len(data).to_bytes(8, "big"))
+        digest.update(data)
+    return "sha256:" + digest.hexdigest()
+
+
+@cache
+def current_build_identity() -> BuildIdentity:
+    package_root = Path(__file__).resolve().parents[2]
+    return BuildIdentity(reader_version=package_version(), source_digest=_source_digest(package_root))

reader_workbench/workbench/records/locking.py

@@ -0,0 +1,185 @@
+"""Filesystem-safe, process-shared locks for provenance publication."""
+
+from __future__ import annotations
+
+import errno
+import os
+import stat
+from collections.abc import Iterator
+from contextlib import contextmanager
+from pathlib import Path
+from typing import Any
+
+from filelock import BaseFileLock, Timeout
+
+if os.name == "posix":
+    import fcntl
+elif os.name == "nt":  # pragma: win32 cover
+    import msvcrt
+
+
+class ProvenanceFileLock(BaseFileLock):
+    """A reentrant advisory lock that never truncates an existing target.
+
+    ``BaseFileLock`` supplies timeout polling and per-thread reentrancy. This
+    implementation owns the filesystem boundary: it opens without truncation,
+    validates the opened descriptor, acquires an OS lock, and then proves that
+    the locked inode is still the single-link object named by the lock path.
+    """
+
+    def _acquire(self) -> None:
+        if os.name not in {"posix", "nt"}:  # pragma: no cover - fail closed on unsupported runtimes
+            raise NotImplementedError("Secure provenance locks require POSIX flock or Windows msvcrt locking")
+
+        descriptor = -1
+        parent_descriptor = -1
+        parent_stat: os.stat_result | None = None
+        locked = False
+        try:
+            lock_path = Path(self.lock_file)
+            if os.name == "posix":
+                parent_flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_DIRECTORY", 0)
+                parent_flags |= getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0)
+                parent_descriptor = os.open(lock_path.parent, parent_flags)
+                parent_stat = os.fstat(parent_descriptor)
+                if not stat.S_ISDIR(parent_stat.st_mode):
+                    raise OSError("Provenance lock parent must be a directory")
+                descriptor = os.open(
+                    lock_path.name,
+                    self._open_flags(),
+                    self._context.mode,
+                    dir_fd=parent_descriptor,
+                )
+                self._validate_descriptor(
+                    descriptor,
+                    lock_path=lock_path,
+                    parent_descriptor=parent_descriptor,
+                    parent_stat=parent_stat,
+                )
+            else:  # pragma: win32 cover
+                descriptor = os.open(lock_path, self._open_flags(), self._context.mode)
+                self._validate_descriptor(descriptor, lock_path=lock_path)
+
+            if not self._try_lock(descriptor):
+                return
+            locked = True
+            self._validate_descriptor(
+                descriptor,
+                lock_path=lock_path,
+                parent_descriptor=parent_descriptor if parent_descriptor >= 0 else None,
+                parent_stat=parent_stat,
+            )
+            self._context.lock_file_fd = descriptor
+            descriptor = -1
+            locked = False
+        finally:
+            if locked:
+                self._unlock(descriptor)
+            if descriptor >= 0:
+                os.close(descriptor)
+            if parent_descriptor >= 0:
+                os.close(parent_descriptor)
+
+    def _release(self) -> None:
+        descriptor = self._context.lock_file_fd
+        if descriptor is None:  # pragma: no cover - BaseFileLock guards this path
+            return
+        self._context.lock_file_fd = None
+        try:
+            self._unlock(descriptor)
+        finally:
+            os.close(descriptor)
+
+    @staticmethod
+    def _open_flags() -> int:
+        flags = os.O_RDWR | os.O_CREAT
+        flags |= getattr(os, "O_CLOEXEC", 0)
+        flags |= getattr(os, "O_NOFOLLOW", 0)
+        flags |= getattr(os, "O_NONBLOCK", 0)
+        flags |= getattr(os, "O_BINARY", 0)
+        return flags
+
+    @staticmethod
+    def _validate_descriptor(
+        descriptor: int,
+        *,
+        lock_path: Path,
+        parent_descriptor: int | None = None,
+        parent_stat: os.stat_result | None = None,
+    ) -> None:
+        opened_stat = os.fstat(descriptor)
+        if not stat.S_ISREG(opened_stat.st_mode) or opened_stat.st_nlink != 1:
+            raise OSError("Provenance lock must be a regular file with a single link")
+
+        if parent_descriptor is None:
+            named_stat = os.stat(lock_path, follow_symlinks=False)
+        else:
+            named_stat = os.stat(lock_path.name, dir_fd=parent_descriptor, follow_symlinks=False)
+            current_parent = os.stat(lock_path.parent, follow_symlinks=False)
+            if parent_stat is None or not _same_inode(parent_stat, current_parent):
+                raise OSError("Provenance lock parent changed while the lock was acquired")
+        if not stat.S_ISREG(named_stat.st_mode) or named_stat.st_nlink != 1:
+            raise OSError("Provenance lock must be a regular file with a single link")
+        if not _same_inode(opened_stat, named_stat):
+            raise OSError("Provenance lock target changed while the lock was acquired")
+
+    @staticmethod
+    def _try_lock(descriptor: int) -> bool:
+        if os.name == "posix":
+            try:
+                fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB)
+            except OSError as exc:
+                if exc.errno in {errno.EACCES, errno.EAGAIN}:
+                    return False
+                if exc.errno == errno.ENOSYS:
+                    raise NotImplementedError("The filesystem does not support provenance file locks") from exc
+                raise
+            return True
+        try:  # pragma: win32 cover
+            msvcrt.locking(descriptor, msvcrt.LK_NBLCK, 1)
+        except OSError as exc:  # pragma: win32 cover
+            if exc.errno in {errno.EACCES, errno.EAGAIN, errno.EDEADLK}:
+                return False
+            raise
+        return True
+
+    @staticmethod
+    def _unlock(descriptor: int) -> None:
+        if os.name == "posix":
+            fcntl.flock(descriptor, fcntl.LOCK_UN)
+        else:  # pragma: win32 cover
+            msvcrt.locking(descriptor, msvcrt.LK_UNLCK, 1)
+
+
+@contextmanager
+def provenance_lock_scope(
+    lock: Any,
+    *,
+    acquire_error: Exception,
+    release_error: Exception,
+    release_note: str,
+) -> Iterator[None]:
+    """Hold a provenance lock without masking a protected operation failure."""
+
+    try:
+        lock.acquire()
+    except (Timeout, OSError, NotImplementedError) as exc:
+        raise acquire_error from exc
+    body_error: BaseException | None = None
+    try:
+        yield
+    except BaseException as exc:
+        body_error = exc
+        raise
+    finally:
+        try:
+            lock.release()
+        except (Timeout, OSError, NotImplementedError) as exc:
+            if body_error is not None:
+                body_error.add_note(f"{release_note} ({type(exc).__name__}).")
+            else:
+                raise release_error from exc
+
+
+def _same_inode(left: os.stat_result, right: os.stat_result) -> bool:
+    return (left.st_dev, left.st_ino) == (right.st_dev, right.st_ino)