dockflow-artifacts 0.2.0__py3-none-any.whl

dockflow_artifacts/__init__.py

@@ -0,0 +1,17 @@
+"""Explicit artifact publication, with optional storage and orchestration adapters."""
+
+from .errors import ArtifactError, ConflictError, IntegrityError
+from .identity import fingerprint, publication_id
+from .publication import Publication, Publisher, format_path, step_path
+
+__all__ = [
+    "ArtifactError",
+    "ConflictError",
+    "IntegrityError",
+    "Publication",
+    "Publisher",
+    "format_path",
+    "fingerprint",
+    "publication_id",
+    "step_path",
+]

dockflow_artifacts/_paths.py

@@ -0,0 +1,91 @@
+"""Private path and content helpers shared by storage implementations."""
+
+from __future__ import annotations
+
+import hashlib
+import re
+import stat
+from pathlib import Path
+from typing import Any
+
+from dockflow_core.canonical import stable_json_bytes
+from dockflow_core.filesystem import hash_stream, reject_symlinks
+
+from .errors import ArtifactError, IntegrityError
+
+
+def relative(value: str, *, internal: bool = False) -> str:
+    if not isinstance(value, str) or not value or len(value.encode()) > 1024:
+        raise ArtifactError(
+            "Destination must be a nonempty relative path of at most 1024 bytes"
+        )
+    for part in value.split("/"):
+        if (
+            part in {"", ".", ".."}
+            or not re.fullmatch(r"[A-Za-z0-9_.-]+", part)
+            or part.endswith(".")
+            or (part in {".dockflow", ".dockflow-artifacts"} and not internal)
+        ):
+            raise ArtifactError(
+                "Paths require safe relative segments (letters, digits, _, - and .)"
+            )
+    return value
+
+
+def no_links(path: Path) -> None:
+    try:
+        reject_symlinks(path)
+    except ValueError:
+        raise ArtifactError("Artifact paths cannot contain symlinks") from None
+
+
+def canonical(value: Any) -> bytes:
+    try:
+        return stable_json_bytes(value)
+    except (TypeError, ValueError) as exc:
+        raise ArtifactError("Metadata must contain finite JSON values") from exc
+
+
+def file_info(path: Path) -> tuple[int, str]:
+    no_links(path)
+    before = path.stat()
+    if not stat.S_ISREG(before.st_mode):
+        raise ArtifactError("Only regular files are supported")
+    with path.open("rb") as stream:
+        result = hash_stream(stream)
+    after = path.stat()
+    if (before.st_ino, before.st_size, before.st_mtime_ns) != (
+        after.st_ino,
+        after.st_size,
+        after.st_mtime_ns,
+    ):
+        raise IntegrityError("Artifact source changed while reading")
+    return result
+
+
+def inventory(source: Path) -> tuple[str, list[dict[str, Any]], str]:
+    no_links(source)
+    if source.is_file():
+        size, digest = file_info(source)
+        return "file", [{"path": "", "size": size, "sha256": digest}], digest
+    if not source.is_dir():
+        raise ArtifactError(
+            "Artifact source must be an existing regular file or directory"
+        )
+    entries = []
+    for path in sorted(source.rglob("*")):
+        no_links(path)
+        name = relative(path.relative_to(source).as_posix())
+        if path.is_dir():
+            if not any(path.iterdir()):
+                raise ArtifactError(
+                    "Empty directories are not supported by portable publication"
+                )
+            continue
+        size, digest = file_info(path)
+        entries.append({"path": name, "size": size, "sha256": digest})
+    if not entries:
+        raise ArtifactError(
+            "Empty directories are not supported by portable publication"
+        )
+    return "directory", entries, hashlib.sha256(canonical(entries)).hexdigest()

dockflow_artifacts/_storage.py

@@ -0,0 +1,239 @@
+"""Immutable conditional storage primitives, with lazy optional S3 imports."""
+
+from __future__ import annotations
+
+import os
+import tempfile
+from functools import cached_property
+from pathlib import Path
+from typing import Any, BinaryIO
+from urllib.parse import unquote, urlsplit
+
+from dockflow_core.filesystem import fsync_ancestors, hash_stream
+
+from ._paths import file_info, no_links, relative
+from .errors import ArtifactError, IntegrityError
+
+
+class FileStore:
+    def __init__(self, root: Path):
+        self.root = root.absolute()
+        no_links(self.root)
+        self.uri = self.root.as_uri()
+
+    def path(self, key: str) -> Path:
+        path = self.root / relative(key, internal=True)
+        no_links(path)
+        return path
+
+    def get(self, key: str) -> bytes | None:
+        path = self.path(key)
+        if not path.exists():
+            return None
+        if not path.is_file() or path.stat().st_size > 16 * 1024 * 1024:
+            raise IntegrityError("Invalid or oversized publication record")
+        return path.read_bytes()
+
+    def has_prefix(self, prefix: str) -> bool:
+        return self.path(prefix).exists()
+
+    def sync(self, key: str) -> None:
+        path = self.path(key)
+        descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0))
+        try:
+            os.fsync(descriptor)
+        finally:
+            os.close(descriptor)
+        fsync_ancestors(path.parent)
+
+    def put(self, key: str, body: BinaryIO) -> bool:
+        path = self.path(key)
+        path.parent.mkdir(parents=True, exist_ok=True)
+        no_links(path)
+        staging = self.path(".dockflow/staging")
+        staging.mkdir(parents=True, exist_ok=True)
+        no_links(staging)
+        descriptor, temporary = tempfile.mkstemp(prefix=".publication-", dir=staging)
+        try:
+            with os.fdopen(descriptor, "wb") as output:
+                while chunk := body.read(1024 * 1024):
+                    output.write(chunk)
+                output.flush()
+                os.fsync(output.fileno())
+            try:
+                os.link(temporary, path)
+            except FileExistsError:
+                # An uncertain previous write or an adopted identical payload
+                # still needs a durable file and directory boundary.
+                self.sync(key)
+                return False
+            fsync_ancestors(path.parent)
+            return True
+        finally:
+            Path(temporary).unlink(missing_ok=True)
+
+    def info(self, key: str) -> tuple[int, str]:
+        try:
+            return file_info(self.path(key))
+        except FileNotFoundError as exc:
+            raise IntegrityError("Published payload is missing") from exc
+
+    def files(self, prefix: str) -> list[str]:
+        root = self.path(prefix)
+        if not root.is_dir():
+            raise IntegrityError("Published directory is missing")
+        result = []
+        for path in root.rglob("*"):
+            no_links(path)
+            if path.is_dir():
+                continue
+            if not path.is_file():
+                raise IntegrityError("Published tree contains a nonregular entry")
+            result.append(path.relative_to(self.root).as_posix())
+        return sorted(result)
+
+
+class S3Store:
+    def __init__(self, root: str, options: dict[str, Any]):
+        parsed = urlsplit(root)
+        self.bucket = parsed.netloc
+        self.prefix = parsed.path.strip("/")
+        if self.prefix:
+            relative(self.prefix)
+        self.uri = root.rstrip("/")
+        self.options = dict(options)
+
+    @cached_property
+    def client(self) -> Any:
+        # Client creation can consult network credential providers. Defer it
+        # until an explicit publication/verification operation reaches storage.
+        try:
+            import boto3
+            import certifi
+            from botocore.config import Config
+            from botocore.session import get_session
+        except ImportError as exc:
+            raise ArtifactError(
+                "Install dockflow-artifacts[s3] to use object storage"
+            ) from exc
+        options = dict(self.options)
+        verify = options.get("verify")
+        if verify is None or verify is True:
+            # Use the maintained bundle; botocore vendors a separate older copy.
+            options["verify"] = (
+                get_session().get_config_variable("ca_bundle")
+                if verify is None
+                else None
+            ) or certifi.where()
+        return boto3.client(
+            "s3", config=Config(s3={"addressing_style": "path"}), **options
+        )
+
+    def key(self, key: str) -> str:
+        return "/".join(filter(None, (self.prefix, relative(key, internal=True))))
+
+    @staticmethod
+    def code(error: Exception) -> str:
+        return str(
+            getattr(error, "response", {}).get("Error", {}).get("Code", "unknown")
+        )
+
+    def get(self, key: str) -> bytes | None:
+        try:
+            response = self.client.get_object(Bucket=self.bucket, Key=self.key(key))
+            with response["Body"] as stream:
+                result = stream.read(16 * 1024 * 1024 + 1)
+            if len(result) > 16 * 1024 * 1024:
+                raise IntegrityError("Oversized publication record")
+            return result
+        except ArtifactError:
+            raise
+        except Exception as exc:
+            if self.code(exc) in {"NoSuchKey", "404"}:
+                return None
+            raise ArtifactError(
+                "Cannot read publication record from object storage"
+            ) from None
+
+    def put(self, key: str, body: BinaryIO) -> bool:
+        body.seek(0, os.SEEK_END)
+        size = body.tell()
+        body.seek(0)
+        if size > 5 * 1024**3:
+            raise ArtifactError("Conditional S3 publication supports files up to 5 GiB")
+        try:
+            self.client.put_object(
+                Bucket=self.bucket,
+                Key=self.key(key),
+                Body=body,
+                ContentLength=size,
+                IfNoneMatch="*",
+            )
+            return True
+        except ArtifactError:
+            raise
+        except Exception as exc:
+            if self.code(exc) in {"PreconditionFailed", "412"}:
+                return False
+            raise ArtifactError(
+                "Conditional object creation failed; verify storage access and If-None-Match support"
+            ) from None
+
+    def info(self, key: str) -> tuple[int, str]:
+        try:
+            response = self.client.get_object(Bucket=self.bucket, Key=self.key(key))
+            with response["Body"] as stream:
+                return hash_stream(stream)
+        except ArtifactError:
+            raise
+        except Exception:
+            raise IntegrityError("Cannot verify published object contents") from None
+
+    def files(self, prefix: str) -> list[str]:
+        try:
+            pages = self.client.get_paginator("list_objects_v2").paginate(
+                Bucket=self.bucket, Prefix=self.key(prefix) + "/"
+            )
+            offset = len(self.prefix) + 1 if self.prefix else 0
+            return sorted(
+                item["Key"][offset:]
+                for page in pages
+                for item in page.get("Contents", [])
+            )
+        except ArtifactError:
+            raise
+        except Exception:
+            raise ArtifactError("Cannot inventory published objects") from None
+
+    def has_prefix(self, prefix: str) -> bool:
+        return bool(self.files(prefix))
+
+
+def make_store(root: str | Path, options: dict[str, Any]) -> FileStore | S3Store:
+    text = str(root)
+    parsed = urlsplit(text)
+    if parsed.username or parsed.password or parsed.query or parsed.fragment:
+        raise ArtifactError(
+            "Artifact roots cannot contain credentials, queries or fragments"
+        )
+    if parsed.scheme in {"s3", "minio"}:
+        if not parsed.netloc or ":" in parsed.netloc or "%" in text:
+            raise ArtifactError(
+                "Object storage requires a bucket and an optional safe prefix"
+            )
+        return S3Store(text, options)
+    if options:
+        raise ArtifactError("S3 configuration cannot be used with a filesystem root")
+    if parsed.scheme == "file":
+        if parsed.netloc not in {"", "localhost"}:
+            raise ArtifactError("File URIs must refer to the local filesystem")
+        path = Path(unquote(parsed.path))
+    elif not parsed.scheme:
+        path = Path(text)
+    else:
+        raise ArtifactError(
+            "Supported roots are absolute paths, file://, s3:// and minio://"
+        )
+    if not path.is_absolute():
+        raise ArtifactError("Filesystem publication roots must be absolute")
+    return FileStore(path)

dockflow_artifacts/errors.py

@@ -0,0 +1,13 @@
+"""Public exceptions; storage clients must not expose configuration secrets."""
+
+
+class ArtifactError(ValueError):
+    """Invalid input, unsupported storage or a failed storage operation."""
+
+
+class ConflictError(ArtifactError):
+    """The destination is reserved for another immutable publication."""
+
+
+class IntegrityError(ArtifactError):
+    """Source or published contents do not match their recorded inventory."""

dockflow_artifacts/identity.py

@@ -0,0 +1,61 @@
+"""Deterministic publication identity, independent of orchestrator cache records."""
+
+from __future__ import annotations
+
+import hashlib
+from pathlib import Path
+from typing import Any
+
+from dockflow_core.canonical import canonical_tree
+
+from ._paths import canonical, inventory
+from .errors import ArtifactError
+
+PUBLICATION_ID_FORMAT = "dockflow-publication-id-v1"
+
+
+def _typed(value: Any) -> list[Any]:
+    """Retain this publication format's typed object keys and strict input types."""
+    try:
+        return canonical_tree(value, typed_keys=True, strict_types=True)
+    except ValueError as exc:
+        raise ArtifactError(str(exc)) from None
+
+
+def publication_id(
+    *,
+    component_name: str,
+    component_version: str,
+    parameters: dict[str, Any],
+    inputs: dict[str, Any] | None = None,
+) -> str:
+    """Hash a component version and its complete declared invocation inputs.
+
+    Parameters must include resolved defaults and any execution context. Inputs
+    identify named artifacts by immutable fingerprint/version and any metadata
+    observed by the component. Nothing is inferred from paths or the environment.
+    This identifier does not look up, enable or override an execution cache.
+    """
+    if any(
+        not isinstance(value, str) or not value
+        for value in (component_name, component_version)
+    ):
+        raise ArtifactError("Component name and version must be nonempty strings")
+    if type(parameters) is not dict or (
+        inputs is not None and type(inputs) is not dict
+    ):
+        raise ArtifactError("Parameters and input identities must be JSON objects")
+    envelope = {
+        "format": PUBLICATION_ID_FORMAT,
+        "component_name": component_name,
+        "component_version": component_version,
+        "parameters": parameters,
+        "inputs": {} if inputs is None else inputs,
+    }
+    return hashlib.sha256(canonical(_typed(envelope))).hexdigest()
+
+
+def fingerprint(source: str | Path) -> dict[str, str]:
+    """Fingerprint a local file/tree explicitly, ignoring its temporary location."""
+    kind, _, digest = inventory(Path(source).absolute())
+    return {"algorithm": "sha256", "kind": kind, "value": digest}

dockflow_artifacts/kfp.py

@@ -0,0 +1,48 @@
+"""Optional authoring helper; emits only the standard KFP importer contract."""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+
+def importer_component(
+    artifact_class: Any,
+    *,
+    metadata: dict[str, Any] | None = None,
+    reimport: bool = False,
+    name: str | None = None,
+) -> Any:
+    """Register a published URI as a typed artifact, retaining normal KFP lineage.
+
+    Call this factory outside the parent pipeline definition. Its result is a
+    callable graph component accepting ``uri`` as a string or parameter channel.
+    Metadata must be static JSON. Pass the resulting task.output to a standard
+    downstream artifact input. The sub-pipeline exposes the URI
+    as a parent input, which the native KFP 2.17 importer can resolve.
+    The caller owns credentials, payload publication and retention.
+    """
+    try:
+        from kfp import dsl
+    except ImportError as exc:
+        raise ImportError(
+            "Install dockflow-artifacts[kfp] to author KFP pipelines"
+        ) from exc
+    try:
+        frozen_metadata = json.loads(json.dumps(metadata, allow_nan=False))
+    except (TypeError, ValueError) as exc:
+        raise ValueError("Importer metadata must be static JSON values") from exc
+    if frozen_metadata is not None and not isinstance(frozen_metadata, dict):
+        raise ValueError("Importer metadata must be a JSON object")
+
+    def register(uri):
+        return dsl.importer(
+            artifact_uri=uri,
+            artifact_class=artifact_class,
+            metadata=frozen_metadata,
+            reimport=reimport,
+        ).output
+
+    # KFP inspects concrete annotations when creating the graph component.
+    register.__annotations__ = {"uri": str, "return": artifact_class}
+    return dsl.pipeline(name=name or "import-published-artifact")(register)

dockflow_artifacts/persistence.py

@@ -0,0 +1,147 @@
+"""Immutable publication records in the unified Dockflow database.
+
+Upload bytes outside SQL transactions. Reserve before uploading, and record
+completion only after verification. Identical retries resolve uncertain commits.
+"""
+
+from __future__ import annotations
+
+import contextlib
+
+from dockflow_core import Database, LockScope, Schema
+from dockflow_core.errors import DockflowError
+
+ARTIFACT_TABLE = "dockflow_artifact_records"
+ARTIFACT_SCHEMA = Schema(
+    name="artifacts",
+    version=1,
+    statements=(
+        "CREATE TABLE dockflow_artifact_records (project_id TEXT NOT NULL, storage_id TEXT NOT NULL, namespace TEXT NOT NULL, record_key TEXT NOT NULL, value TEXT NOT NULL, PRIMARY KEY(project_id,storage_id,namespace,record_key))",
+    ),
+    tables=frozenset({ARTIFACT_TABLE}),
+)
+
+
+def publication_scope(adapter: Database, project_id: str) -> LockScope:
+    """Shared with callers that coordinate publication and offline maintenance."""
+    return LockScope(adapter, "dockflow-project", project_id)
+
+
+def initialize_schema(adapter: Database, connection) -> None:
+    adapter.initialize_schema(connection, ARTIFACT_SCHEMA)
+
+
+def has_project_records(adapter: Database, connection, project_id: str) -> bool:
+    return (
+        adapter.execute(
+            connection,
+            "SELECT 1 FROM dockflow_artifact_records WHERE project_id=? LIMIT 1",
+            (project_id,),
+        ).fetchone()
+        is not None
+    )
+
+
+def project_storage_ids(adapter: Database, connection, project_id: str) -> set[str]:
+    return {
+        row[0]
+        for row in adapter.execute(
+            connection,
+            "SELECT DISTINCT storage_id FROM dockflow_artifact_records WHERE project_id=?",
+            (project_id,),
+        ).fetchall()
+    }
+
+
+def project_destinations(adapter: Database, connection, project_id: str) -> list[str]:
+    return [
+        row[0]
+        for row in adapter.execute(
+            connection,
+            "SELECT record_key FROM dockflow_artifact_records WHERE project_id=? AND namespace='publication-manifests'",
+            (project_id,),
+        ).fetchall()
+    ]
+
+
+def project_rows(adapter: Database, connection, project_id: str):
+    """Export publication records through a caller's consistent snapshot."""
+    cursor = adapter.execute(
+        connection,
+        "SELECT * FROM dockflow_artifact_records WHERE project_id=?",
+        (project_id,),
+    )
+    yield ARTIFACT_TABLE, [column[0] for column in cursor.description], cursor
+
+
+class ArtifactRecords:
+    """Storage-relative records; a supplied writer preserves execution fencing."""
+
+    def __init__(
+        self, adapter: Database, project_id: str, storage_id: str, *, writer=None
+    ):
+        self.database = adapter
+        self.scope = publication_scope(adapter, project_id)
+        self.project_id = project_id
+        self.storage_id = storage_id
+        self.writer = writer
+
+    @contextlib.contextmanager
+    def transaction(self):
+        if self.writer is not None:
+            with self.writer() as connection:
+                yield connection
+            return
+        connection = None
+        try:
+            self.database.prepare()
+            connection = self.database.connect()
+            self.database.begin(connection)
+            self.scope.gate(connection)
+            self.scope.serialize(connection)
+            initialize_schema(self.database, connection)
+            yield connection
+            connection.execute("COMMIT")
+        except DockflowError:
+            raise
+        except Exception:
+            raise DockflowError(
+                "Artifact database operation failed", "UNAVAILABLE", 503
+            ) from None
+        finally:
+            if connection is not None:
+                connection.close()
+
+    def get(self, namespace: str, key: str) -> bytes | None:
+        with self.database.snapshot() as connection:
+            self.database.validate_schema(connection, ARTIFACT_SCHEMA)
+            row = self.database.execute(
+                connection,
+                "SELECT value FROM dockflow_artifact_records WHERE project_id=? AND storage_id=? AND namespace=? AND record_key=?",
+                (self.project_id, self.storage_id, namespace, key),
+            ).fetchone()
+            return row[0].encode() if row else None
+
+    def put_many(self, namespace: str, records: dict[str, bytes]) -> None:
+        with self.transaction() as connection:
+            for key, value in records.items():
+                parameters = (self.project_id, self.storage_id, namespace, key)
+                self.database.execute(
+                    connection,
+                    "INSERT INTO dockflow_artifact_records VALUES (?,?,?,?,?) ON CONFLICT DO NOTHING",
+                    (*parameters, value.decode()),
+                )
+                observed = self.database.execute(
+                    connection,
+                    "SELECT value FROM dockflow_artifact_records WHERE project_id=? AND storage_id=? AND namespace=? AND record_key=?",
+                    parameters,
+                ).fetchone()
+                if observed[0] != value.decode():
+                    raise DockflowError(
+                        "Artifact location is reserved for another publication",
+                        "CONFLICT",
+                        409,
+                    )
+
+    def put(self, namespace: str, key: str, value: bytes) -> None:
+        self.put_many(namespace, {key: value})

dockflow_artifacts/publication.py

@@ -0,0 +1,407 @@
+"""Public publication API and versioned completion records."""
+
+from __future__ import annotations
+
+import functools
+import hashlib
+import io
+import json
+import re
+import shutil
+import string
+import tempfile
+import uuid
+from dataclasses import asdict, dataclass
+from pathlib import Path
+from typing import Any
+from urllib.parse import urlsplit
+
+from dockflow_core import Database, DockflowError
+from dockflow_core.database import configured_database
+
+from ._paths import canonical, file_info, inventory, no_links, relative
+from ._storage import FileStore, make_store
+from .errors import ArtifactError, ConflictError, IntegrityError
+from .persistence import ArtifactRecords
+
+
+@dataclass(frozen=True)
+class Publication:
+    """Completed artifact address and integrity/provenance metadata."""
+
+    uri: str
+    manifest_uri: str
+    digest: str
+    kind: str
+    artifact_type: str
+    metadata: dict[str, Any]
+
+    def to_dict(self) -> dict[str, Any]:
+        """Return a detached JSON-compatible receipt, without credentials."""
+        return asdict(self)
+
+
+def format_path(template: str, **values: str | int) -> str:
+    """Expand simple named fields; each value must be a single safe path segment."""
+    for _, field, spec, conversion in string.Formatter().parse(template):
+        if field is None:
+            continue
+        if not field.isidentifier() or spec or conversion or field not in values:
+            raise ArtifactError("Path templates require provided, simple named fields")
+        value = str(values[field])
+        if "/" in relative(value):
+            raise ArtifactError("Template values must each be a single path segment")
+    return relative(template.format(**values))
+
+
+def step_path(
+    filename: str,
+    *,
+    step_number: int,
+    component_name: str,
+    pipeline_name: str,
+    publication_id: str,
+    step_digits: int,
+    exploration_id: str = "global",
+) -> str:
+    """Name an output by context, padded step and deterministic publication ID.
+
+    The pipeline supplies execution order; the component wrapper supplies its
+    name and filename. This helper does not inspect a graph or allocate counters.
+    """
+    if type(step_number) is not int or step_number < 1:
+        raise ArtifactError("Step number must be a positive integer")
+    if type(step_digits) is not int or not 1 <= step_digits <= 12:
+        raise ArtifactError("Step digits must be an integer from 1 to 12")
+    if not isinstance(publication_id, str) or not re.fullmatch(
+        r"[a-f0-9]{64}", publication_id
+    ):
+        raise ArtifactError("Publication ID must be a full lowercase SHA-256 digest")
+    for value in (filename, component_name, exploration_id, pipeline_name):
+        if not isinstance(value, str) or "/" in relative(value):
+            raise ArtifactError("Step path values must each be a single path segment")
+    return format_path(
+        "{exploration}/{pipeline}/step_{number}__{component}__{identity}/{filename}",
+        exploration=exploration_id,
+        pipeline=pipeline_name,
+        number=str(step_number).zfill(step_digits),
+        component=component_name,
+        identity=publication_id,
+        filename=filename,
+    )
+
+
+def _publication_errors(method):
+    @functools.wraps(method)
+    def call(*args, **kwargs):
+        try:
+            return method(*args, **kwargs)
+        except DockflowError as error:
+            exception = ConflictError if error.code == "CONFLICT" else ArtifactError
+            raise exception(error.message) from None
+        except OSError:
+            raise ArtifactError(
+                "Artifact publication I/O failed; check storage access and durability"
+            ) from None
+
+    return call
+
+
+class Publisher:
+    """Publish immutable local contents below an explicitly configured storage root."""
+
+    @_publication_errors
+    def __init__(
+        self,
+        root: str | Path,
+        *,
+        s3: dict[str, Any] | None = None,
+        database: dict | Database | None = None,
+        project_id: str = "default",
+    ):
+        if not isinstance(project_id, str) or not re.fullmatch(
+            r"[A-Za-z0-9][A-Za-z0-9_.-]{0,127}", project_id
+        ):
+            raise ArtifactError(
+                "Project ID must contain safe letters, digits, _, - or ."
+            )
+        options = dict(s3 or {})
+        allowed = {
+            "endpoint_url",
+            "region_name",
+            "aws_access_key_id",
+            "aws_secret_access_key",
+            "aws_session_token",
+            "verify",
+        }
+        if set(options) - allowed:
+            raise ArtifactError("Unknown S3 configuration field")
+        for key, value in options.items():
+            if key == "verify":
+                if not isinstance(value, bool | str):
+                    raise ArtifactError(
+                        "S3 verify must be a boolean or certificate path"
+                    )
+            elif not isinstance(value, str) or not value:
+                raise ArtifactError("S3 configuration fields must be nonempty strings")
+        if "endpoint_url" in options:
+            endpoint = urlsplit(options["endpoint_url"])
+            if (
+                endpoint.scheme not in {"http", "https"}
+                or not endpoint.hostname
+                or endpoint.username
+                or endpoint.password
+                or endpoint.query
+                or endpoint.fragment
+            ):
+                raise ArtifactError(
+                    "S3 endpoint must be HTTP(S) without credentials or query"
+                )
+        self._store = make_store(root, options)
+        if database is None:
+            if not isinstance(self._store, FileStore):
+                raise ArtifactError(
+                    "Remote artifact storage requires explicit database configuration"
+                )
+            database = {
+                "backend": "sqlite",
+                "database": str(self._store.root.parent / ".dockflow/state.sqlite"),
+            }
+        self.database = configured_database(database)
+        if (
+            isinstance(self._store, FileStore)
+            and self.database.backend == "sqlite"
+            and Path(self.database.options["database"]).is_relative_to(self._store.root)
+        ):
+            raise ArtifactError("SQLite database must be outside the artifact root")
+        self.project_id = project_id
+
+    @classmethod
+    def from_config(cls, path: str | Path) -> Publisher:
+        """Read schema-version 2 JSON configuration explicitly at invocation time."""
+        raw = Path(path).read_bytes()
+        if len(raw) > 1024 * 1024:
+            raise ArtifactError("Publication configuration is too large")
+        try:
+            config = json.loads(raw)
+        except (ValueError, UnicodeError):
+            raise ArtifactError(
+                "Publication configuration must be valid JSON"
+            ) from None
+        if (
+            not isinstance(config, dict)
+            or set(config) - {"schema_version", "root", "s3", "database", "project_id"}
+            or type(config.get("schema_version")) is not int
+            or config["schema_version"] != 2
+            or not isinstance(config.get("root"), str)
+            or ("s3" in config and not isinstance(config["s3"], dict))
+        ):
+            raise ArtifactError(
+                "Expected schema_version=2, root, database and optional s3/project_id configuration"
+            )
+        options = config.get("database")
+        if (
+            isinstance(options, dict)
+            and options.get("backend") == "sqlite"
+            and isinstance(options.get("database"), str)
+        ):
+            options = {
+                **options,
+                "database": str(
+                    (Path(path).absolute().parent / options["database"]).absolute()
+                ),
+            }
+        return cls(
+            config["root"],
+            s3=config.get("s3"),
+            database=options,
+            project_id=config.get("project_id", "default"),
+        )
+
+    def _records(self, *, create: bool) -> ArtifactRecords:
+        marker = ".dockflow/storage.json"
+        raw = self._store.get(marker)
+        if raw is None and create:
+            self._store.put(
+                marker,
+                io.BytesIO(
+                    canonical(
+                        {
+                            "format": "dockflow-artifact-store-v1",
+                            "identity": str(uuid.uuid4()),
+                        }
+                    )
+                ),
+            )
+            raw = self._store.get(marker)
+        try:
+            value = json.loads(raw or b"null")
+            if value["format"] != "dockflow-artifact-store-v1":
+                raise ValueError
+            identity = str(uuid.UUID(value["identity"]))
+        except (ValueError, TypeError, KeyError, AttributeError):
+            raise IntegrityError(
+                "Artifact storage identity is missing or invalid"
+            ) from None
+        if create and isinstance(self._store, FileStore):
+            self._store.sync(marker)
+        return ArtifactRecords(self.database, self.project_id, identity)
+
+    def _receipt(self, manifest: dict[str, Any]) -> Publication:
+        base = self._store.uri
+        return Publication(
+            uri=base + "/" + manifest["destination"],
+            manifest_uri=f"dockflow://{self.project_id}/publications/{self._records(create=False).storage_id}/{hashlib.sha256(manifest['destination'].encode()).hexdigest()}",
+            digest=manifest["digest"],
+            kind=manifest["kind"],
+            artifact_type=manifest["artifact_type"],
+            metadata=json.loads(canonical(manifest["metadata"])),
+        )
+
+    def _check_layout(self) -> None:
+        if self._store.has_prefix(".dockflow-artifacts") or self._store.has_prefix(
+            ".dockflow/dockflow-artifacts"
+        ):
+            raise ArtifactError(
+                "Legacy .dockflow-artifacts records found; use a fresh publication "
+                "root or reset disposable artifacts before using the new layout"
+            )
+
+    def _verify_contents(self, manifest: dict[str, Any]) -> None:
+        destination = relative(manifest["destination"])
+        expected = []
+        for item in manifest["entries"]:
+            suffix = relative(item["path"]) if item["path"] else ""
+            key = destination + ("/" + suffix if suffix else "")
+            expected.append(key)
+            if self._store.info(key) != (item["size"], item["sha256"]):
+                raise IntegrityError(
+                    "Published contents do not match their recorded SHA-256"
+                )
+        if manifest["kind"] == "directory" and self._store.files(destination) != sorted(
+            expected
+        ):
+            raise IntegrityError("Published directory inventory changed")
+
+    @_publication_errors
+    def verify(self, destination: str) -> Publication:
+        """Verify all payload bytes of a completed publication and return its receipt."""
+        destination = relative(destination)
+        self._check_layout()
+        raw = self._records(create=False).get("publication-manifests", destination)
+        if raw is None:
+            raise IntegrityError("Publication has no completion manifest")
+        try:
+            manifest = json.loads(raw)
+            if (
+                manifest["schema_version"] != 1
+                or manifest["destination"] != destination
+                or manifest["kind"] not in {"file", "directory"}
+                or not manifest["entries"]
+                or not re.fullmatch(r"[a-f0-9]{64}", manifest["digest"])
+            ):
+                raise ValueError
+            for entry in manifest["entries"]:
+                if type(entry["size"]) is not int or entry["size"] < 0:
+                    raise ValueError
+                if not re.fullmatch(r"[a-f0-9]{64}", entry["sha256"]):
+                    raise ValueError
+            if manifest["kind"] == "file":
+                if (
+                    len(manifest["entries"]) != 1
+                    or manifest["entries"][0]["path"] != ""
+                ):
+                    raise ValueError
+                digest = manifest["entries"][0]["sha256"]
+            else:
+                if any(not entry["path"] for entry in manifest["entries"]):
+                    raise ValueError
+                digest = hashlib.sha256(canonical(manifest["entries"])).hexdigest()
+            if digest != manifest["digest"]:
+                raise ValueError
+            self._verify_contents(manifest)
+            return self._receipt(manifest)
+        except (KeyError, TypeError, ValueError) as exc:
+            if isinstance(exc, ArtifactError):
+                raise
+            raise IntegrityError("Invalid publication manifest") from None
+
+    @_publication_errors
+    def publish(
+        self,
+        source: str | Path,
+        destination: str,
+        *,
+        artifact_type: str = "system.Artifact",
+        metadata: dict[str, Any] | None = None,
+    ) -> Publication:
+        """Publish once, resume an identical incomplete write, or verify an identical result."""
+        destination = relative(destination)
+        self._check_layout()
+        source = Path(source).absolute()
+        no_links(source)
+        if isinstance(self._store, FileStore):
+            target = self._store.path(destination)
+            if source == target or source in target.parents or target in source.parents:
+                raise ArtifactError("Source and destination cannot overlap")
+        if not isinstance(artifact_type, str) or not artifact_type:
+            raise ArtifactError("Artifact type must be a nonempty schema title")
+        if metadata is not None and not isinstance(metadata, dict):
+            raise ArtifactError("Artifact metadata must be a JSON object")
+        kind, entries, digest = inventory(source)
+        manifest = {
+            "schema_version": 1,
+            "destination": destination,
+            "kind": kind,
+            "entries": entries,
+            "digest": digest,
+            "artifact_type": artifact_type,
+            "metadata": json.loads(canonical(metadata or {})),
+        }
+        encoded = canonical(manifest)
+        if len(encoded) > 16 * 1024 * 1024:
+            raise ArtifactError("Publication inventory exceeds the 16 MiB record limit")
+        # The whole namespace reservation is atomic; no SQL transaction spans uploads.
+        records = self._records(create=True)
+        parts = destination.split("/")
+        claims = {
+            "/".join(parts[:index]): canonical(
+                {
+                    "schema_version": 1,
+                    "kind": "namespace",
+                    "path": "/".join(parts[:index]),
+                }
+            )
+            for index in range(1, len(parts))
+        }
+        claims[destination] = encoded
+        records.put_many("publication-claims", claims)
+        completed = records.get("publication-manifests", destination)
+        if completed is not None:
+            if completed != encoded:
+                raise IntegrityError(
+                    "Completion manifest differs from the reserved publication"
+                )
+            return self.verify(destination)
+        for entry in entries:
+            path = source / entry["path"] if kind == "directory" else source
+            key = destination + ("/" + entry["path"] if entry["path"] else "")
+            no_links(path)
+            # Snapshot one file at a time; conditional uploads never stream from
+            # a mutable producer file or buffer a whole model in memory.
+            with tempfile.TemporaryDirectory(prefix="dockflow-artifacts-") as temporary:
+                staged = Path(temporary) / "content"
+                shutil.copyfile(path, staged, follow_symlinks=False)
+                if file_info(staged) != (entry["size"], entry["sha256"]):
+                    raise IntegrityError("Source changed during publication")
+                with staged.open("rb") as stream:
+                    self._store.put(key, stream)
+                if self._store.info(key) != (entry["size"], entry["sha256"]):
+                    raise IntegrityError(
+                        "Destination has different bytes; nothing was overwritten"
+                    )
+        if inventory(source) != (kind, entries, digest):
+            raise IntegrityError("Source changed during publication")
+        self._verify_contents(manifest)
+        records.put("publication-manifests", destination, encoded)
+        return self._receipt(manifest)

dockflow_artifacts/py.typed

@@ -0,0 +1 @@
+

dockflow_artifacts-0.2.0.dist-info/METADATA

@@ -0,0 +1,91 @@
+Metadata-Version: 2.4
+Name: dockflow-artifacts
+Version: 0.2.0
+Summary: Reusable immutable artifact publication and orchestration integration
+Author: Dockflow contributors
+License-Expression: MIT
+License-File: LICENSE
+Classifier: Programming Language :: Python :: 3
+Classifier: Typing :: Typed
+Requires-Dist: dockflow-core==0.2.0
+Requires-Dist: kfp>=2.17.0,<2.18 ; extra == 'kfp'
+Requires-Dist: certifi>=2026.10.19,<2027 ; extra == 'kfp'
+Requires-Dist: dockflow-core[postgresql]==0.2.0 ; extra == 'postgresql'
+Requires-Dist: boto3>=1.40,<2 ; extra == 's3'
+Requires-Dist: certifi>=2026.10.19,<2027 ; extra == 's3'
+Requires-Python: >=3.12
+Project-URL: Homepage, https://github.com/dockflow-project/dockflow
+Project-URL: Documentation, https://github.com/dockflow-project/dockflow/blob/main/docs/artifact-library/README.md
+Project-URL: Issues, https://github.com/dockflow-project/dockflow/issues
+Project-URL: Changelog, https://github.com/dockflow-project/dockflow/blob/main/CHANGELOG.md
+Provides-Extra: kfp
+Provides-Extra: postgresql
+Provides-Extra: s3
+Description-Content-Type: text/markdown
+
+# Dockflow Artifacts
+
+A Python library for immutable artifact publication at caller-chosen paths,
+content verification, and optional orchestration integration. Its first adapters
+support filesystems, S3-compatible stores and KFP typed importers.
+
+Python 3.12–3.13. MIT licensed. Developed with AI assistance.
+
+Install with Python 3.12 or newer:
+
+```bash
+pip install dockflow-artifacts==0.2.0
+# Optional S3 integration:
+pip install 'dockflow-artifacts[s3]==0.2.0'
+```
+
+For the optional `kfp` extra, follow the installation guide's explicit urllib3
+override. This library can be installed without the Dockflow CLI or service.
+
+```python
+from pathlib import Path
+from dockflow_artifacts import Publisher, format_path
+
+store = Publisher(Path("/absolute/path/to/artifacts"))
+result = store.publish(
+    "model.json",
+    format_path("models/{experiment}/{version}/model.json", experiment="demo", version="v1"),
+    artifact_type="system.Model",
+    metadata={"algorithm": "example"},
+)
+print(result.uri)
+store.verify("models/demo/v1/model.json")
+```
+
+Publication bookkeeping uses SQLite/PostgreSQL through the shared
+`dockflow-core` package. Filesystem roots default to `.dockflow/state.sqlite`
+under their parent; remote roots require explicit database configuration.
+Pass Pipelines’ database and project ID to share its state. Install the
+`postgresql` extra for an external PostgreSQL database.
+
+The file lives at exactly the requested path. Different bytes or metadata cannot
+replace an existing publication. Identical retries verify/reuse it. Importing
+the library performs no configuration, network calls or filesystem writes;
+publishing and verification perform explicit I/O.
+
+Component scripts can remain independent. A wrapper runs a script and publishes
+its files; the pipeline uses `dockflow_artifacts.kfp.importer_component` to register
+the returned URI as a normal `Model` or `Dataset`. No Dockflow server is required.
+
+[Documentation](https://github.com/dockflow-project/dockflow/blob/main/docs/artifact-library/README.md) covers configuration, publication semantics,
+KFP integration and the public API. [Examples](https://github.com/dockflow-project/dockflow/blob/main/packages/artifacts/examples/) use synthetic data.
+[Contributing](https://github.com/dockflow-project/dockflow/blob/main/CONTRIBUTING.md) describes development checks.
+
+```bash
+uv run pytest packages/artifacts/tests
+uv run ruff check .
+uv run ruff format --check .
+uv run pyright
+uv build --package dockflow-artifacts --force-pep517
+```
+
+The repository-root `VERSION` governs the shared Dockflow application version;
+package metadata is synchronized by the shared tooling.
+The library is a member of the existing monorepo workspace, composition and
+lifecycle. Macrostates baselines remain separate. The root `uv.lock` pins contributor tools,
+not downstream users. Run development commands from the repository root.

dockflow_artifacts-0.2.0.dist-info/RECORD

@@ -0,0 +1,13 @@
+dockflow_artifacts/__init__.py,sha256=_JzvmLa7kR8BImfxQwU6_wGt8FsdPGx5Du2Fl3aHamM,467
+dockflow_artifacts/_paths.py,sha256=w9GLlcFn_SVb_0jD3v2qrsDR6blwoAVmKw9QQ3gqCZo,3027
+dockflow_artifacts/_storage.py,sha256=DbN76o5DS4NwHHf_xHvPy0tRNnewNgBCO955IvjDBEE,8712
+dockflow_artifacts/errors.py,sha256=aJq_eXPr-cjieb8v6uzkQaiqk1WxC7-4BEmVkRcvG20,419
+dockflow_artifacts/identity.py,sha256=XLxFbav5c7JXJ9KRrYptP5raH-eEfgwnM-181F1EztA,2188
+dockflow_artifacts/kfp.py,sha256=tcWPzcP37VdoCKcW6H01kaJu5KIaydq_cpsIkpj2pqY,1828
+dockflow_artifacts/persistence.py,sha256=EY6cCmi5IAc6sbujTawTclFyjxhwJsRsW-Ycyl0-3mk,5439
+dockflow_artifacts/publication.py,sha256=Gzu4XlJwLdtNHYx82KC3dPNUjdk8EIwEF7eVbtWjKtI,16579
+dockflow_artifacts/py.typed,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+dockflow_artifacts-0.2.0.dist-info/licenses/LICENSE,sha256=uIXEE60dM30LHZXUm4HhF2tw0Zy02FRsqdapPi9x1tI,1078
+dockflow_artifacts-0.2.0.dist-info/WHEEL,sha256=a0ta4TK7UKMB0EbDesmi2mn0Djkly7aj_7Tu2R7SseE,81
+dockflow_artifacts-0.2.0.dist-info/METADATA,sha256=kcrjMjFX7eA5HumWXuGn5O5Ep_792_-L2Az7iLX1KSw,3987
+dockflow_artifacts-0.2.0.dist-info/RECORD,,

dockflow_artifacts-0.2.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: uv 0.11.33
+Root-Is-Purelib: true
+Tag: py3-none-any

dockflow_artifacts-0.2.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Dockflow contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.