pyingestkit 1.0.0__py3-none-any.whl

pyingestkit/__init__.py

@@ -0,0 +1,111 @@
+"""PyIngestKit public API."""
+
+import logging as _stdlib_logging
+
+from ._version import __version__ as __version__
+from .artifacts import ArtifactURI, S3ArtifactStore, StoredArtifact
+from .contracts import DatasetContract, FieldContract
+from .core.context import RunContext
+from .core.job import Job
+from .core.pipeline import Pipeline
+from .core.result import RunResult, RunStatus, StepResult
+from .core.step import Step
+from .dataset import Dataset
+from .declarative import JobDefinition, StepDefinition, StepInvocation, job, step
+from .diff import DatasetDiff, DatasetDiffer, DiffEntry, DiffKind, DiffPolicy, SchemaDiff
+from .parsers import CsvParser, ExcelParser, JsonParser, NdjsonParser, ParquetParser
+from .profiling import DatasetProfile, DatasetProfiler, FieldProfile
+from .quality import QualityReport
+from .replay import ReplayContext, ReplayRawArtifact, ReplayResult, ReplayService
+from .runtime.runner import Runner
+from .targets import (
+    IdempotencyAction,
+    IdempotencyPolicy,
+    LoadMode,
+    PostgresTarget,
+    Target,
+    TargetCapabilities,
+    TargetLoadDecision,
+    TargetLoadExecutor,
+    TargetLoadRequest,
+    TargetLoadResult,
+    TargetLoadStatus,
+)
+from .validation import ValidationIssue, ValidationResult
+from .versioning import (
+    DatasetFingerprint,
+    DatasetFingerprinter,
+    DatasetFingerprintPolicy,
+    DatasetVersion,
+    DatasetVersionStore,
+    FilesystemDatasetVersionStore,
+    PublishedDataset,
+    S3DatasetVersionStore,
+    SnapshotCodec,
+)
+
+__all__ = [
+    "ArtifactURI",
+    "CsvParser",
+    "Dataset",
+    "DatasetContract",
+    "DatasetDiff",
+    "DatasetDiffer",
+    "DatasetFingerprint",
+    "DatasetFingerprinter",
+    "DatasetFingerprintPolicy",
+    "DatasetProfile",
+    "DatasetProfiler",
+    "DatasetVersion",
+    "DatasetVersionStore",
+    "DiffEntry",
+    "DiffKind",
+    "DiffPolicy",
+    "ExcelParser",
+    "FieldContract",
+    "FieldProfile",
+    "FilesystemDatasetVersionStore",
+    "Job",
+    "JobDefinition",
+    "JsonParser",
+    "IdempotencyAction",
+    "IdempotencyPolicy",
+    "NdjsonParser",
+    "ParquetParser",
+    "Pipeline",
+    "PublishedDataset",
+    "QualityReport",
+    "ReplayContext",
+    "ReplayRawArtifact",
+    "ReplayResult",
+    "ReplayService",
+    "RunContext",
+    "RunResult",
+    "RunStatus",
+    "Runner",
+    "S3ArtifactStore",
+    "S3DatasetVersionStore",
+    "StoredArtifact",
+    "SchemaDiff",
+    "SnapshotCodec",
+    "Step",
+    "StepDefinition",
+    "StepInvocation",
+    "StepResult",
+    "LoadMode",
+    "PostgresTarget",
+    "Target",
+    "TargetCapabilities",
+    "TargetLoadDecision",
+    "TargetLoadExecutor",
+    "TargetLoadRequest",
+    "TargetLoadResult",
+    "TargetLoadStatus",
+    "ValidationIssue",
+    "ValidationResult",
+    "job",
+    "step",
+]
+
+# Library best practice: never configure application handlers at import time.
+_stdlib_logging.getLogger(__name__).addHandler(_stdlib_logging.NullHandler())

pyingestkit/_version.py

@@ -0,0 +1 @@
+__version__ = "1.0.0"

pyingestkit/artifacts/__init__.py

@@ -0,0 +1,17 @@
+from .base import ArtifactStore
+from .factory import create_artifact_store
+from .filesystem import LocalArtifactStore
+from .raw import RawArtifact
+from .s3 import S3ArtifactStore
+from .stored import StoredArtifact
+from .uri import ArtifactURI
+
+__all__ = [
+    "ArtifactStore",
+    "ArtifactURI",
+    "LocalArtifactStore",
+    "RawArtifact",
+    "S3ArtifactStore",
+    "StoredArtifact",
+    "create_artifact_store",
+]

pyingestkit/artifacts/base.py

@@ -0,0 +1,145 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from pathlib import Path
+from typing import Any
+from uuid import UUID
+
+from pyingestkit.core.exceptions import StorageError
+from pyingestkit.provenance.hashing import sha256_bytes
+
+from .raw import RawArtifact
+from .stored import StoredArtifact
+from .uri import ArtifactURI
+
+
+class ArtifactStore(ABC):
+    """Run-artifact persistence contract.
+
+    V0.6 separates a durable storage URI from the local materialization path.
+    Existing third-party stores remain source-compatible because URI/read/materialize
+    methods have conservative local-file defaults rather than new abstract methods.
+    """
+
+    @abstractmethod
+    def prepare_run(self, job_id: str, run_id: UUID) -> Path:
+        raise NotImplementedError
+
+    @abstractmethod
+    def write_raw(
+        self,
+        job_id: str,
+        run_id: UUID,
+        *,
+        name: str,
+        data: bytes,
+        source_uri: str,
+        content_type: str | None = None,
+        resolved_url: str | None = None,
+        status_code: int | None = None,
+        etag: str | None = None,
+        last_modified: str | None = None,
+    ) -> RawArtifact:
+        raise NotImplementedError
+
+    @abstractmethod
+    def write_json(self, job_id: str, run_id: UUID, relative_path: str, payload: Any) -> Path:
+        raise NotImplementedError
+
+    @abstractmethod
+    def path_for(self, job_id: str, run_id: UUID, relative_path: str) -> Path:
+        raise NotImplementedError
+
+    def uri_for(self, job_id: str, run_id: UUID, relative_path: str) -> ArtifactURI:
+        """Return the canonical storage URI for an artifact path.
+
+        V0.5-compatible stores automatically get a ``file://`` implementation.
+        Remote stores override this without changing callers.
+        """
+
+        return ArtifactURI.from_path(self.path_for(job_id, run_id, relative_path))
+
+    def read_bytes(self, uri: ArtifactURI | str) -> bytes:
+        """Read persisted bytes by canonical URI.
+
+        The default implementation intentionally supports local ``file://`` only.
+        Remote backends opt in by overriding this method.
+        """
+
+        location = uri if isinstance(uri, ArtifactURI) else ArtifactURI(uri)
+        if not location.is_local:
+            raise StorageError(f"Artifact store cannot read remote URI scheme {location.scheme!r}")
+        path = location.as_path()
+        try:
+            return path.read_bytes()
+        except OSError as exc:
+            raise StorageError(f"Unable to read artifact URI {location}") from exc
+
+    def write_json_artifact(
+        self, job_id: str, run_id: UUID, relative_path: str, payload: Any
+    ) -> StoredArtifact:
+        """Persist JSON and return its durable URI and integrity metadata.
+
+        This is additive to the V0.5 ``write_json`` contract. Third-party stores that only
+        implement ``write_json`` automatically gain a local-file implementation.
+        """
+
+        path = self.write_json(job_id, run_id, relative_path, payload)
+        try:
+            data = path.read_bytes()
+        except OSError as exc:
+            raise StorageError(f"Unable to inspect JSON artifact at {path}") from exc
+        return StoredArtifact(
+            relative_path=relative_path,
+            path=str(path),
+            storage_uri=str(self.uri_for(job_id, run_id, relative_path)),
+            content_type="application/json",
+            size_bytes=len(data),
+            sha256=sha256_bytes(data),
+        )
+
+    def _materialize_verified(
+        self, *, local_path: Path, storage_uri: ArtifactURI, expected_sha256: str, label: str
+    ) -> Path:
+        if local_path.is_file():
+            try:
+                data = local_path.read_bytes()
+            except OSError as exc:
+                raise StorageError(f"Unable to read {label} materialization {local_path}") from exc
+        else:
+            data = self.read_bytes(storage_uri)
+            local_path.parent.mkdir(parents=True, exist_ok=True)
+            temp = local_path.with_name(f".{local_path.name}.materializing")
+            try:
+                temp.write_bytes(data)
+                temp.replace(local_path)
+            except OSError as exc:
+                temp.unlink(missing_ok=True)
+                raise StorageError(f"Unable to materialize {label} at {local_path}") from exc
+
+        actual = sha256_bytes(data)
+        if actual != expected_sha256:
+            raise StorageError(
+                f"{label} materialization SHA-256 mismatch: expected {expected_sha256}, got {actual}"
+            )
+        return local_path
+
+    def materialize_raw(self, artifact: RawArtifact) -> Path:
+        """Ensure RAW is present at its local materialization path and verify SHA-256."""
+
+        return self._materialize_verified(
+            local_path=artifact.local_path,
+            storage_uri=artifact.location_uri,
+            expected_sha256=artifact.sha256,
+            label="RAW",
+        )
+
+    def materialize_artifact(self, artifact: StoredArtifact) -> Path:
+        """Materialize a non-RAW run artifact from its durable URI and verify SHA-256."""
+
+        return self._materialize_verified(
+            local_path=artifact.local_path,
+            storage_uri=artifact.location_uri,
+            expected_sha256=artifact.sha256,
+            label="artifact",
+        )

pyingestkit/artifacts/factory.py

@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+from pyingestkit.config.models import ArtifactBackend, ArtifactConfig
+from pyingestkit.core.exceptions import ConfigurationError
+
+from .base import ArtifactStore
+from .filesystem import LocalArtifactStore
+from .s3 import S3ArtifactStore
+
+
+def create_artifact_store(config: ArtifactConfig, *, workspace: str | Path) -> ArtifactStore:
+    if config.backend is ArtifactBackend.LOCAL:
+        return LocalArtifactStore(workspace)
+
+    s3 = config.s3
+    if s3.bucket is None:
+        raise ConfigurationError("S3 artifact backend requires artifacts.s3.bucket")
+    endpoint_url = None
+    if s3.endpoint_url_env:
+        endpoint_url = os.getenv(s3.endpoint_url_env)
+    return S3ArtifactStore(
+        bucket=s3.bucket,
+        prefix=s3.prefix,
+        cache_root=s3.cache_path or workspace,
+        region_name=s3.region_name,
+        endpoint_url=endpoint_url,
+    )

pyingestkit/artifacts/filesystem.py

@@ -0,0 +1,97 @@
+from __future__ import annotations
+
+import json
+import logging
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+from uuid import UUID, uuid4
+
+from pyingestkit.core.exceptions import StorageError
+from pyingestkit.provenance.hashing import sha256_bytes
+
+from .base import ArtifactStore
+from .naming import job_parts, relative_artifact_path, safe_component
+from .raw import RawArtifact
+from .uri import ArtifactURI
+
+logger = logging.getLogger(__name__)
+
+
+class LocalArtifactStore(ArtifactStore):
+    def __init__(self, root: str | Path = ".pyingest") -> None:
+        self.root = Path(root)
+
+    def _job_parts(self, job_id: str) -> tuple[str, ...]:
+        return job_parts(job_id)
+
+    def run_root(self, job_id: str, run_id: UUID) -> Path:
+        return self.root / "runs" / Path(*self._job_parts(job_id)) / str(run_id)
+
+    def prepare_run(self, job_id: str, run_id: UUID) -> Path:
+        run_root = self.run_root(job_id, run_id)
+        for name in ("raw", "staging", "candidate", "reports"):
+            (run_root / name).mkdir(parents=True, exist_ok=True)
+        return run_root
+
+    def path_for(self, job_id: str, run_id: UUID, relative_path: str) -> Path:
+        relative = relative_artifact_path(relative_path)
+        path = self.run_root(job_id, run_id).joinpath(*relative.parts)
+        path.parent.mkdir(parents=True, exist_ok=True)
+        return path
+
+    def uri_for(self, job_id: str, run_id: UUID, relative_path: str) -> ArtifactURI:
+        relative = relative_artifact_path(relative_path)
+        path = self.run_root(job_id, run_id).joinpath(*relative.parts)
+        return ArtifactURI.from_path(path)
+
+    def write_raw(
+        self,
+        job_id: str,
+        run_id: UUID,
+        *,
+        name: str,
+        data: bytes,
+        source_uri: str,
+        content_type: str | None = None,
+        resolved_url: str | None = None,
+        status_code: int | None = None,
+        etag: str | None = None,
+        last_modified: str | None = None,
+    ) -> RawArtifact:
+        self.prepare_run(job_id, run_id)
+        digest = sha256_bytes(data)
+        relative_path = f"raw/{safe_component(name)}"
+        path = self.path_for(job_id, run_id, relative_path)
+        try:
+            with path.open("xb") as handle:
+                handle.write(data)
+        except FileExistsError as exc:
+            raise StorageError(
+                f"RAW artifacts are immutable: refusing to overwrite existing path {path}"
+            ) from exc
+        logger.debug("RAW artifact written path=%s bytes=%d sha256=%s", path, len(data), digest)
+        return RawArtifact(
+            artifact_id=str(uuid4()),
+            source_uri=source_uri,
+            retrieved_at=datetime.now(UTC),
+            content_type=content_type,
+            size_bytes=len(data),
+            sha256=digest,
+            path=str(path),
+            resolved_url=resolved_url,
+            status_code=status_code,
+            etag=etag,
+            last_modified=last_modified,
+            storage_uri=str(self.uri_for(job_id, run_id, relative_path)),
+        )
+
+    def write_json(self, job_id: str, run_id: UUID, relative_path: str, payload: Any) -> Path:
+        path = self.path_for(job_id, run_id, relative_path)
+        temp = path.with_name(f".{path.name}.tmp")
+        temp.write_text(
+            json.dumps(payload, indent=2, sort_keys=True, default=str), encoding="utf-8"
+        )
+        temp.replace(path)
+        logger.debug("JSON artifact written path=%s", path)
+        return path

pyingestkit/artifacts/naming.py

@@ -0,0 +1,32 @@
+from __future__ import annotations
+
+import re
+from pathlib import PurePosixPath
+from uuid import UUID
+
+from pyingestkit.core.exceptions import StorageError
+
+_SAFE = re.compile(r"[^A-Za-z0-9._-]+")
+
+
+def safe_component(value: str) -> str:
+    cleaned = _SAFE.sub("_", value).strip("._")
+    return cleaned or "unnamed"
+
+
+def job_parts(job_id: str) -> tuple[str, ...]:
+    return tuple(safe_component(part) for part in job_id.split("."))
+
+
+def relative_artifact_path(value: str) -> PurePosixPath:
+    if not value or "\\" in value:
+        raise StorageError(f"Invalid artifact relative path: {value!r}")
+    path = PurePosixPath(value)
+    if not path.parts or path.is_absolute() or any(part == ".." for part in path.parts):
+        raise StorageError(f"Artifact path must stay inside the run workspace: {value!r}")
+    return path
+
+
+def run_relative_key(job_id: str, run_id: UUID, relative_path: str) -> str:
+    relative = relative_artifact_path(relative_path)
+    return str(PurePosixPath("runs", *job_parts(job_id), str(run_id), *relative.parts))

pyingestkit/artifacts/raw.py

@@ -0,0 +1,43 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import datetime
+from pathlib import Path
+
+from .uri import ArtifactURI
+
+
+@dataclass(frozen=True, slots=True)
+class RawArtifact:
+    artifact_id: str
+    source_uri: str
+    retrieved_at: datetime
+    content_type: str | None
+    size_bytes: int
+    sha256: str
+    path: str
+    resolved_url: str | None = None
+    status_code: int | None = None
+    etag: str | None = None
+    last_modified: str | None = None
+    storage_uri: str | None = None
+    acquisition_mode: str = "LIVE"
+    origin_run_id: str | None = None
+    origin_artifact_id: str | None = None
+    origin_retrieved_at: datetime | None = None
+
+    def __post_init__(self) -> None:
+        if self.storage_uri is not None:
+            ArtifactURI(self.storage_uri)
+
+    @property
+    def location_uri(self) -> ArtifactURI:
+        """Canonical persisted location; V0.5 ``path`` remains the local materialization."""
+
+        if self.storage_uri is not None:
+            return ArtifactURI(self.storage_uri)
+        return ArtifactURI.from_path(self.path)
+
+    @property
+    def local_path(self) -> Path:
+        return Path(self.path)