artest-runtime 0.2.0__py3-none-any.whl

artest/runtime/__init__.py

@@ -0,0 +1,36 @@
+"""Typed DAG runtime with an event journal and a content-addressed artifact store.
+
+Steps are typed async functions. Every state change is appended to a journal, which is what makes
+runs resumable after a crash or a pause, and what feeds the live UI and the exported report.
+"""
+
+from artest.runtime.artifacts import ArtifactMissingError, ArtifactStore, StoredBlob
+from artest.runtime.events import ControlEvent, RunStatus, control_event
+from artest.runtime.journal import Entry, Journal, RunRecord, RunTakenError, utc_now
+from artest.runtime.pipeline import Pipeline, PipelineError
+from artest.runtime.runner import Pause, Runner, RunOutcome, StepContext
+from artest.runtime.telemetry import TelemetryUnavailableError, configure_telemetry
+from artest.runtime.views import effective
+
+__all__ = [
+    "ArtifactMissingError",
+    "ArtifactStore",
+    "ControlEvent",
+    "Entry",
+    "Journal",
+    "Pause",
+    "Pipeline",
+    "PipelineError",
+    "RunOutcome",
+    "RunRecord",
+    "RunStatus",
+    "RunTakenError",
+    "Runner",
+    "StepContext",
+    "StoredBlob",
+    "TelemetryUnavailableError",
+    "configure_telemetry",
+    "control_event",
+    "effective",
+    "utc_now",
+]

artest/runtime/artifacts.py

@@ -0,0 +1,68 @@
+"""The content-addressed artifact store: immutable bytes on disk, named by their SHA-256.
+
+The same screenshot stored twice takes space once. Writes go to a temporary file that is then renamed,
+so a reader never sees half an artifact. Callers write an artifact before appending any event that
+references it, so a crash can leave an unreferenced file but never a reference to a missing one.
+"""
+
+import asyncio
+import hashlib
+import os
+import re
+import secrets
+from dataclasses import dataclass
+from pathlib import Path
+
+from artest.runtime.journal import Journal
+
+_SHA256 = re.compile(r"^[0-9a-f]{64}$")
+
+
+@dataclass(frozen=True)
+class StoredBlob:
+    sha256: str
+    media_type: str
+    size: int
+
+
+class ArtifactMissingError(LookupError):
+    pass
+
+
+class ArtifactStore:
+    def __init__(self, root: Path, journal: Journal) -> None:
+        self._root = root
+        self._journal = journal
+
+    def path(self, sha256: str) -> Path:
+        if not _SHA256.match(sha256):
+            msg = f"not a SHA-256: {sha256!r}"
+            raise ValueError(msg)
+        return self._root / sha256[:2] / sha256
+
+    async def put(self, data: bytes, media_type: str) -> StoredBlob:
+        sha256 = hashlib.sha256(data).hexdigest()
+        path = self.path(sha256)
+        if not path.exists():
+            await asyncio.to_thread(_write_atomically, path, data)
+        await self._journal.record_artifact(sha256, media_type, len(data))
+        return StoredBlob(sha256=sha256, media_type=media_type, size=len(data))
+
+    async def get(self, sha256: str) -> bytes:
+        try:
+            return await asyncio.to_thread(self.path(sha256).read_bytes)
+        except FileNotFoundError as error:
+            raise ArtifactMissingError(sha256) from error
+
+    def exists(self, sha256: str) -> bool:
+        return self.path(sha256).exists()
+
+
+def _write_atomically(path: Path, data: bytes) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    temporary = path.with_name(f".{path.name}.{secrets.token_hex(4)}.tmp")
+    with temporary.open("wb") as file:
+        file.write(data)
+        file.flush()
+        os.fsync(file.fileno())
+    temporary.replace(path)

artest/runtime/events.py

@@ -0,0 +1,108 @@
+"""Control events: what the runtime records about runs and steps.
+
+They share the journal with the pipeline's own domain events. The runtime knows nothing about what steps
+do, so it never looks inside a domain event beyond its ``type``. The step and attempt an event belongs
+to live in the journal entry, not in the payload.
+"""
+
+from enum import StrEnum
+from typing import Annotated, Literal
+
+from pydantic import AwareDatetime, BaseModel, ConfigDict, Field, StringConstraints, TypeAdapter
+
+Sha256 = Annotated[str, StringConstraints(pattern=r"^[0-9a-f]{64}$")]
+
+
+class RuntimeModel(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="forbid", json_schema_serialization_defaults_required=True)
+
+
+class RunStatus(StrEnum):
+    RUNNING = "running"
+    PAUSED = "paused"
+    COMPLETED = "completed"
+    FAILED = "failed"
+
+
+class RunStarted(RuntimeModel):
+    type: Literal["run_started"] = "run_started"
+    pipeline: str
+    inputs: dict[str, Sha256] = Field(
+        default_factory=dict[str, Sha256], description="Each run input, stored as JSON in the artifact store"
+    )
+
+
+class RunResumed(RuntimeModel):
+    type: Literal["run_resumed"] = "run_resumed"
+
+
+class StepStarted(RuntimeModel):
+    """A new attempt of a step. Consumers drop the step's domain events from earlier attempts."""
+
+    type: Literal["step_started"] = "step_started"
+
+
+class StepCompleted(RuntimeModel):
+    type: Literal["step_completed"] = "step_completed"
+    output: Sha256 = Field(description="The step's output, stored as JSON in the artifact store")
+
+
+class StepFailed(RuntimeModel):
+    type: Literal["step_failed"] = "step_failed"
+    error: str
+
+
+class StepPaused(RuntimeModel):
+    type: Literal["step_paused"] = "step_paused"
+    reason: str
+    resume_after: AwareDatetime | None = None
+    detail: str = ""
+
+
+class RunPaused(RuntimeModel):
+    type: Literal["run_paused"] = "run_paused"
+    reason: str
+    resume_after: AwareDatetime | None = None
+    steps: tuple[str, ...]
+
+
+class RunCompleted(RuntimeModel):
+    type: Literal["run_completed"] = "run_completed"
+
+
+class RunFailed(RuntimeModel):
+    type: Literal["run_failed"] = "run_failed"
+    steps: tuple[str, ...]
+    interrupted: bool = False
+    """No step failed: the run was stopped, it crashed between steps, or the process running it died. Its
+    steps stay as they were, and a later run of the same ID goes on from them."""
+
+
+ControlEvent = Annotated[
+    RunStarted
+    | RunResumed
+    | StepStarted
+    | StepCompleted
+    | StepFailed
+    | StepPaused
+    | RunPaused
+    | RunCompleted
+    | RunFailed,
+    Field(discriminator="type"),
+]
+
+control_event: TypeAdapter[ControlEvent] = TypeAdapter(ControlEvent)
+
+CONTROL_EVENT_TYPES = frozenset(
+    {
+        "run_started",
+        "run_resumed",
+        "step_started",
+        "step_completed",
+        "step_failed",
+        "step_paused",
+        "run_paused",
+        "run_completed",
+        "run_failed",
+    }
+)