sonata-engine 0.6.0__py3-none-any.whl

sonata_engine/__init__.py

@@ -0,0 +1,101 @@
+"""Sonata workflow engine."""
+
+from sonata_engine.core import (
+    CompiledTask,
+    CompiledWorkflow,
+    Evidence,
+    Resource,
+    ReusableTask,
+    Selection,
+    Steps,
+    Task,
+    TaskExecution,
+    TaskInputs,
+    TaskOutcome,
+    Workflow,
+    WorkflowResult,
+)
+from sonata_engine.errors import (
+    AmbiguousTaskStateError,
+    CorruptJournalError,
+    InvalidTaskOutcomeError,
+    MissingAcquireUnitError,
+    NoUpstreamValueError,
+    ResourceDependencyCycleError,
+    ResourceUnavailableError,
+    ResumeConfigurationError,
+    SelectionError,
+    StepScopeUnavailableError,
+    UndeclaredResourceError,
+    UnsupportedJournalSchemaError,
+    WorkflowTopologyMismatchError,
+)
+from sonata_engine.journal import JournalConfig, Verifier
+from sonata_engine.retention import (
+    UnknownRetainedResourceError,
+    release_retained,
+)
+from sonata_engine.workflow import (
+    TaskDefinition,
+    TaskRun,
+    WorkflowCompletion,
+    WorkflowContext,
+    WorkflowEvent,
+    WorkflowObserver,
+    WorkflowRun,
+    WorkflowSink,
+    WorkflowState,
+    bind_workflow_sink,
+    status,
+    subtask,
+    workflow_log,
+)
+
+__version__ = "0.6.0"
+
+__all__ = [
+    "AmbiguousTaskStateError",
+    "CompiledTask",
+    "CompiledWorkflow",
+    "CorruptJournalError",
+    "Evidence",
+    "InvalidTaskOutcomeError",
+    "JournalConfig",
+    "MissingAcquireUnitError",
+    "NoUpstreamValueError",
+    "Resource",
+    "ResourceDependencyCycleError",
+    "ResourceUnavailableError",
+    "ResumeConfigurationError",
+    "ReusableTask",
+    "Selection",
+    "SelectionError",
+    "StepScopeUnavailableError",
+    "Steps",
+    "Task",
+    "TaskDefinition",
+    "TaskExecution",
+    "TaskInputs",
+    "TaskOutcome",
+    "TaskRun",
+    "UndeclaredResourceError",
+    "UnknownRetainedResourceError",
+    "UnsupportedJournalSchemaError",
+    "Verifier",
+    "Workflow",
+    "WorkflowCompletion",
+    "WorkflowContext",
+    "WorkflowEvent",
+    "WorkflowObserver",
+    "WorkflowResult",
+    "WorkflowRun",
+    "WorkflowSink",
+    "WorkflowState",
+    "WorkflowTopologyMismatchError",
+    "__version__",
+    "bind_workflow_sink",
+    "release_retained",
+    "status",
+    "subtask",
+    "workflow_log",
+]

sonata_engine/core/__init__.py

@@ -0,0 +1,38 @@
+"""Core workflow primitives: tasks, resources, selection, and compiled results.
+
+This package is the stable surface for building and running a workflow: the
+`Task` hierarchy and its `Steps` composite, the `Resource` lifecycle pairing,
+`Workflow` itself, and the immutable compiled/result types the runner returns.
+Everything here is re-exported for import from `sonata_engine.core`; the
+sibling modules are implementation detail behind that list.
+"""
+
+from sonata_engine.core.compiled import (
+    CompiledTask,
+    CompiledWorkflow,
+    TaskExecution,
+    WorkflowResult,
+)
+from sonata_engine.core.inputs import TaskInputs
+from sonata_engine.core.outcome import Evidence, TaskOutcome
+from sonata_engine.core.resource_task import Resource
+from sonata_engine.core.selection import Selection
+from sonata_engine.core.steps import Steps
+from sonata_engine.core.task import ReusableTask, Task
+from sonata_engine.core.workflow import Workflow
+
+__all__ = [
+    "CompiledTask",
+    "CompiledWorkflow",
+    "Evidence",
+    "Resource",
+    "ReusableTask",
+    "Selection",
+    "Steps",
+    "Task",
+    "TaskExecution",
+    "TaskInputs",
+    "TaskOutcome",
+    "Workflow",
+    "WorkflowResult",
+]

sonata_engine/core/compiled.py

@@ -0,0 +1,116 @@
+"""Immutable artifacts produced by compiling and running a workflow.
+
+`CompiledWorkflow` is what `Workflow.compile()` returns: the recorded tasks
+rewritten into a flat sequence of `CompiledTask` units with compiler-assigned
+IDs. `WorkflowResult` is what a run returns: one `TaskExecution` per unit, in
+execution order.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from dataclasses import dataclass
+from typing import Any, Literal
+
+from sonata_engine.core.outcome import TaskOutcome
+from sonata_engine.core.resource_task import Resource
+from sonata_engine.core.task import Task
+from sonata_engine.errors import MissingAcquireUnitError
+
+TaskKind = Literal["consumer", "acquire", "release"]
+TaskExecutionStatus = Literal["passed", "skipped"]
+
+
+@dataclass(frozen=True, slots=True)
+class CompiledTask[T]:
+    """A task with its compiler-assigned, stable identity.
+
+    `task_id` only ever exists here; `Task` instances never carry one.
+
+    `kind` discriminates ordinary consumer units from the acquire/release units
+    the compiler splices in for resources. For an `acquire`/`release` unit,
+    `resource` names which `Resource` it belongs to, letting the runner
+    pair a release with its acquire and release out of linear order on failure.
+    """
+
+    task_id: str
+    task: Task[T]
+    required_resources: tuple[Resource, ...] = ()
+    kind: TaskKind = "consumer"
+    resource: Resource | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class CompiledWorkflow:
+    """The immutable result of `Workflow.compile()`."""
+
+    workflow_id: str
+    tasks: tuple[CompiledTask[object], ...]
+
+    @property
+    def fingerprint(self) -> str:
+        """Deterministic identity of topology and reusable-task semantics."""
+        acquire_ids = {
+            id(task.resource): task.task_id
+            for task in self.tasks
+            if task.kind == "acquire" and task.resource is not None
+        }
+
+        def _acquire_id(resource: Resource[Any]) -> str:
+            try:
+                return acquire_ids[id(resource)]
+            except KeyError as exc:
+                raise MissingAcquireUnitError(
+                    f"required_resources names {resource.title!r}, which has no "
+                    "acquire unit in this compiled workflow"
+                ) from exc
+
+        topology = [
+            (
+                task.task_id,
+                task.kind,
+                f"{type(task.task).__module__}.{type(task.task).__qualname__}",
+                task.task._fingerprint_payload(),
+                tuple(_acquire_id(resource) for resource in task.required_resources),
+            )
+            for task in self.tasks
+        ]
+        canonical = json.dumps(
+            [self.workflow_id, topology],
+            ensure_ascii=True,
+            separators=(",", ":"),
+        ).encode()
+        return f"sha256:{hashlib.sha256(canonical).hexdigest()}"
+
+
+@dataclass(frozen=True, slots=True)
+class TaskExecution:
+    """The runtime result of one compiled task unit.
+
+    `outcome` is `None` exactly when `status` is `"skipped"` — the `None`
+    reports the skip, not a task that ran and returned nothing. A task that
+    ran always has a `TaskOutcome`, whose `value` may itself be `None`. So
+    read a result as `execution.outcome.value if execution.outcome else ...`;
+    there is deliberately no `execution.value` shortcut, because collapsing
+    the two cases would throw that distinction away.
+    """
+
+    task_id: str
+    status: TaskExecutionStatus
+    outcome: TaskOutcome[Any] | None
+
+
+@dataclass(frozen=True, slots=True)
+class WorkflowResult:
+    """Immutable results from one successful workflow run."""
+
+    workflow_id: str
+    tasks: tuple[TaskExecution, ...]
+
+    def by_id(self, task_id: str) -> TaskExecution:
+        """Return the execution whose `task_id` matches, or raise `KeyError`."""
+        for execution in self.tasks:
+            if execution.task_id == task_id:
+                return execution
+        raise KeyError(task_id)

sonata_engine/core/inputs.py

@@ -0,0 +1,98 @@
+"""What a task is handed when it runs.
+
+A `TaskInputs` carries the acquired resource values a task declared, plus the
+value the preceding step produced when the task sits inside a composite. Both
+are reads through a running workflow: a task cannot obtain a resource it never
+declared, and the container is frozen once built.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterable, Mapping, Set
+from dataclasses import dataclass, field
+from types import MappingProxyType
+from typing import TYPE_CHECKING, Any, cast
+
+from sonata_engine.errors import (
+    NoUpstreamValueError,
+    ResourceUnavailableError,
+    UndeclaredResourceError,
+)
+
+if TYPE_CHECKING:
+    from sonata_engine.core.resource_task import Resource
+    from sonata_engine.core.step_scope import _StepScopeProtocol
+
+# Distinct from `None`, which is a legitimate and reconstructible step value.
+_NO_UPSTREAM: Any = object()
+
+
+@dataclass(frozen=True, slots=True)
+class TaskInputs:
+    """The resource values a task is permitted to observe during one run."""
+
+    _values: Mapping[int, object]
+    _accessible: Set[int]
+    _upstream: Any = _NO_UPSTREAM
+    # compare=False is load-bearing, not cosmetic: `_StepScope` holds a `TaskInputs`
+    # in its own `base_inputs` field, so if this field participated in `__eq__`,
+    # comparing two `TaskInputs` would walk into a scope and back into inputs.
+    _step_scope: _StepScopeProtocol | None = field(
+        default=None, compare=False, repr=False
+    )
+
+    def __post_init__(self) -> None:
+        """Freeze the resource-value mapping and the accessible-id set."""
+        object.__setattr__(self, "_values", MappingProxyType(dict(self._values)))
+        object.__setattr__(self, "_accessible", frozenset(self._accessible))
+
+    @classmethod
+    def empty(cls) -> TaskInputs:
+        """Return inputs that grant access to no resource and have no upstream."""
+        return cls({}, frozenset())
+
+    @classmethod
+    def _for_resources(
+        cls, values: Mapping[Resource[Any], object], accessible: Set[Resource[Any]]
+    ) -> TaskInputs:
+        return cls._for_resource_values(
+            {id(resource): value for resource, value in values.items()}, accessible
+        )
+
+    @classmethod
+    def _for_resource_values(
+        cls, values: Mapping[int, object], accessible: Iterable[Resource[Any]]
+    ) -> TaskInputs:
+        return cls(values, frozenset(map(id, accessible)))
+
+    def resource[T](self, resource: Resource[T]) -> T:
+        """Return the acquired value of a declared `Resource`, typed as its value.
+
+        Raises `UndeclaredResourceError` if this task never declared the
+        resource, or `ResourceUnavailableError` if it declared it but no value
+        was published for it.
+        """
+        resource_id = id(resource)
+        if resource_id not in self._accessible:
+            raise UndeclaredResourceError(
+                f"resource {resource.title!r} is not declared"
+            )
+        try:
+            value = self._values[resource_id]
+        except KeyError as exc:
+            raise ResourceUnavailableError(
+                f"resource {resource.title!r} is unavailable"
+            ) from exc
+        return cast(T, value)
+
+    def upstream(self) -> Any:  # noqa: ANN401
+        """Return the value the preceding step produced.
+
+        Raises when nothing precedes this step: the first step of a top-level
+        composite, or a task not running inside one.
+        """
+        if self._upstream is _NO_UPSTREAM:
+            raise NoUpstreamValueError(
+                "no upstream value: nothing ran before this step"
+            )
+        return self._upstream

sonata_engine/core/outcome.py

@@ -0,0 +1,37 @@
+"""What a task reports when it finishes.
+
+A `TaskOutcome` pairs the value a task produced with the `Evidence` that
+justifies reusing it on a later run. The value never leaves the process; the
+evidence is what the journal stores and what resume re-verifies.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+
+@dataclass(frozen=True, slots=True)
+class Evidence:
+    """A checkable claim that a task's output still exists and is unchanged.
+
+    `kind` names the verifier that knows how to check it (for example
+    `file-digest`), `reference` is what that verifier looks at, and `digest`
+    is the optional expected fingerprint. On resume, every entry of a reusable
+    task must verify before its evidence is trusted.
+    """
+
+    kind: str
+    reference: str
+    digest: str | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class TaskOutcome[T]:
+    """The value a task returned, plus the evidence a later resume can check.
+
+    `value` is an in-process data channel and is never journalled, so a skipped
+    task is reconstructed with `None` rather than its original value.
+    """
+
+    value: T | None = None
+    evidence: tuple[Evidence, ...] = field(default_factory=tuple)

sonata_engine/core/resource_task.py

@@ -0,0 +1,77 @@
+"""Resources and the acquire/release units that manage their lifetime.
+
+A `Resource` bundles the two callbacks a task needs to obtain and later give
+back an expensive thing (a VM, a token, a mounted image). The compiler turns
+each one into `ResourceOp` units spliced around its consumers.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from enum import StrEnum
+from typing import Any, override
+
+from sonata_engine.core.inputs import TaskInputs
+from sonata_engine.core.outcome import TaskOutcome
+from sonata_engine.core.task import Task
+
+
+@dataclass(frozen=True, slots=True, eq=False)
+class Resource[T]:
+    """A pair of acquire/release side effects a task depends on.
+
+    A `Resource` is not itself added to a workflow; consumer tasks reference it
+    via `Workflow.add(task, requires=(resource,))`. `compile()` splices an
+    acquire unit before the first consumer and a release unit after the last.
+
+    Identity semantics (`eq=False`): the compiler keys resources by `id()` and
+    deliberately treats two structurally identical `Resource` instances as
+    distinct (each gets its own acquire/release pair and runtime value). Value
+    equality would contradict that -- and would also make `hash()` recurse
+    forever on a cyclic `requires` graph.
+    """
+
+    title: str
+    acquire: Callable[[TaskInputs], T] = field(repr=False)
+    release: Callable[[TaskInputs, T], None] = field(repr=False)
+    requires: tuple[Resource[Any], ...] = ()
+    # `keep` retains a resource so a later run need not rebuild it. A resource
+    # that holds a secret -- a staged token, a signing key, an open credential
+    # lease -- declares always_release instead, so leaving it behind is not
+    # something a caller can cause by forgetting to classify it.
+    always_release: bool = False
+    # Rebuilds an acquired value from its journal record, for a teardown running
+    # in a later process. The journal can only hold JSON, so a release written
+    # against a dataclass needs its own type back rather than a dict.
+    revive: Callable[[Any], T] | None = field(default=None, repr=False)
+    acquire_idempotent: bool = False
+
+    @property
+    def release_title(self) -> str:
+        """Title for the release unit, derived from the resource's own title."""
+        return f"Release {self.title.removeprefix('Acquire ')}"
+
+
+class ResourceOperation(StrEnum):
+    """Which half of a resource's lifecycle a `ResourceOp` performs."""
+
+    ACQUIRE = "acquire"
+    RELEASE = "release"
+
+
+@dataclass(frozen=True, slots=True, eq=True)
+class ResourceOp(Task[object]):
+    """Engine-owned lifecycle unit identified by a resource and operation."""
+
+    title: str
+    resource: Resource[Any] = field(repr=False)
+    operation: ResourceOperation = ResourceOperation.ACQUIRE
+    idempotent: bool = False
+
+    @override
+    def run(self, inputs: TaskInputs) -> TaskOutcome[object]:
+        if self.operation is ResourceOperation.ACQUIRE:
+            return TaskOutcome(value=self.resource.acquire(inputs))
+        self.resource.release(inputs, inputs.resource(self.resource))
+        return TaskOutcome(value=None)

sonata_engine/core/selection.py

@@ -0,0 +1,35 @@
+"""Narrowing a workflow to a slice of its consumer tasks."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from sonata_engine.errors import SelectionError
+
+
+@dataclass(frozen=True, slots=True)
+class Selection:
+    """Which consumer tasks survive compilation, addressed by title slug.
+
+    Selection deliberately names tasks by slug rather than by compiled
+    `task_id`: ordinals renumber over the survivors, so an ID is not a stable
+    handle for the very operation that changes it.
+
+    Only consumer tasks are selectable. Acquire/release units are engine-owned
+    and are re-spliced around whichever consumers survive, so a slice keeps its
+    cleanup without the caller arranging anything.
+    """
+
+    only: str | None = None
+    start: str | None = None
+    until: str | None = None
+
+    def __post_init__(self) -> None:
+        """Reject `only` combined with `start` or `until`."""
+        if self.only is not None and (self.start is not None or self.until is not None):
+            raise SelectionError("only is mutually exclusive with start and until")
+
+    @property
+    def is_empty(self) -> bool:
+        """True when this selection filters nothing."""
+        return self.only is None and self.start is None and self.until is None

sonata_engine/core/slug.py

@@ -0,0 +1,15 @@
+"""Turning a task title into the addressable slug used by `Selection`."""
+
+import re
+
+_SLUG_INVALID_CHARS = re.compile(r"[^a-z0-9]+")
+
+
+def slugify(title: str) -> str:
+    """Return `title` as a lowercase, hyphen-only slug.
+
+    Lowercases first, then collapses each run of characters outside `[a-z0-9]`
+    into a single hyphen and trims the hyphens off both ends. A title made
+    entirely of non-alphanumeric characters therefore slugs to the empty string.
+    """
+    return _SLUG_INVALID_CHARS.sub("-", title.lower()).strip("-")

sonata_engine/core/step_scope.py

@@ -0,0 +1,28 @@
+"""The runner-side handle a composite needs to execute its steps."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any, Protocol
+
+if TYPE_CHECKING:
+    from sonata_engine.core.compiled import TaskExecution
+    from sonata_engine.core.task import Task
+
+
+class _StepScopeProtocol(Protocol):
+    """The runner-side handle a composite uses to execute its steps.
+
+    It holds the compiled unit id, the journal and the resume flag — none of
+    which a task is told — so a composite can run a step without knowing where
+    it sits. A composite passes a step and a slug; the scope names it, decides,
+    records, reports and runs it.
+
+    Private (leading underscore) and exported from nowhere: `TaskInputs._step_scope`,
+    the field this types, is itself underscore-private engine plumbing that a
+    hand-written task is not meant to reach for. There is no public field to type
+    against, so there is nothing to gain from making this Protocol public.
+    """
+
+    def run_step(self, step: Task[Any], slug: str, upstream: Any) -> TaskExecution:  # noqa: ANN401
+        """Run one step beneath this scope and return how it went."""
+        ...

sonata_engine/core/steps.py

@@ -0,0 +1,88 @@
+"""Composites assembled from smaller tasks rather than written as one body."""
+
+from __future__ import annotations
+
+from typing import Any, override
+
+from sonata_engine.core.inputs import TaskInputs
+from sonata_engine.core.outcome import TaskOutcome
+from sonata_engine.core.slug import slugify
+from sonata_engine.core.task import Task
+from sonata_engine.errors import StepScopeUnavailableError
+
+
+class Steps(Task[Any]):
+    """A task assembled from steps rather than written.
+
+    Each step is an ordinary `Task`, run in order, on this task's own thread.
+    Each receives the value the step before it produced, so a pipeline needs no
+    wiring; a step that needs nothing simply never asks. The whole thing stays
+    one compiled unit: one ordinal, one thing `Selection` can name, one fate.
+
+    Steps are journalled individually, so a resumed composite skips the steps it
+    already finished. What may be skipped is decided exactly as it is for a
+    compiled unit — a `ReusableTask` whose evidence still verifies. Its only
+    legal value is None, which is reconstructed when the step is skipped.
+
+    `idempotent` is always `True`. It says this coordinator is safe to re-enter
+    after a failed attempt, and says nothing about the steps: each carries its
+    own flag. Without it the enclosing unit's failed record would refuse the
+    resume this class exists to make cheap.
+    """
+
+    idempotent = True
+
+    def __init__(self, *, title: str, steps: tuple[Task[Any], ...]) -> None:
+        """Assemble `steps` into one composite named `title`.
+
+        Each step is slugged from its own title to give it a stable name inside
+        the composite. An empty `steps` sequence, a title that slugs to nothing,
+        or two steps that slug the same are all rejected here, since the slug is
+        what the journal keys the step's records on.
+        """
+        if not steps:
+            raise ValueError("Steps requires at least one step")
+
+        slugs: list[str] = []
+        for step in steps:
+            slug = slugify(step.title)
+            if not slug:
+                raise ValueError(f"step title {step.title!r} produces an empty slug")
+            if slug in slugs:
+                raise ValueError(f"duplicate step slug {slug!r} in {title!r}")
+            slugs.append(slug)
+
+        self.title = title
+        self._steps = steps
+        self._slugs = tuple(slugs)
+
+    @override
+    def _fingerprint_payload(self) -> object:
+        return tuple(
+            (
+                slug,
+                f"{type(step).__module__}.{type(step).__qualname__}",
+                step._fingerprint_payload(),
+            )
+            for slug, step in zip(self._slugs, self._steps, strict=True)
+        )
+
+    @override
+    def run(self, inputs: TaskInputs) -> TaskOutcome[Any]:
+        scope = inputs._step_scope
+        if scope is None:
+            raise StepScopeUnavailableError(
+                f"{type(self).__name__} {self.title!r} must be run by the workflow "
+                "runner: add it to a Workflow rather than calling run() directly"
+            )
+
+        upstream: Any = inputs._upstream
+        for slug, step in zip(self._slugs, self._steps, strict=True):
+            execution = scope.run_step(step, slug, upstream)
+            # A step that ran contributes its value, including a legitimate
+            # None; a skipped step contributes None, the only value a skippable
+            # ReusableTask may return.
+            upstream = (
+                execution.outcome.value if execution.outcome is not None else None
+            )
+        return TaskOutcome(value=upstream)