flowlit 0.1.0__py3-none-any.whl

flowlit/__init__.py

@@ -0,0 +1,30 @@
+"""Flowlit: a lightweight, event-driven workflow execution engine.
+
+The codebase is organized as four layers, each only allowed to depend on
+the ones below it:
+
+    interfaces  (CLI, future MCP server -- thin adapters, no business logic)
+        v
+    infrastructure  (YAML loading, in-memory event bus/repository, executors)
+        v
+    application  (use cases, ports/Protocols, orchestration)
+        v
+    domain  (Step/Workflow entities, DAG validation -- zero third-party deps)
+
+`domain` and `application` never import from `infrastructure` or
+`interfaces`. Wiring of concrete implementations happens in exactly one
+place: `flowlit.interfaces.composition_root`.
+
+Above all four layers sits one public extension seam, re-exported here
+for ergonomics: `register_executor()` (see `flowlit.plugins`) lets code
+that imports flowlit as a library add its own step-type executors
+without forking flowlit's own source.
+"""
+
+from flowlit.plugins import clear_registered_executors, register_executor, registered_executors
+
+__all__ = [
+    "clear_registered_executors",
+    "register_executor",
+    "registered_executors",
+]

flowlit/application/__init__.py

@@ -0,0 +1,7 @@
+"""Application layer: use cases, ports, and orchestration.
+
+Depends only on `flowlit.domain` plus the standard library (`asyncio`,
+`typing`). Defines the ports (`Protocol` classes) that infrastructure
+adapters implement, and the `WorkflowService` use-case API that all
+interface adapters (CLI, future MCP server) call through.
+"""

flowlit/application/dto.py

@@ -0,0 +1,100 @@
+"""Boundary DTOs returned by `WorkflowService`.
+
+Plain, immutable, primitives-only dataclasses -- never a domain `Step` or
+`Workflow`. Every interface adapter (CLI now, MCP server later) can
+serialize these straight to JSON with `dataclasses.asdict()` without
+importing anything from `flowlit.domain`.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from dataclasses import dataclass
+from typing import Any
+
+
+@dataclass(frozen=True)
+class StepDTO:
+    id: str
+    type: str
+    status: str
+    spec: Mapping[str, Any]
+    """The *raw* spec, exactly as declared in the plan -- placeholders
+    (`${{ steps.<id>.output.<path> }}`) and all, unresolved. See
+    `resolved_spec` for the substituted values, once available."""
+    resolved_spec: Mapping[str, Any] | None
+    """The spec with every placeholder substituted, the same way
+    `StepDispatchRequested` carries it to the executor -- computed
+    on demand (via `Workflow.resolve_step_spec()`), not stored, so this
+    is always current as of *this* call, not a snapshot from dispatch
+    time. `None` if it can't be resolved (yet, or ever) -- most commonly
+    because the step is still `PENDING`, but also for e.g. a `SKIPPED`
+    step whose own spec referenced a dependency that never produced
+    output. See `flowlit.application.errors.OutputResolutionError` for
+    what "can't be resolved" actually means; a `None` here is never
+    itself an error to the caller, just "nothing to show yet."""
+    output: Mapping[str, Any] | None
+    error: str | None
+    optional: bool
+    blocking: bool
+    silent_failures: bool
+    when: str | None
+    """The raw `when:` condition, if this step has one -- see
+    `flowlit.domain.step.Step`. `None` means the step is unconditional,
+    not that its condition was falsy (a falsy condition shows up as
+    `status == "skipped"` instead)."""
+    when_not: str | None
+    """The raw `when_not:` condition, if this step has one -- negated
+    `when:`, same caveat about `None` not meaning "was truthy"."""
+    else_: frozenset[str] | None
+    """The step ids this step's `else:` names, if it has one -- true iff
+    every one of them ended up skipped by its own condition (not
+    cascade) -- see `skip_cause`."""
+    skip_cause: str | None
+    """Only meaningful when `status == "skipped"`, `None` otherwise:
+    `"condition"` if this step's own `when`/`when_not`/`else` evaluated
+    falsy (a deliberate, expected non-selection), `"cascade"` if it was
+    swept up by an upstream failure or an external `cancel_workflow()`
+    instead, never getting the chance to evaluate anything of its own --
+    see `flowlit.domain.step.SkipCause`."""
+    timeout_seconds: float | None
+    """This step's own `timeout_seconds`, if set -- see
+    `flowlit.domain.step.Step`. `None` means unbounded, not that a
+    timeout already happened (that shows up as `status == "failed"`,
+    with `error` saying so)."""
+    retry: int
+    """Additional attempts after the first, if any -- see
+    `flowlit.domain.step.Step`. `0` (the default) means no retry;
+    doesn't reflect how many attempts were actually made if the step is
+    still in flight or already terminal -- `error` says that, e.g.
+    "failed after 3 attempt(s): ..."."""
+    retry_backoff_seconds: float
+    """Delay before the first retry, doubling each attempt -- only
+    meaningful when `retry > 0`."""
+    started_at: str | None
+    """ISO 8601 (`datetime.isoformat()`), `None` if the step never ran
+    (e.g. it was skipped) -- see `flowlit.domain.step.Step`."""
+    ended_at: str | None
+    """ISO 8601, `None` until the step reaches a terminal status."""
+
+
+@dataclass(frozen=True)
+class WorkflowStatusDTO:
+    id: str
+    run_id: str
+    """This execution's identity, distinct from `id` (the plan's own
+    declared, run-independent identity) -- see
+    `flowlit.domain.workflow.Workflow.run_id`. What every step's
+    `${{ step.idempotency_key }}` is built from."""
+    concurrency_mode: str
+    """`"allow_parallel"` or `"singleton"` -- see
+    `flowlit.domain.workflow.ConcurrencyMode`."""
+    name: str
+    status: str
+    steps: tuple[StepDTO, ...]
+    created_at: str
+    """ISO 8601 -- always set; stamped once, in `Workflow.create()`."""
+    started_at: str | None
+    """ISO 8601, `None` until the workflow leaves PENDING."""
+    ended_at: str | None
+    """ISO 8601, `None` until the workflow reaches a terminal status."""

flowlit/application/errors.py

@@ -0,0 +1,119 @@
+"""Application-level errors.
+
+Callers of `WorkflowService` (the CLI now, an MCP server later) only ever
+need to catch these -- never a domain error, a pydantic `ValidationError`,
+or a `yaml.YAMLError` directly. Adapters that produce those lower-level
+errors wrap them here (via `raise ... from exc`, preserving the cause for
+debugging) so the public API has one small, deliberate error surface.
+"""
+
+from __future__ import annotations
+
+
+class ApplicationError(Exception):
+    """Base class for all errors raised by the application layer."""
+
+
+class PlanSchemaError(ApplicationError):
+    """The plan text isn't valid YAML, or doesn't match the plan schema
+    (missing/extra/mistyped fields) -- a problem with the document itself,
+    independent of whether its dependency graph would be valid.
+    """
+
+
+class PlanValidationError(ApplicationError):
+    """The plan is well-formed YAML matching the schema, but the step
+    graph it describes is not a valid DAG (a cycle, or a dependency on an
+    unknown step id).
+    """
+
+
+class UnknownStepTypeError(ApplicationError):
+    """A step's `type` has no registered executor."""
+
+    def __init__(self, step_type: str) -> None:
+        self.step_type = step_type
+        super().__init__(f"no executor registered for step type '{step_type}'")
+
+
+class SpecValidationError(ApplicationError):
+    """A step's `spec` doesn't match its executor's own declared shape --
+    raised by `TypedExecutor.validate_spec()` implementations (see
+    `flowlit.application.ports.executor`), never a raw pydantic
+    `ValidationError` or any other library-specific exception, so callers
+    only ever need to catch this one type.
+
+    Raised (and handled) in two different places, deliberately: eagerly
+    in `WorkflowService.submit_plan()` for any step whose spec has no
+    `${{ steps.<id>.output.<path> }}` placeholders (fail the whole plan
+    before execution starts), and again in `ExecutorDispatcher` right
+    before dispatch (the only point a placeholder-bearing spec can
+    honestly be checked, once it's been resolved) -- there, it's just an
+    ordinary step failure, not a re-raise.
+    """
+
+
+class WorkflowNotFoundError(ApplicationError):
+    """No workflow with the given id has been submitted (or it hasn't
+    been persisted yet)."""
+
+    def __init__(self, workflow_id: str) -> None:
+        self.workflow_id = workflow_id
+        super().__init__(f"no workflow found with id '{workflow_id}'")
+
+
+class WorkflowAlreadyStartedError(ApplicationError):
+    """A workflow was asked to start (via `WorkflowService.start_workflow()`
+    or `run_to_completion()`) more than once. Each submitted workflow can
+    only be started once -- poll `get_status()`/`get_next_steps()` to
+    observe a run that's already in progress, rather than starting it
+    again.
+    """
+
+    def __init__(self, workflow_id: str) -> None:
+        self.workflow_id = workflow_id
+        super().__init__(f"workflow '{workflow_id}' has already been started")
+
+
+class WorkflowAlreadyInFlightError(ApplicationError):
+    """A `concurrency.mode: singleton` plan was submitted while another
+    run of the same plan `id` is still PENDING or RUNNING, and this
+    submission doesn't match it closely enough to be treated as a
+    duplicate of it (see `WorkflowRunIdConflictError` for the one case
+    that's ambiguous rather than a plain reject).
+
+    Raised for: no `run_id` supplied at all (`submit_plan()`, which
+    never claims a stable identity to dedup against), or a `run_id` that
+    doesn't match the in-flight run's own. The caller should back off and
+    retry later, or -- if it actually meant to attach to the existing
+    run -- resubmit via `submit_plan_with_run_id()` with that run's own
+    `run_id` and identical plan text instead.
+    """
+
+    def __init__(self, plan_id: str, in_flight_handle: str) -> None:
+        self.plan_id = plan_id
+        self.in_flight_handle = in_flight_handle
+        super().__init__(
+            f"plan '{plan_id}' is a singleton and already has a run in flight: "
+            f"'{in_flight_handle}'"
+        )
+
+
+class WorkflowRunIdConflictError(ApplicationError):
+    """A `concurrency.mode: singleton` plan was submitted with the same
+    `run_id` as its currently in-flight run, but different plan text.
+
+    Reusing a `run_id` is how a caller declares "this is the same
+    logical request as before" -- but the request itself has changed, so
+    silently either running the old text or the new one would be
+    guessing. Modeled on Stripe's own idempotency-key contract: a key
+    reused with a different payload is a conflict, not a dedup.
+    """
+
+    def __init__(self, plan_id: str, run_id: str) -> None:
+        self.plan_id = plan_id
+        self.run_id = run_id
+        super().__init__(
+            f"plan '{plan_id}' run '{run_id}' is already in flight with different plan text -- "
+            "reusing a run_id must resubmit the exact same plan"
+        )

flowlit/application/events.py

@@ -0,0 +1,201 @@
+"""The events published on the `EventBus` as a workflow runs.
+
+All immutable value objects (`@dataclass(frozen=True)`, the closest
+Python equivalent to a Java record). Subscribers are keyed by these
+classes' *type*, not by a string topic name -- see `ports.event_bus`.
+
+    Orchestrator                                Dispatcher / Executor
+    ------------                                ---------------------
+    publishes StepReady        (observers only)
+    publishes StepDispatchRequested  ---------->  runs the step
+    subscribes to
+      StepCompleted / StepFailed  <-------------  publishes StepCompleted or StepFailed
+    publishes StepSkipped
+      (cascading, for a failed
+      optional+blocking step's
+      dependents)
+    publishes WorkflowCompleted
+      or WorkflowFailed once
+      the workflow is settled
+    publishes WorkflowCancelled
+      (Orchestrator.cancel(),
+      called from outside --
+      not triggered by any
+      step's own outcome)
+
+`StepReady` and `StepDispatchRequested` carry the same shape of fields
+and are published back to back for the same step, but they're
+deliberately two distinct event types, not one event with two
+subscribers. Their `spec` does differ in one respect: `StepReady` always
+carries the step's *raw* spec, exactly as declared in the plan
+(`${{ steps.<id>.output.<path> }}` placeholders and all), since it's
+announced before the orchestrator has even attempted to resolve them;
+`StepDispatchRequested` carries the *resolved* spec, with every
+placeholder substituted against the actual output of the steps it
+references (see `flowlit.domain.spec_resolution`) -- executors only
+ever see concrete values, never a placeholder syntax to worry about.
+
+* `StepReady` is a pure announcement -- "this step's dependencies are
+  satisfied and it has been handed off for execution." Any number of
+  observers can subscribe (the CLI's progress printer today; a future
+  audit log or MCP notification stream later). Nothing subscribed to it
+  causes further work.
+* `StepDispatchRequested` is the actual trigger -- "now go run it." Only
+  `ExecutorDispatcher` subscribes to it.
+
+Splitting them this way makes "every observer has seen StepReady before
+any executor-side effect (including its own logging) can happen"
+*structural*: it falls out of the orchestrator publishing `StepReady`
+before `StepDispatchRequested`, and the event bus fully delivering one
+event to all of its subscribers before returning (see
+`AsyncioEventBus`). It does not depend on which of several subscribers
+to the same event happened to register first -- relying on that turned
+out to be exactly the kind of accident-of-wiring-order bug this split
+exists to rule out.
+
+`WorkflowCompleted` fires for a successful *or* partially-degraded-but-
+still-successful run alike (`WorkflowStatus.COMPLETED` or
+`COMPLETED_WITH_FAILURES` -- see `flowlit.domain.workflow`); the exact
+distinction is a matter of looking at the workflow's own status
+afterward, not a different event.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from dataclasses import dataclass
+from typing import Any
+
+
+@dataclass(frozen=True)
+class StepReady:
+    """Published by the orchestrator when a step's dependencies have all
+    been satisfied and it has been handed off for execution. Purely
+    informational -- for observers (e.g. the CLI's progress printer),
+    not for whatever actually runs the step. See `StepDispatchRequested`
+    for that.
+    """
+
+    workflow_id: str
+    step_id: str
+    step_type: str
+    spec: Mapping[str, Any]
+    timeout_seconds: float | None = None
+    """This step's `timeout_seconds` (see `flowlit.domain.step.Step`),
+    carried here purely for observers -- e.g. a progress printer that
+    wants to say "submitted, 30s budget." `StepDispatchRequested` is
+    what actually enforces it; see there."""
+    retry: int = 0
+    retry_backoff_seconds: float = 1.0
+    """This step's `retry`/`retry_backoff_seconds`, carried here purely
+    for observers, same spirit as `timeout_seconds` above --
+    `StepDispatchRequested` is what actually enforces them."""
+
+
+@dataclass(frozen=True)
+class StepDispatchRequested:
+    """Published by the orchestrator immediately after `StepReady`, once
+    that announcement has been fully delivered to every observer. This
+    is what `ExecutorDispatcher` actually subscribes to in order to run
+    the step -- kept separate from `StepReady` so "the step was
+    announced" and "the step is now being executed" can never race.
+    """
+
+    workflow_id: str
+    step_id: str
+    step_type: str
+    spec: Mapping[str, Any]
+    timeout_seconds: float | None = None
+    """If set, `ExecutorDispatcher` enforces this as a hard ceiling on
+    the executor's entire `execute()` call via `asyncio.wait_for()`,
+    uniformly across every executor type -- see `Step.timeout_seconds`
+    for the full contract this places on executor authors."""
+    retry: int = 0
+    retry_backoff_seconds: float = 1.0
+    """If `retry > 0`, `ExecutorDispatcher` retries a genuine execution
+    failure (an `ExecutorResult(success=False, ...)`, an unhandled
+    exception, or a timeout) up to this many additional times, with
+    `retry_backoff_seconds` delay before the first retry, doubling each
+    attempt -- see `Step.retry`. Each attempt gets its own fresh
+    `timeout_seconds` budget, not one shared across all of them."""
+
+
+@dataclass(frozen=True)
+class StepCompleted:
+    """Published by the executor dispatcher once a step's executor
+    reports success."""
+
+    workflow_id: str
+    step_id: str
+    output: Mapping[str, Any] | None
+
+
+@dataclass(frozen=True)
+class StepFailed:
+    """Published by the executor dispatcher when a step's executor
+    reports failure, raises, or when no executor is registered for its
+    type."""
+
+    workflow_id: str
+    step_id: str
+    error: str
+
+
+@dataclass(frozen=True)
+class StepSkipped:
+    """Published by the orchestrator for each step that
+    `Workflow.cascade_skip_dependents_of()` marks SKIPPED -- i.e. a step
+    that can never run because it (transitively) depends on a step that
+    failed while `optional` and `blocking`. Purely informational, same
+    spirit as `StepReady`."""
+
+    workflow_id: str
+    step_id: str
+    reason: str
+
+
+@dataclass(frozen=True)
+class StepCancelled:
+    """Published for each step whose task was still in flight when a
+    *required* step failed elsewhere and the orchestrator cancelled
+    every other in-flight step. Distinct from `StepSkipped`: a skipped
+    step never even started; a cancelled step was genuinely RUNNING and
+    got stopped. Purely informational -- doesn't affect the workflow's
+    outcome, which is already decided by the required failure that
+    triggered the cancellation.
+    """
+
+    workflow_id: str
+    step_id: str
+    reason: str
+
+
+@dataclass(frozen=True)
+class WorkflowCompleted:
+    """Published by the orchestrator once the workflow is settled with a
+    successful outcome -- `WorkflowStatus.COMPLETED` or
+    `COMPLETED_WITH_FAILURES`. Check the workflow's own status (e.g. via
+    `WorkflowService.get_status`) to tell which."""
+
+    workflow_id: str
+
+
+@dataclass(frozen=True)
+class WorkflowFailed:
+    """Published by the orchestrator the moment a *required* step fails
+    -- immediately, not waiting for other in-flight steps to settle,
+    since the outcome is already decided at that point."""
+
+    workflow_id: str
+    reason: str
+
+
+@dataclass(frozen=True)
+class WorkflowCancelled:
+    """Published by `Orchestrator.cancel()` once a workflow has been
+    stopped from outside -- the only one of the three "workflow is done"
+    events that isn't triggered by the steps' own outcomes (see
+    `WorkflowStatus.CANCELLED`)."""
+
+    workflow_id: str
+    reason: str

flowlit/application/executor_dispatcher.py

@@ -0,0 +1,112 @@
+"""`ExecutorDispatcher`: the single bus subscriber that routes
+`StepDispatchRequested` events to the right executor.
+
+Rather than every concrete `Executor` subscribing to the bus itself, one
+dispatcher subscribes once and looks the executor up in the
+`ExecutorRegistry` by step type. That keeps `Executor` implementations
+bus-agnostic (trivially unit-testable by calling `execute()` directly)
+and keeps "which step types exist" a single, obvious place to look
+(the registry), rather than scattered across each executor's own
+subscription call.
+
+Deliberately subscribes to `StepDispatchRequested`, not the earlier
+`StepReady` announcement -- see `events.py` for why those are two
+separate event types.
+
+This is also the single choke point every step's execution passes
+through regardless of type -- which is why `Step.timeout_seconds` and
+`Step.retry` are both enforced right here, via `asyncio.wait_for()` and
+a plain retry loop, rather than inside any individual executor:
+wrapping the call here means `noop`, `sleep`, and any custom executor
+someone registers all get the same timeout/retry behavior `shell`/
+`http` would otherwise be the only ones to have, with zero changes to
+any of them. See `flowlit.application.ports.executor.Executor` for the
+cancellation contract this places on executor authors.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+
+from flowlit.application.errors import SpecValidationError, UnknownStepTypeError
+from flowlit.application.events import StepCompleted, StepDispatchRequested, StepFailed
+from flowlit.application.executor_registry import ExecutorRegistry
+from flowlit.application.ports.event_bus import EventBus
+
+_logger = logging.getLogger(__name__)
+
+
+class ExecutorDispatcher:
+    def __init__(self, event_bus: EventBus, registry: ExecutorRegistry) -> None:
+        self._event_bus = event_bus
+        self._registry = registry
+        event_bus.subscribe(StepDispatchRequested, self._on_dispatch_requested)
+
+    async def _on_dispatch_requested(self, event: StepDispatchRequested) -> None:
+        try:
+            executor = self._registry.get(event.step_type)
+        except UnknownStepTypeError as exc:
+            # Never retried -- an unregistered step type fails identically
+            # on every attempt, so retrying it is pure waste.
+            await self._publish_failure(event, str(exc))
+            return
+
+        attempts = event.retry + 1
+        last_error = "unknown error"
+        for attempt in range(attempts):
+            try:
+                if event.timeout_seconds is None:
+                    result = await executor.execute(event.step_id, event.spec)
+                else:
+                    result = await asyncio.wait_for(
+                        executor.execute(event.step_id, event.spec),
+                        timeout=event.timeout_seconds,
+                    )
+            except TimeoutError:
+                # asyncio.wait_for() has already cancelled and fully unwound
+                # the executor's execute() coroutine by the time this
+                # exception reaches us -- see Executor's docstring for what
+                # that requires of the executor itself (release its own
+                # resources on CancelledError, not only its own timeout path).
+                last_error = f"step timed out after {event.timeout_seconds} seconds"
+            except SpecValidationError as exc:
+                # Raised by a TypedExecutor's validate_spec() (see
+                # BaseExecutor.execute()) -- the only point a placeholder-
+                # bearing spec can honestly be checked, since WorkflowService.
+                # submit_plan()'s own eager check skips those. Never
+                # retried either, for the same reason as UnknownStepTypeError
+                # above: an invalid spec is invalid on every attempt.
+                await self._publish_failure(event, f"invalid spec: {exc}")
+                return
+            except Exception as exc:  # noqa: BLE001 - an executor bug must fail the step, not the process
+                last_error = f"executor raised an unexpected error: {exc}"
+            else:
+                if result.success:
+                    await self._event_bus.publish(
+                        StepCompleted(event.workflow_id, event.step_id, result.output)
+                    )
+                    return
+                last_error = result.error or "executor reported failure"
+
+            if attempt < attempts - 1:
+                delay = event.retry_backoff_seconds * (2**attempt)
+                _logger.info(
+                    "step %s: attempt %s/%s failed (%s), retrying in %ss",
+                    event.step_id,
+                    attempt + 1,
+                    attempts,
+                    last_error,
+                    delay,
+                )
+                await asyncio.sleep(delay)
+
+        # Only mention attempt count when there actually was more than
+        # one -- keeps the retry=0 (default) error message identical to
+        # what it was before this feature existed.
+        if attempts > 1:
+            last_error = f"failed after {attempts} attempt(s): {last_error}"
+        await self._publish_failure(event, last_error)
+
+    async def _publish_failure(self, event: StepDispatchRequested, error: str) -> None:
+        await self._event_bus.publish(StepFailed(event.workflow_id, event.step_id, error))

flowlit/application/executor_registry.py

@@ -0,0 +1,26 @@
+"""`ExecutorRegistry`: routes a step type to the `Executor` that handles it.
+
+Extension pattern for adding a new step type: implement the `Executor`
+port (no base class to inherit -- just the matching `execute` method),
+then add one `"my_type": MyExecutor()` entry where the registry is built
+in `flowlit.interfaces.composition_root`. No plugin discovery or
+registration decorators for v1.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+
+from flowlit.application.errors import UnknownStepTypeError
+from flowlit.application.ports.executor import Executor
+
+
+class ExecutorRegistry:
+    def __init__(self, executors_by_step_type: Mapping[str, Executor]) -> None:
+        self._executors_by_step_type = dict(executors_by_step_type)
+
+    def get(self, step_type: str) -> Executor:
+        try:
+            return self._executors_by_step_type[step_type]
+        except KeyError:
+            raise UnknownStepTypeError(step_type) from None