clankloop 0.0.0__py3-none-any.whl

clankloop/core/lts.py

@@ -0,0 +1,330 @@
+#!/usr/bin/env python3
+"""Labeled transition system analysis of an execution graph.
+
+Walks every reachable ``(task, retry-budget-vector, assigned-paths)`` state of
+a compiled :class:`~clankloop.core.graph.ExecutionGraph`, materializing the
+full state space as a labelled transition system (:class:`LabeledTransitionSystem`).
+Both the data-dependency check (:func:`validate_data_dependencies`) and the
+PlantUML renderer project from this generic structure rather than re-walking
+the graph.
+
+The LTS is unbuildable for an ill-formed graph: :func:`analyze` raises
+:class:`~clankloop.core.errors.UnknownTaskError` for a dangling branch target
+and :class:`~clankloop.core.errors.InfiniteLoopError` for an unbounded cycle
+detected during the DFS. These propagate to the caller, which can apply
+user-facing context.
+"""
+
+
+from collections import defaultdict
+from dataclasses import dataclass
+from itertools import takewhile
+from typing import TYPE_CHECKING, Iterator, Mapping, Sequence
+
+from clankloop.core.env import assigned_paths, data_dependencies as env_data_dependencies
+from clankloop.core.errors import (
+    InfiniteLoopError,
+    UnassignedDataDependencyError,
+    UnknownTaskError,
+)
+from clankloop.core.types import (
+    Action,
+    AlwaysCondition,
+    Condition,
+    NextAction,
+    OnFailureCondition,
+    OnRetryCondition,
+    OnSuccessCondition,
+    SetAction,
+)
+
+if TYPE_CHECKING:
+    from clankloop.core.graph import ExecutionGraph
+
+
+@dataclass(frozen=True)
+class LTSNode:
+    """A node in the labeled transition system.
+
+    The state is the task the walker is at, plus the remaining retry budget
+    for each :class:`OnRetryCondition` in the graph (budgets only decrement
+    and are bounded, so the product of tasks and budget vectors is finite and
+    the walk terminates), plus the set of env paths known to be assigned on
+    the path that reached this state. ``assigned`` grows monotonically as
+    producers (SetActions on matched branches) fire, and is part of the
+    state so the data-dependency check is path-sensitive.
+    """
+
+    task: str
+    counters: tuple[int, ...]
+    assigned: frozenset[str] = frozenset()
+
+
+@dataclass(frozen=True)
+class LTSEdge:
+    """A materialized transition in the LTS, labelled with its cause."""
+
+    source: LTSNode
+    target: LTSNode
+    label: str
+
+
+@dataclass(frozen=True)
+class LabeledTransitionSystem:
+    """The full reachable state space of a graph as a labeled transition system.
+
+    Built by :func:`analyze`; consumed by :func:`validate_data_dependencies`
+    (checks) and the PlantUML renderer (rendering). Both project from this
+    generic structure rather than re-walking the graph.
+    """
+
+    start: LTSNode
+    nodes: frozenset[LTSNode]
+    edges: frozenset[LTSEdge]
+
+
+def _validate_targets(graph: "ExecutionGraph") -> None:
+    """Every NextAction on every task must reference a task in the graph.
+
+    Independent of reachability — a dangling target on an unreachable task is
+    still an unbuildable graph. Once this passes, the walker's own target
+    check is a pure safety net that can never fire.
+    """
+    next_actions = (
+        (task_name, action)
+        for task_name, branches in graph.exec_actions.items()
+        for _condition, actions in branches
+        for action in actions
+        if isinstance(action, NextAction)
+    )
+    for task_name, action in next_actions:
+        if action.next_task not in graph.execs:
+            raise UnknownTaskError(
+                f"Task {task_name!r} in pipeline {graph.name!r} "
+                f"branches to unknown task {action.next_task!r}"
+            )
+
+
+class _ConditionSimulator:
+    """Simulates condition matching against synthetic return codes.
+
+    Closes over the retry-condition slot mapping so callers don't thread it
+    through every call. Returns ``(matched, decrement_slot)`` — the slot is
+    the index into the budget vector to decrement on this transition, or
+    ``None`` when the condition consumes no budget.
+    """
+
+    def __init__(self, retry_conditions: list[OnRetryCondition]):
+        self._slot_of = {id(c): i for i, c in enumerate(retry_conditions)}
+
+    def matches(
+        self, condition: Condition, returncode: int, counters: tuple[int, ...]
+    ) -> tuple[bool, int | None]:
+        if isinstance(condition, OnSuccessCondition):
+            return (returncode == 0, None)
+        if isinstance(condition, OnFailureCondition):
+            return (returncode != 0, None)
+        if isinstance(condition, AlwaysCondition):
+            return (True, None)
+        if isinstance(condition, OnRetryCondition):
+            slot = self._slot_of[id(condition)]
+            if returncode != 0 and counters[slot] > 0:
+                return (True, slot)
+            return (False, None)
+        return (False, None)
+
+
+class _Walker:
+    """DFS over the reachable state space, materializing the LTS.
+
+    Holds the mutable accumulators (visited nodes, materialized edges, the
+    DFS stack) and the condition simulator. The DFS skeleton in :meth:`_dfs`
+    is pure graph traversal; :meth:`_transitions` names the "emulate both
+    outcomes" step; :meth:`_resolve_outcome` names the "first matching
+    condition wins" step that mirrors
+    :meth:`ExecutionGraph.execute` branch resolution.
+    """
+
+    def __init__(self, graph: "ExecutionGraph", sim: _ConditionSimulator):
+        self.graph = graph
+        self.sim = sim
+        self.nodes: set[LTSNode] = set()
+        self.edges: set[LTSEdge] = set()
+        self.on_stack: set[LTSNode] = set()
+        self.trail: list[str] = []
+
+    def walk(self, start: LTSNode) -> LabeledTransitionSystem:
+        self._dfs(start)
+        return LabeledTransitionSystem(
+            start=start,
+            nodes=frozenset(self.nodes),
+            edges=frozenset(self.edges),
+        )
+
+    def _dfs(self, state: LTSNode) -> None:
+        if state in self.on_stack:
+            raise InfiniteLoopError(
+                f"Pipeline {self.graph.name!r} has an unbounded cycle: "
+                f"{' -> '.join(self.trail + [state.task])}"
+            )
+        if state in self.nodes:
+            return
+        self.nodes.add(state)
+        self.on_stack.add(state)
+        self.trail.append(state.task)
+        for label, target in self._transitions(state):
+            self.edges.add(LTSEdge(state, target, label))
+            self._dfs(target)
+        self.trail.pop()
+        self.on_stack.discard(state)
+
+    def _transitions(self, state: LTSNode) -> Iterator[tuple[str, LTSNode]]:
+        """Yield ``(label, target)`` for each outcome this state transitions on."""
+        for success in (True, False):
+            resolved = self._resolve_outcome(state, success)
+            if resolved is not None:
+                yield resolved
+
+    def _resolve_outcome(
+        self, state: LTSNode, success: bool
+    ) -> tuple[str, LTSNode] | None:
+        """Find the first matching condition and resolve its successor.
+
+        Mirrors :meth:`ExecutionGraph.execute`: the first matching condition
+        wins; SetActions before the NextAction fold their write-paths into
+        the successor's assigned set; actions after the NextAction never run.
+        A branch that matches but produces no NextAction falls through to the
+        next branch.
+        """
+        returncode = 0 if success else 1
+        for condition, actions in self.graph.exec_actions.get(state.task, []):
+            matched, slot = self.sim.matches(condition, returncode, state.counters)
+            if not matched:
+                continue
+            next_task = self._redirect_target(actions)
+            if next_task is None:
+                continue  # matched but no redirect; try the next branch
+            if next_task not in self.graph.execs:
+                raise UnknownTaskError(
+                    f"Task {state.task!r} in pipeline {self.graph.name!r} "
+                    f"branches to unknown task {next_task!r}"
+                )
+            return (
+                condition.label,
+                LTSNode(
+                    task=next_task,
+                    counters=self._decrement(slot, state.counters),
+                    assigned=state.assigned | self._collect_writes(actions),
+                ),
+            )
+        return None
+
+    @staticmethod
+    def _decrement(slot: int | None, counters: tuple[int, ...]) -> tuple[int, ...]:
+        """Return *counters* with the budget at *slot* decremented, or unchanged."""
+        if slot is None:
+            return counters
+        mutated = list(counters)
+        mutated[slot] -= 1
+        return tuple(mutated)
+
+    @staticmethod
+    def _collect_writes(actions: Sequence[Action]) -> frozenset[str]:
+        """Env paths written by SetActions before the first NextAction."""
+        return frozenset(
+            str(action.var_name)
+            for action in takewhile(lambda a: not isinstance(a, NextAction), actions)
+            if isinstance(action, SetAction)
+        )
+
+    @staticmethod
+    def _redirect_target(actions: Sequence[Action]) -> str | None:
+        """The task name the first NextAction redirects to, or None."""
+        for action in actions:
+            if isinstance(action, NextAction):
+                return action.next_task
+        return None
+
+
+def analyze(
+    graph: "ExecutionGraph",
+    entry_task: str,
+    parameter_paths: frozenset[str] = frozenset(),
+) -> LabeledTransitionSystem:
+    """Walk every reachable state, materializing the labeled transition system.
+
+    Emulates both outcomes (success, failure) per task. State is
+    ``(task, retry-budget-vector, assigned-paths)``. Transitions are
+    materialized as :class:`LTSEdge` with the matched condition's label.
+
+    Raises:
+        UnknownTaskError: If *entry_task* is not in the graph, or a branch
+            targets a task not in the graph (the LTS is unbuildable).
+        InfiniteLoopError: If the DFS revisits a state on its current stack
+            (an unbounded cycle — no budget decrements, so it repeats
+            forever).
+    """
+    if entry_task not in graph.execs:
+        raise UnknownTaskError(
+            f"Entry task {entry_task!r} is not in pipeline {graph.name!r}"
+        )
+    _validate_targets(graph)
+
+    retry_conditions = [
+        cond
+        for branches in graph.exec_actions.values()
+        for cond, _ in branches
+        if isinstance(cond, OnRetryCondition)
+    ]
+    sim = _ConditionSimulator(retry_conditions)
+    start = LTSNode(
+        task=entry_task,
+        counters=tuple(c.retries for c in retry_conditions),
+        assigned=assigned_paths(graph.env) | parameter_paths,
+    )
+
+    return _Walker(graph, sim).walk(start)
+
+
+def data_dependencies(
+    graph: "ExecutionGraph",
+) -> dict[str, frozenset[str]]:
+    """Resolve each task's transitive read set against the graph's env.
+
+    A static fact per task, independent of state. Consumed by
+    :func:`validate_data_dependencies` together with the LTS.
+    """
+    return {
+        name: env_data_dependencies(graph.env, execution.consumes())
+        for name, execution in graph.execs.items()
+    }
+
+
+def validate_data_dependencies(
+    lts: LabeledTransitionSystem,
+    reads: Mapping[str, frozenset[str]],
+) -> None:
+    """Check the LTS against per-task read sets.
+
+    Raises :class:`UnassignedDataDependencyError` if any task's read set is
+    not contained in the intersection of ``assigned`` across all LTS nodes
+    for that task. The intersection is the set of paths guaranteed to reach
+    the task: a key set on some paths but not all is not guaranteed, so a
+    read of it could render against an unassigned value on the unset path.
+    """
+    by_task: dict[str, list[LTSNode]] = defaultdict(list)
+    for node in lts.nodes:
+        by_task[node.task].append(node)
+    violations = [
+        (task, reads.get(task, frozenset()) - frozenset.intersection(*[n.assigned for n in nodes]))
+        for task, nodes in by_task.items()
+    ]
+    missing = [(task, deps) for task, deps in violations if deps]
+    if missing:
+        descriptions = ", ".join(
+            f"{task!r} reads {sorted(deps)}"
+            for task, deps in missing
+        )
+        raise UnassignedDataDependencyError(
+            f"Task(s) read env path(s) not assigned on every path reaching them: {descriptions}"
+        )

clankloop/core/types.py

@@ -0,0 +1,230 @@
+#!/usr/bin/env python3
+"""Domain types and protocols for the execution graph.
+
+Pure data and structural contracts with zero runtime logic: the result of
+executing a task, the IO channels the core tees through, the per-execution
+context, the :class:`Execution` task protocol, and the :class:`Condition` /
+:class:`Action` branching protocols together with their concrete classes.
+
+Nothing here imports the runtime graph or the static analyser — this is the
+vocabulary module everything else builds on.
+"""
+
+
+import logging
+from dataclasses import dataclass
+from typing import Mapping, Protocol, TextIO, runtime_checkable
+
+from clankloop.core.env import EnvPath, Object, UnassignedValueError
+
+logger = logging.getLogger("clankloop")
+
+
+@dataclass
+class ExecutionResult:
+    """Result of executing a task.
+
+    Attributes:
+        stdout: Captured standard output.
+        stderr: Captured standard error.
+        returncode: Process exit code (0 = success).
+    """
+
+    stdout: str
+    stderr: str
+    returncode: int
+
+
+@dataclass(frozen=True)
+class IOChannels:
+    """CLI-owned streams the core tees subprocess output through.
+
+    ``None`` fields mean "discard forwarded output for that stream" (the
+    subprocess is still captured for SetActions/conditions). The default
+    ``IOChannels()`` is non-interactive + discard-forward + capture — today's
+    ``capture_output=True`` behaviour. The CLI supplies real streams for
+    ``run``; ``taskcar`` uses the default so its stdout stays reserved for the
+    JSON result.
+    """
+
+    stdin: TextIO | None = None
+    stdout: TextIO | None = None
+    stderr: TextIO | None = None
+
+
+@dataclass(frozen=True)
+class ExecutionContext:
+    """Per-execution context passed to :meth:`Execution.execute`."""
+
+    env: Object
+    environ: Mapping[str, str]
+    io: IOChannels = IOChannels()
+
+
+@runtime_checkable
+class Execution(Protocol):
+    """A task the graph engine can execute.
+
+    Structural protocol: any object with a ``name`` plus :meth:`consumes` and
+    :meth:`execute` qualifies. Implementations are not required to inherit
+    from this class; the protocol merely documents the contract.
+    """
+
+    name: str
+
+    def consumes(self) -> frozenset[str]:
+        """Direct template identifiers in the command.
+
+        Paths are dotted root-relative strings (e.g. ``pipeline.globals.workdir``).
+        Read unconditionally at execute time when the command template renders,
+        independent of which branch the graph takes afterwards, so the set is a
+        static fact about the task. Export key-paths are deliberately excluded:
+        they are extracted under a tolerant try/except, so an unassigned export
+        is by design not a hard dependency.
+        """
+        ...
+
+    def execute(self, context: ExecutionContext) -> ExecutionResult:
+        ...
+
+
+def extract_env(env: Object, exports: dict[str, EnvPath]) -> dict[str, str]:
+    """Extract values from the environment for subprocess execution."""
+    result = {}
+    for bash_name, key_path in exports.items():
+        try:
+            value = env.evaluate(key_path)
+            result[bash_name] = str(value)
+        except UnassignedValueError:
+            logger.debug(
+                "Failed to extract env variable", extra={"key_path": str(key_path)}
+            )
+
+    return result
+
+
+@runtime_checkable
+class Condition(Protocol):
+    """A condition that decides whether a branch fires.
+
+    Structural protocol: any object with :meth:`matches` qualifies.
+
+    ``label`` is the one presentation member: a short, human-readable cause
+    (e.g. ``"On Success"``) carried on the type so consumers (the LTS walker,
+    the PlantUML renderer) never need ``isinstance`` to name a transition.
+    """
+
+    label: str
+
+    def matches(self, result: ExecutionResult) -> bool:
+        ...
+
+
+class OnSuccessCondition:
+    """Condition that matches when the previous task succeeded (return code 0)."""
+
+    label: str = "On Success"
+
+    def matches(self, result: ExecutionResult) -> bool:
+        return result.returncode == 0
+
+
+class OnFailureCondition:
+    """Condition that matches when the previous task failed (non-zero return code)."""
+
+    label: str = "On Failure"
+
+    def matches(self, result: ExecutionResult) -> bool:
+        return result.returncode != 0
+
+
+class AlwaysCondition:
+    """Condition that always matches, regardless of the previous task's result."""
+
+    label: str = "Always"
+
+    def matches(self, result: ExecutionResult) -> bool:
+        return True
+
+
+class OnRetryCondition:
+    """Condition that matches when the previous task failed and retries remain.
+
+    Holds a mutable decrementing counter initialized at compile time from
+    :class:`~clankloop.loopfile.v2.loopfile.RetryFromAction.retries`. Each
+    successful match consumes one retry; when the counter reaches zero the
+    condition stops matching and execution falls through to the failure branch.
+    """
+
+    label: str = "On Retry"
+
+    def __init__(self, retries: int):
+        self._budget = retries
+        self._remaining = retries
+
+    @property
+    def retries(self) -> int:
+        """The configured retry budget (immutable); the live counter is _remaining."""
+        return self._budget
+
+    def matches(self, result: ExecutionResult) -> bool:
+        if result.returncode == 0:
+            return False
+        if self._remaining > 0:
+            self._remaining -= 1
+            return True
+        return False
+
+
+@runtime_checkable
+class Action(Protocol):
+    """An action that runs when its branch matches.
+
+    Structural protocol: any object with :meth:`next` qualifies.
+    """
+
+    def next(self, env: Object, result: ExecutionResult) -> str | None:
+        ...
+
+
+class SetAction:
+    """Action that sets a variable in the graph's environment."""
+
+    def __init__(
+        self,
+        var_name: EnvPath,
+        value: str | None = None,
+        stdout: bool = False,
+        stderr: bool = False,
+    ):
+        self.var_name = var_name
+        self.value = value
+        self.stdout = stdout
+        self.stderr = stderr
+
+    def next(self, env: Object, result: ExecutionResult) -> str | None:
+        value = None
+        if self.value is not None:
+            value = self.value
+        elif self.stdout:
+            value = result.stdout
+        elif self.stderr:
+            value = result.stderr
+        assert value is not None, "SetAction must specify value, stdout, or stderr"
+        logger.debug(
+            "Setting variable",
+            extra={
+                "variable": self.var_name,
+                "value": value,
+            },
+        )
+        env.set_value(self.var_name, value)
+        return None
+
+
+class NextAction:
+    def __init__(self, next_task: str):
+        self.next_task = next_task
+
+    def next(self, env: Object, result: ExecutionResult) -> str | None:
+        return self.next_task