hiveplane 0.1.0__py3-none-any.whl

hiveplane/__init__.py

@@ -0,0 +1,12 @@
+"""HivePlane — control plane for production agent fleets.
+
+Register agents, certify them against a reproducible benchmark, admit only
+certified agents to production, run them under budget/policy/sandbox, intervene
+on live runs, deliver results, and observe the fleet.
+"""
+
+from __future__ import annotations
+
+__version__ = "0.1.0"
+
+__all__ = ["__version__"]

hiveplane/adapters/__init__.py

@@ -0,0 +1,49 @@
+"""Runtime adapters that translate between control-plane concepts and runtimes (M16-M17)."""
+
+from __future__ import annotations
+
+from hiveplane.adapters.base import Adapter, AdapterRunExecutor
+from hiveplane.adapters.errors import (
+    AdapterError,
+    EntrypointLoadError,
+    MissingAdapterDependencyError,
+    RunCancelledError,
+    RunTerminatedError,
+    ToolCallBlockedError,
+    ToolCallDeniedError,
+    ToolCallEscalatedError,
+    UnsupportedAdapterError,
+    WorkerError,
+)
+from hiveplane.adapters.graph import CompiledGraph, GraphSnapshot
+from hiveplane.adapters.langgraph import LangGraphAdapter
+from hiveplane.adapters.loader import Entrypoint, EntrypointLoader
+from hiveplane.adapters.raw_worker import RawWorkerAdapter
+from hiveplane.adapters.reporter import RunReporter
+from hiveplane.adapters.stub import StubAdapter
+from hiveplane.adapters.worker import RunControl, WorkerContext
+
+__all__ = [
+    "Adapter",
+    "AdapterError",
+    "AdapterRunExecutor",
+    "CompiledGraph",
+    "Entrypoint",
+    "EntrypointLoadError",
+    "EntrypointLoader",
+    "GraphSnapshot",
+    "LangGraphAdapter",
+    "MissingAdapterDependencyError",
+    "RawWorkerAdapter",
+    "RunCancelledError",
+    "RunControl",
+    "RunReporter",
+    "RunTerminatedError",
+    "StubAdapter",
+    "ToolCallBlockedError",
+    "ToolCallDeniedError",
+    "ToolCallEscalatedError",
+    "UnsupportedAdapterError",
+    "WorkerContext",
+    "WorkerError",
+]

hiveplane/adapters/base.py

@@ -0,0 +1,96 @@
+"""The runtime adapter contract (M16, DD-02, DD-14).
+
+Adapters translate the control-plane contract to a concrete runtime. They report
+state transitions, tool calls, and usage; they never decide policy. Every tool
+call routes through the control-plane tool boundary, and every model call is
+checked against the certification attestation during admission.
+
+:class:`Adapter` is the contract the raw-worker and LangGraph adapters implement
+(#43, #44). :class:`AdapterRunExecutor` bridges an adapter onto the run
+lifecycle's ``RunExecutor`` seam so the lifecycle never sees adapter internals.
+"""
+
+from __future__ import annotations
+
+from typing import Protocol, runtime_checkable
+
+from hiveplane.core.run import RunState
+from hiveplane.core.usage import UsageReport
+from hiveplane.core.workload import AgentWorkload
+from hiveplane.execution.models import RunContext
+from hiveplane.execution.tools import ToolCallResult
+
+
+@runtime_checkable
+class Adapter(Protocol):
+    """The typed contract every runtime adapter implements."""
+
+    def register(self, workload: AgentWorkload) -> None:
+        """Bind the adapter to a workload manifest before any run starts."""
+        ...
+
+    def submit(self, context: RunContext) -> None:
+        """Start a run; the adapter reports transitions and usage from here."""
+        ...
+
+    def pause(self, run_id: str) -> bool:
+        """Cooperate with a pause request, returning whether it was accepted."""
+        ...
+
+    def resume(self, run_id: str) -> bool:
+        """Resume a paused run, returning whether it was accepted."""
+        ...
+
+    def cancel(self, run_id: str) -> None:
+        """Stop a run and release its resources."""
+        ...
+
+    def status(self, run_id: str) -> RunState:
+        """Return the adapter's honest view of the run state."""
+        ...
+
+    def usage(self, run_id: str) -> UsageReport | None:
+        """Return usage since the last report, if any."""
+        ...
+
+    def tool_calls(self, run_id: str) -> list[ToolCallResult]:
+        """Return the tool calls the adapter routed through the boundary."""
+        ...
+
+
+class AdapterRunExecutor:
+    """Adapts an :class:`Adapter` to the run lifecycle's ``RunExecutor`` seam."""
+
+    def __init__(self, adapter: Adapter) -> None:
+        self._adapter = adapter
+
+    def start(self, context: RunContext) -> None:
+        """Delegate run start to the adapter's ``submit``."""
+        self._adapter.submit(context)
+
+    def reattach(self, context: RunContext) -> bool:
+        """Delegate a post-restart re-attach to the adapter, if it supports one."""
+        reattach = getattr(self._adapter, "reattach", None)
+        if reattach is None:
+            return False
+        return bool(reattach(context))
+
+    def pause(self, run_id: str) -> bool:
+        """Delegate a pause request to the adapter."""
+        return self._adapter.pause(run_id)
+
+    def resume(self, run_id: str) -> bool:
+        """Delegate a resume request to the adapter."""
+        return self._adapter.resume(run_id)
+
+    def cancel(self, run_id: str) -> None:
+        """Delegate a cancel request to the adapter."""
+        self._adapter.cancel(run_id)
+
+    def status(self, run_id: str) -> RunState:
+        """Return the adapter's reported run state."""
+        return self._adapter.status(run_id)
+
+    def usage(self, run_id: str) -> UsageReport | None:
+        """Return the adapter's pending usage report, if any."""
+        return self._adapter.usage(run_id)

hiveplane/adapters/dispatch.py

@@ -0,0 +1,95 @@
+"""Per-workload adapter dispatch (M23, #109).
+
+A single control plane may host workloads that use different runtime adapters
+(e.g. repo-agent on raw-worker, docs-agent on langgraph). :class:`DispatchingAdapter`
+implements the :class:`~hiveplane.adapters.base.Adapter` contract by routing each
+workload to the adapter selected by ``spec.runtime.adapter`` and remembering which
+adapter owns each run so per-run operations reach the right runtime.
+"""
+
+from __future__ import annotations
+
+import threading
+from collections.abc import Mapping
+
+from hiveplane.adapters.base import Adapter
+from hiveplane.adapters.errors import UnsupportedAdapterError
+from hiveplane.core.run import RunState
+from hiveplane.core.spec import RuntimeAdapter
+from hiveplane.core.usage import UsageReport
+from hiveplane.core.workload import AgentWorkload
+from hiveplane.execution.models import RunContext
+from hiveplane.execution.tools import ToolCallResult
+
+
+class DispatchingAdapter:
+    """Routes runs to the adapter named by each workload's manifest."""
+
+    def __init__(self, adapters: Mapping[RuntimeAdapter, Adapter]) -> None:
+        self._adapters: dict[RuntimeAdapter, Adapter] = dict(adapters)
+        self._selected: dict[str, Adapter] = {}
+        self._lock = threading.Lock()
+
+    @property
+    def adapters(self) -> Mapping[RuntimeAdapter, Adapter]:
+        """Return the configured adapter per runtime selection."""
+        return dict(self._adapters)
+
+    def register(self, workload: AgentWorkload) -> None:
+        """Register ``workload`` with the adapter its manifest selects."""
+        self._adapter_for(workload).register(workload)
+
+    def submit(self, context: RunContext) -> None:
+        """Route a run to its workload's adapter and remember the owner."""
+        adapter = self._adapter_for(context.workload)
+        with self._lock:
+            self._selected[context.run.id] = adapter
+        adapter.submit(context)
+
+    def reattach(self, context: RunContext) -> bool:
+        """Route a recovered run to its workload's adapter and remember the owner."""
+        adapter = self._adapter_for(context.workload)
+        with self._lock:
+            self._selected[context.run.id] = adapter
+        reattach = getattr(adapter, "reattach", None)
+        if reattach is None:
+            return False
+        return bool(reattach(context))
+
+    def pause(self, run_id: str) -> bool:
+        """Pause the run on the adapter that owns it."""
+        return self._owner(run_id).pause(run_id)
+
+    def resume(self, run_id: str) -> bool:
+        """Resume the run on the adapter that owns it."""
+        return self._owner(run_id).resume(run_id)
+
+    def cancel(self, run_id: str) -> None:
+        """Cancel the run on the adapter that owns it."""
+        self._owner(run_id).cancel(run_id)
+
+    def status(self, run_id: str) -> RunState:
+        """Return the owning adapter's view of a run's state."""
+        return self._owner(run_id).status(run_id)
+
+    def usage(self, run_id: str) -> UsageReport | None:
+        """Return usage from the adapter that owns the run."""
+        return self._owner(run_id).usage(run_id)
+
+    def tool_calls(self, run_id: str) -> list[ToolCallResult]:
+        """Return tool calls from the adapter that owns the run."""
+        return self._owner(run_id).tool_calls(run_id)
+
+    def _adapter_for(self, workload: AgentWorkload) -> Adapter:
+        selected = workload.spec.runtime.adapter
+        adapter = self._adapters.get(selected)
+        if adapter is None:
+            raise UnsupportedAdapterError(selected.value)
+        return adapter
+
+    def _owner(self, run_id: str) -> Adapter:
+        with self._lock:
+            adapter = self._selected.get(run_id)
+        if adapter is None:
+            raise UnsupportedAdapterError(f"no submitted run {run_id!r}")
+        return adapter

hiveplane/adapters/errors.py

@@ -0,0 +1,91 @@
+"""Adapter and worker domain errors (M16)."""
+
+from __future__ import annotations
+
+from hiveplane.core.run import RunState
+from hiveplane.execution.tools import ToolCallResult
+
+
+class AdapterError(Exception):
+    """Base class for runtime adapter errors."""
+
+
+class UnsupportedAdapterError(AdapterError):
+    """Raised when an adapter is asked to run a workload it does not support."""
+
+    def __init__(self, adapter: str) -> None:
+        super().__init__(f"adapter {adapter!r} is not supported by this runtime")
+        self.adapter = adapter
+
+
+class EntrypointLoadError(AdapterError):
+    """Raised when a workload entrypoint cannot be resolved or imported."""
+
+    def __init__(self, entrypoint: str, reason: str) -> None:
+        super().__init__(f"cannot load entrypoint {entrypoint!r}: {reason}")
+        self.entrypoint = entrypoint
+        self.reason = reason
+
+
+class MissingAdapterDependencyError(AdapterError):
+    """Raised when an adapter's optional runtime dependency is not installed."""
+
+    def __init__(self, adapter: str, reason: str) -> None:
+        super().__init__(f"adapter {adapter!r} is unavailable: {reason}")
+        self.adapter = adapter
+        self.reason = reason
+
+
+class WorkerError(AdapterError):
+    """Base class for errors a worker raises through the control plane."""
+
+
+class ToolCallDeniedError(WorkerError):
+    """Raised when the tool boundary denies a tool call."""
+
+    def __init__(self, result: ToolCallResult) -> None:
+        super().__init__(f"tool call {result.tool_id!r} denied: {result.reason}")
+        self.result = result
+
+
+class ToolCallBlockedError(WorkerError):
+    """Raised when the tool boundary blocks a tool call for injection."""
+
+    def __init__(self, result: ToolCallResult) -> None:
+        super().__init__(f"tool call {result.tool_id!r} blocked: {result.reason}")
+        self.result = result
+
+
+class ToolCallEscalatedError(WorkerError):
+    """Raised when a tool call requires approval and the run is paused."""
+
+    def __init__(self, result: ToolCallResult) -> None:
+        super().__init__(f"tool call {result.tool_id!r} requires approval")
+        self.result = result
+
+
+class RunTerminatedError(WorkerError):
+    """Raised when the run reached a terminal state during a report."""
+
+    def __init__(self, state: RunState) -> None:
+        super().__init__(f"run is {state.value}")
+        self.state = state
+
+
+class RunCancelledError(WorkerError):
+    """Raised at a checkpoint when the run has been cancelled."""
+
+    def __init__(self, state: RunState = RunState.CANCELLED) -> None:
+        super().__init__(f"run is {state.value}")
+        self.state = state
+
+
+class ModelIdentityMismatchError(WorkerError):
+    """Raised when inference reports a model other than the bound identity (T11)."""
+
+    def __init__(self, expected: str, actual: str) -> None:
+        super().__init__(
+            f"model identity mismatch: expected {expected!r}, provider reported {actual!r}"
+        )
+        self.expected = expected
+        self.actual = actual

hiveplane/adapters/graph.py

@@ -0,0 +1,40 @@
+"""The minimal compiled-graph surface the LangGraph adapter depends on (M17).
+
+Keeping this as a Protocol lets the adapter stay typed without importing
+langgraph at module import time; the concrete object is duck-typed at runtime.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterator
+from typing import Any, Protocol, runtime_checkable
+
+
+@runtime_checkable
+class GraphSnapshot(Protocol):
+    """A point-in-time view of a compiled graph's state."""
+
+    @property
+    def next(self) -> tuple[str, ...]:
+        """Nodes scheduled to run next; empty when the graph is finished."""
+        ...
+
+    @property
+    def values(self) -> dict[str, Any]:
+        """The current graph state values."""
+        ...
+
+
+@runtime_checkable
+class CompiledGraph(Protocol):
+    """The subset of a compiled LangGraph the adapter drives."""
+
+    def stream(
+        self, payload: Any, config: dict[str, Any], *, stream_mode: str = "values"
+    ) -> Iterator[dict[str, Any]]:
+        """Stream state values as the graph advances."""
+        ...
+
+    def get_state(self, config: dict[str, Any]) -> GraphSnapshot:
+        """Return the current state snapshot for a thread."""
+        ...

hiveplane/adapters/langgraph.py

@@ -0,0 +1,306 @@
+"""The LangGraph adapter: wraps a compiled graph behind the Adapter contract (M17).
+
+The graph runs on the configured spawner. Supersteps are streamed so the worker
+can checkpoint cooperatively (operator pause/resume/cancel), and a LangGraph
+``__interrupt__`` maps to a paused run that resumes with ``Command(resume=...)``.
+"""
+
+from __future__ import annotations
+
+import threading
+from collections.abc import Callable
+from contextlib import suppress
+from datetime import UTC, datetime
+from typing import Any, cast
+
+from hiveplane import telemetry
+from hiveplane.adapters.errors import (
+    EntrypointLoadError,
+    MissingAdapterDependencyError,
+    RunCancelledError,
+    ToolCallEscalatedError,
+    UnsupportedAdapterError,
+    WorkerError,
+)
+from hiveplane.adapters.graph import CompiledGraph
+from hiveplane.adapters.loader import EntrypointLoader
+from hiveplane.adapters.reporter import RunReporter
+from hiveplane.adapters.worker import RunControl, WorkerContext
+from hiveplane.budget.pricing import CostTable
+from hiveplane.core.event import EventType
+from hiveplane.core.run import RunState
+from hiveplane.core.spec import RuntimeAdapter
+from hiveplane.core.usage import UsageReport
+from hiveplane.core.workload import AgentWorkload
+from hiveplane.execution.errors import IllegalTransitionError
+from hiveplane.execution.models import RunContext
+from hiveplane.execution.tools import ToolCallResult, ToolGateway
+from hiveplane.llm.provider import LLMProvider
+
+_INTERRUPT_KEY = "__interrupt__"
+
+#: Runs a unit of work; the default spawns a daemon thread.
+Spawner = Callable[[Callable[[], None]], None]
+
+
+def _require_command() -> Callable[..., Any]:
+    """Return LangGraph's ``Command``, or raise if the extra is not installed."""
+    try:
+        from langgraph.types import Command
+    except ImportError as exc:
+        raise MissingAdapterDependencyError("langgraph", str(exc)) from exc
+    return Command
+
+
+def _build_command(**kwargs: Any) -> Any:
+    """Build a LangGraph ``Command`` lazily, so the extra is only needed on resume."""
+    command = _require_command()
+    return command(**kwargs)
+
+
+def _thread_spawner(work: Callable[[], None]) -> None:
+    threading.Thread(target=work, daemon=True).start()
+
+
+def _configurable(run_id: str, ctx: WorkerContext) -> dict[str, Any]:
+    return {"configurable": {"thread_id": run_id, "hiveplane_ctx": ctx}}
+
+
+class LangGraphAdapter:
+    """Runs a compiled LangGraph and reports through the control-plane boundary."""
+
+    def __init__(
+        self,
+        reporter: RunReporter,
+        tools: ToolGateway,
+        loader: EntrypointLoader,
+        *,
+        clock: Callable[[], datetime] | None = None,
+        spawner: Spawner | None = None,
+        command_factory: Callable[..., Any] | None = None,
+        provider: LLMProvider | None = None,
+        cost_table: CostTable | None = None,
+    ) -> None:
+        self._reporter = reporter
+        self._tools = tools
+        self._loader = loader
+        self._clock = clock or (lambda: datetime.now(UTC))
+        self._spawner = spawner or _thread_spawner
+        self._command_factory: Callable[..., Any] = command_factory or _build_command
+        self._provider = provider
+        self._cost_table = cost_table
+        self._lock = threading.Lock()
+        self._graphs: dict[str, CompiledGraph] = {}
+        self._sessions: dict[str, tuple[RunContext, WorkerContext, RunControl]] = {}
+        self._states: dict[str, RunState] = {}
+        self._usage: dict[str, UsageReport | None] = {}
+        self._interrupted: dict[str, bool] = {}
+        self._escalated: dict[str, bool] = {}
+
+    def register(self, workload: AgentWorkload) -> None:
+        """Load and cache a workload's compiled graph, rejecting other adapters."""
+        if workload.spec.runtime.adapter is not RuntimeAdapter.LANGGRAPH:
+            raise UnsupportedAdapterError(workload.spec.runtime.adapter.value)
+        graph = self._loader.load_object(workload.spec.runtime.entrypoint)
+        for attr in ("stream", "get_state"):
+            if not callable(getattr(graph, attr, None)):
+                raise EntrypointLoadError(
+                    workload.spec.runtime.entrypoint, f"graph has no {attr}()"
+                )
+        self._graphs[workload.name] = cast("CompiledGraph", graph)
+
+    def submit(self, context: RunContext) -> None:
+        """Start the graph on the configured spawner and return immediately."""
+        graph = self._graph(context.workload)
+        control = RunControl()
+        ctx = WorkerContext(
+            run=context.run,
+            workload=context.workload,
+            sandbox=context.sandbox,
+            tools=self._tools,
+            reporter=self._reporter,
+            control=control,
+            tool_calls=[],
+            clock=self._clock,
+            provider=self._provider,
+            cost_table=self._cost_table,
+        )
+        with self._lock:
+            self._states[context.run.id] = RunState.RUNNING
+            self._sessions[context.run.id] = (context, ctx, control)
+            self._usage[context.run.id] = None
+            self._interrupted[context.run.id] = False
+            self._escalated[context.run.id] = False
+        self._spawner(
+            telemetry.propagate_context(
+                lambda: self._drive(graph, context, ctx, {"task": dict(context.run.task)})
+            )
+        )
+
+    def pause(self, run_id: str) -> bool:
+        """Request a cooperative pause between supersteps."""
+        session = self._sessions.get(run_id)
+        if session is None:
+            return False
+        session[2].pause()
+        return True
+
+    def reattach(self, context: RunContext) -> bool:
+        """Rebuild a recovered paused graph run from its durable checkpoint (#111).
+
+        The graph state lives in the checkpointer (durable as of #122), so only
+        the in-process session needs rebuilding; resume drives the graph again
+        with ``Command(resume=True)`` to continue from its last checkpoint.
+        """
+        run = context.run
+        self._graph(context.workload)
+        control = RunControl()
+        ctx = WorkerContext(
+            run=run,
+            workload=context.workload,
+            sandbox=context.sandbox,
+            tools=self._tools,
+            reporter=self._reporter,
+            control=control,
+            tool_calls=[],
+            clock=self._clock,
+            provider=self._provider,
+            cost_table=self._cost_table,
+        )
+        with self._lock:
+            self._states[run.id] = RunState.PAUSED
+            self._sessions[run.id] = (context, ctx, control)
+            self._usage[run.id] = None
+            self._interrupted[run.id] = True
+            self._escalated[run.id] = False
+        return True
+
+    def resume(self, run_id: str) -> bool:
+        """Resume a paused run: review interrupt, cooperative pause, or escalation."""
+        session = self._sessions.get(run_id)
+        if session is None:
+            return False
+        context, ctx, control = session
+        if self._escalated.get(run_id):
+            # Re-dispatch (#129): the escalated tool call carries an approved
+            # approval now, so drive the graph again from its initial task.
+            graph = self._graph(context.workload)
+            with self._lock:
+                self._escalated[run_id] = False
+                self._states[run_id] = RunState.RUNNING
+            control.resume()  # clear the pause the escalation set
+            self._spawner(
+                telemetry.propagate_context(
+                    lambda: self._drive(
+                        graph, context, ctx, {"task": dict(context.run.task)}
+                    )
+                )
+            )
+            return True
+        if self._interrupted.get(run_id):
+            command = self._command_factory(resume=True)
+            graph = self._graph(context.workload)
+            with self._lock:
+                self._interrupted[run_id] = False
+                self._states[run_id] = RunState.RUNNING
+            self._spawner(
+                telemetry.propagate_context(lambda: self._drive(graph, context, ctx, command))
+            )
+        else:
+            control.resume()
+        return True
+
+    def cancel(self, run_id: str) -> None:
+        """Request cancellation; the run state is owned by the control plane."""
+        session = self._sessions.get(run_id)
+        if session is not None:
+            session[2].cancel()
+
+    def status(self, run_id: str) -> RunState:
+        """Return the adapter's last recorded state."""
+        return self._states.get(run_id, RunState.QUEUED)
+
+    def usage(self, run_id: str) -> UsageReport | None:
+        """Return the last usage report seen for the run."""
+        return self._usage.get(run_id)
+
+    def tool_calls(self, run_id: str) -> list[ToolCallResult]:
+        """Return the tool calls the graph routed through the boundary."""
+        session = self._sessions.get(run_id)
+        return list(session[1].tool_calls) if session is not None else []
+
+    def _graph(self, workload: AgentWorkload) -> CompiledGraph:
+        graph = self._graphs.get(workload.name)
+        if graph is None:
+            self.register(workload)
+            graph = self._graphs[workload.name]
+        return graph
+
+    def _drive(
+        self,
+        graph: CompiledGraph,
+        context: RunContext,
+        ctx: WorkerContext,
+        payload: Any,
+    ) -> None:
+        run = context.run
+        config = _configurable(run.id, ctx)
+        with telemetry.span("execution", run=run, workload=context.workload) as active:
+            try:
+                for chunk in graph.stream(payload, config, stream_mode="values"):
+                    if _INTERRUPT_KEY in chunk:
+                        active.set_attribute("outcome", "paused")
+                        self._paused(run.id)
+                        return
+                    ctx.checkpoint()
+            except RunCancelledError:
+                active.set_attribute("outcome", "cancelled")
+                return
+            except ToolCallEscalatedError:
+                active.set_attribute("outcome", "escalated")
+                # The gateway already paused the run at the service level; only
+                # record internal state so resume() re-drives (#129).
+                with self._lock:
+                    self._escalated[run.id] = True
+                    self._states[run.id] = RunState.PAUSED
+                self._reporter.record_event(
+                    run.id, EventType.OPERATOR_ACTION, "adapter", detail="tool call escalated"
+                )
+                return
+            except WorkerError as exc:
+                active.set_attribute("outcome", "failed")
+                self._fail(run.id, str(exc))
+                return
+            except Exception as exc:
+                active.set_attribute("outcome", "failed")
+                self._fail(run.id, f"{type(exc).__name__}: {exc}")
+                return
+            snapshot = graph.get_state(config)
+            if snapshot.next:
+                active.set_attribute("outcome", "paused")
+                self._paused(run.id)
+                return
+            active.set_attribute("outcome", "completed")
+            with self._lock:
+                self._states[run.id] = RunState.COMPLETED
+            self._reporter.transition(
+                run.id, RunState.COMPLETED, actor="adapter", result=snapshot.values
+            )
+
+    def _paused(self, run_id: str) -> None:
+        with self._lock:
+            self._interrupted[run_id] = True
+            self._states[run_id] = RunState.PAUSED
+        self._reporter.transition(run_id, RunState.PAUSED, actor="adapter", detail="interrupted")
+
+    def _fail(self, run_id: str, reason: str) -> None:
+        with suppress(IllegalTransitionError):
+            self._reporter.transition(
+                run_id,
+                RunState.FAILED,
+                actor="adapter",
+                detail=reason,
+                failure_reason=reason,
+            )
+        with self._lock:
+            self._states[run_id] = RunState.FAILED