reactifact 0.6.0__py3-none-any.whl

reactifact/testing/fault.py

@@ -0,0 +1,164 @@
+"""Tool-level fault injection and call recording for `reactifact.testing`.
+
+reactifact has no central tool registry — a `Tool` lives in a plain
+`dict[str, Tool]` on whichever `ToolUse`/`ToolUseHITL` produce instance an
+agent wires up (`reactifact/tool_use.py`). To inject a fault (or just record
+calls) for a tool by name, this module scans every agent's `produces` for
+such a dict and wraps the matching `Tool.execute` in place, restoring the
+original object afterward.
+
+**Important caveat**: `_ToolLoopBase._run_tool` (`reactifact/tool_use.py`)
+catches any exception raised by `Tool.execute` and turns it into a plain text
+string handed back to the LLM (`"Tool 'x' failed: ..."`) — it never
+propagates out of the produce. So a fault injected via `lab.fail(...)` does
+**not** abort the scenario run: the agent's LLM sees a tool-failure message
+and may retry, give up, or answer anyway, exactly like a real transient tool
+failure. `result.tools.called(name)` will show the failed call (`.error` set)
+while `result.errors.none()` can still legitimately pass — the agent didn't
+crash, it just saw an error and continued. Tools invoked directly from custom
+`Produce` code (outside a `ToolUse` loop) are not affected by this
+swallowing: an injected exception propagates normally there.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Iterator, Sequence
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Any
+
+from reactifact.tools import Tool, ToolOutput
+
+if TYPE_CHECKING:
+    from reactifact.agents import Agent
+
+
+@dataclass
+class ToolFault:
+    """A queued fault for one tool name.
+
+    `times=None` (default) raises on every call; `times=N` raises for the
+    first `N` calls, then delegates to the real tool — the natural shape for
+    testing "fails then recovers on retry" behavior.
+    """
+
+    tool_name: str
+    error: BaseException | Callable[[], BaseException]
+    times: int | None = None
+
+
+@dataclass
+class ToolCallRecord:
+    """One recorded tool invocation, real or faulted."""
+
+    tool: str
+    args: dict[str, Any]
+    output: ToolOutput | None
+    error: str | None
+
+
+class ToolCallRecorder:
+    """Collects `ToolCallRecord`s. Installed on every run, fault or not."""
+
+    def __init__(self) -> None:
+        self.calls: list[ToolCallRecord] = []
+
+    def record(
+        self,
+        tool: str,
+        args: dict[str, Any],
+        output: ToolOutput | None,
+        error: str | None,
+    ) -> None:
+        self.calls.append(
+            ToolCallRecord(tool=tool, args=args, output=output, error=error)
+        )
+
+
+def _iter_tool_dicts(agents: Sequence[Agent]) -> Iterator[dict[str, Tool]]:
+    """Yields every `dict[str, Tool]`-shaped mapping found on any agent produce.
+
+    Duck-typed (checks `.execute` on the values) rather than importing the
+    private `_ToolLoopBase` class, so it keeps working if that internal is
+    renamed or restructured.
+    """
+    for agent in agents:
+        for produce_obj in getattr(agent, "produces", None) or []:
+            tools = getattr(produce_obj, "tools", None)
+            if (
+                isinstance(tools, dict)
+                and tools
+                and all(hasattr(v, "execute") for v in tools.values())
+            ):
+                yield tools
+
+
+class _WrappedTool(Tool):
+    """Wraps one `Tool` instance: records every call, and raises `fault`'s
+    error for its first `fault.times` calls (or forever, if `times=None`)."""
+
+    def __init__(
+        self, inner: Tool, fault: ToolFault | None, recorder: ToolCallRecorder
+    ) -> None:
+        self.name = inner.name
+        self.description = inner.description
+        self.destructive = inner.destructive
+        self.schema = inner.schema
+        self._inner = inner
+        self._fault = fault
+        self._remaining = fault.times if fault is not None else None
+        self._recorder = recorder
+
+    async def execute(self, args: dict[str, Any]) -> ToolOutput:
+        fault = self._fault
+        if fault is not None and (self._remaining is None or self._remaining > 0):
+            if self._remaining is not None:
+                self._remaining -= 1
+            err = fault.error() if callable(fault.error) else fault.error
+            self._recorder.record(self.name, args, None, str(err))
+            raise err
+        try:
+            output = await self._inner.execute(args)
+        except Exception as exc:
+            self._recorder.record(self.name, args, None, str(exc))
+            raise
+        self._recorder.record(self.name, args, output, output.error or None)
+        return output
+
+
+def _wrap(tool: Tool, fault: ToolFault | None, recorder: ToolCallRecorder) -> Tool:
+    return _WrappedTool(tool, fault, recorder)
+
+
+class FaultInstaller:
+    """Context manager: wraps matching tools for the duration of one run.
+
+    Restores the exact original `Tool` objects in `__exit__`, even if the
+    wrapped run raises.
+    """
+
+    def __init__(
+        self,
+        agents: Sequence[Agent],
+        faults: list[ToolFault],
+        recorder: ToolCallRecorder,
+    ) -> None:
+        self._agents = agents
+        self._faults = {f.tool_name: f for f in faults}
+        self._recorder = recorder
+        self._originals: list[tuple[dict[str, Tool], str, Tool]] = []
+
+    def __enter__(self) -> FaultInstaller:
+        seen: set[int] = set()
+        for tools in _iter_tool_dicts(self._agents):
+            if id(tools) in seen:
+                continue
+            seen.add(id(tools))
+            for name, tool in list(tools.items()):
+                self._originals.append((tools, name, tool))
+                tools[name] = _wrap(tool, self._faults.get(name), self._recorder)
+        return self
+
+    def __exit__(self, *exc: object) -> None:
+        for tools, name, original in self._originals:
+            tools[name] = original
+        self._originals.clear()

reactifact/testing/lab.py

@@ -0,0 +1,350 @@
+"""`ScenarioLab` — the single entry point for `reactifact.testing`.
+
+Ties `fault.py` (tool fault injection + call recording), `mock.py` (fault
+injection for any other resource — the LLM, the embedder, a source),
+`record.py` (record/replay LLM wrapping) and `assertions.py` (chained
+assertions) into one "seed some artifacts, run the agents, assert on what
+happened" call:
+
+    lab = ScenarioLab([my_agent], resources=lambda: build_resources())
+    lab.fail("search", TimeoutError("boom"), times=1)
+    lab.fail_resource("llm", ConnectionError("model unreachable"))
+    result = await lab.run(Question(text="..."))
+    result.artifacts(Answer).exists()
+    result.tools.called("search")
+    result.errors.none()
+
+A fresh `Context`/`Runtime` is built on every `run()` — scenarios never share
+state, so a queued fault or the tool-call recorder can't leak from one `run()`
+into the next.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Sequence
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, TypeVar
+
+from pydantic import BaseModel
+
+from reactifact.agents import Agent
+from reactifact.budget import Budget, RunStats
+from reactifact.context import Context
+from reactifact.resources import RuntimeResources
+from reactifact.runtime import Runtime
+from reactifact.tracing.models import RunTrace
+from reactifact.tracing.tracer import Tracer
+
+from .assertions import (
+    ArtifactAssertions,
+    ErrorAssertions,
+    LLMAssertions,
+    PathAssertions,
+    ToolAssertions,
+)
+from .fault import FaultInstaller, ToolCallRecord, ToolCallRecorder, ToolFault
+from .mock import ResourceFault, ResourceFaultInstaller
+from .record import Mode, wrap_llm
+
+T = TypeVar("T", bound=BaseModel)
+
+
+class _CapturingTracer(Tracer):
+    """Captures the single `RunTrace` a scenario's one `arun()` produces."""
+
+    def __init__(self) -> None:
+        super().__init__()
+        self.trace: RunTrace | None = None
+
+    async def on_turn_end(self, trace: RunTrace) -> None:
+        self.trace = trace
+
+
+@dataclass
+class ScenarioResult:
+    """Everything a scenario assertion needs, read off one `ScenarioLab.run()`.
+
+    `context`/`stats` are exposed directly for anything the assertion groups
+    don't cover; the properties below are the intended entry points.
+    """
+
+    context: Context
+    stats: RunStats | None
+    trace: RunTrace | None
+    calls: list[ToolCallRecord]
+
+    def artifacts(self, artifact_type: type[T]) -> ArtifactAssertions[T]:
+        return ArtifactAssertions(self.context, artifact_type)
+
+    @property
+    def tools(self) -> ToolAssertions:
+        return ToolAssertions(self.calls)
+
+    @property
+    def path(self) -> PathAssertions:
+        return PathAssertions(self.trace)
+
+    @property
+    def llm(self) -> LLMAssertions:
+        return LLMAssertions(self.trace)
+
+    @property
+    def errors(self) -> ErrorAssertions:
+        return ErrorAssertions(self.trace, self.stats)
+
+
+class ScenarioLab:
+    """Runs `agents` against seeded artifacts, once per `run()` call.
+
+    `mode` controls the LLM behind `resources.llm` (see `reactifact.testing.record`):
+    - `"live"` (default): the real provider, unchanged.
+    - `"record"`: wraps it in `ReplayLLM(mode="record")`, appending every call
+      to `recording_path`.
+    - `"replay"`: wraps it in `ReplayLLM(mode="replay")` — no live network
+      calls; a call that diverges from the recording raises `ReplayMiss`
+      rather than being answered with a guess (§59).
+
+    `isolate_errors` defaults to `True` here (unlike `Runtime`'s own default)
+    so a scenario can assert on a crashed agent via `result.errors` instead of
+    the whole `run()` raising; pass `False` to let an unexpected agent
+    exception fail the test the ordinary way.
+    """
+
+    def __init__(
+        self,
+        agents: list[Agent],
+        *,
+        resources: RuntimeResources | Callable[[], RuntimeResources] | None = None,
+        budget: Budget | None = None,
+        max_concurrency: int | None = None,
+        isolate_errors: bool = True,
+        mode: Mode = "live",
+        recording_path: str | Path = "scenario_calls.jsonl",
+    ) -> None:
+        self._agents = agents
+        self._resources = resources
+        self._budget = budget
+        self._max_concurrency = max_concurrency
+        self._isolate_errors = isolate_errors
+        self._mode: Mode = mode
+        self._recording_path = Path(recording_path)
+        self._faults: list[ToolFault] = []
+        self._resource_faults: list[ResourceFault] = []
+
+    def fail(
+        self,
+        tool_name: str,
+        error: BaseException | Callable[[], BaseException],
+        *,
+        times: int | None = None,
+    ) -> None:
+        """Queues a fault for the next `run()`.
+
+        `tool_name` raises `error` (or the result of calling it, if callable —
+        useful for a fresh exception instance per call) instead of executing.
+        `times=None` (default) faults every call; `times=N` faults the first
+        `N` calls, then delegates to the real tool. Queued faults are consumed
+        by the next `run()` and don't carry over to the one after it.
+        """
+        self._faults.append(ToolFault(tool_name, error, times=times))
+
+    def fail_resource(
+        self,
+        name: str,
+        error: BaseException | Callable[[], BaseException],
+        *,
+        method: str | None = None,
+        times: int | None = None,
+    ) -> None:
+        """Queues a fault for a named resource — the general-purpose analog
+        of `fail()` for anything that isn't a tool: `"llm"`, `"embedder"`, a
+        source id (`resources.sources[id]`), or a name set via
+        `resources.set(name, ...)`.
+
+        Wraps the resource in a duck-typed proxy for the next `run()`/
+        `.turn()`: `method=None` (default) fails every callable on it;
+        naming one method (e.g. `"embed"`, `"search"`) faults only that
+        method. `times=None` faults every call; `times=N` faults the first
+        `N`, then delegates to the real resource — same shape as `fail()`.
+        Raises `ScenarioError` at run time if `name` doesn't match any
+        resource, or matches one that's `None` (nothing configured to fail).
+        """
+        self._resource_faults.append(
+            ResourceFault(name, error, method=method, times=times)
+        )
+
+    def _build_resources(self) -> RuntimeResources:
+        resources = self._resources() if callable(self._resources) else self._resources
+        resources = resources or RuntimeResources()
+        resources.llm = wrap_llm(
+            resources.llm, mode=self._mode, recording_path=self._recording_path
+        )
+        return resources
+
+    async def run(self, *seed: Any, max_iterations: int = 100) -> ScenarioResult:
+        """Seeds `seed` artifacts into a fresh `Context`, runs the agents to
+        completion (or budget/iteration exhaustion), and returns the result.
+        """
+        context = Context(resources=self._build_resources())
+        for data in seed:
+            context.create(data)
+        return await _execute_turn(
+            agents=self._agents,
+            faults=self._take_faults(),
+            resource_faults=self._take_resource_faults(),
+            budget=self._budget,
+            max_concurrency=self._max_concurrency,
+            isolate_errors=self._isolate_errors,
+            context=context,
+            max_iterations=max_iterations,
+        )
+
+    def scenario(self) -> Scenario:
+        """Starts a multi-turn scenario: one `Context` reused across `.turn()`
+        calls, for flows that need more than one round of user input to reach
+        the state under test (e.g. a design pick, then a plan, then an
+        approval — each its own turn on the same project). The record/replay
+        LLM wrapper is built once here, so its call index persists correctly
+        across turns.
+        """
+        context = Context(resources=self._build_resources())
+        return Scenario(
+            agents=self._agents,
+            budget=self._budget,
+            max_concurrency=self._max_concurrency,
+            isolate_errors=self._isolate_errors,
+            take_faults=self._take_faults,
+            take_resource_faults=self._take_resource_faults,
+            context=context,
+        )
+
+    def _take_faults(self) -> list[ToolFault]:
+        """Pops and clears the queued tool faults — one-shot, for the next turn."""
+        faults, self._faults = self._faults, []
+        return faults
+
+    def _take_resource_faults(self) -> list[ResourceFault]:
+        """Pops and clears the queued resource faults — one-shot, for the
+        next turn."""
+        faults, self._resource_faults = self._resource_faults, []
+        return faults
+
+
+async def _execute_turn(
+    *,
+    agents: Sequence[Agent],
+    faults: list[ToolFault],
+    resource_faults: list[ResourceFault],
+    budget: Budget | None,
+    max_concurrency: int | None,
+    isolate_errors: bool,
+    context: Context,
+    max_iterations: int,
+) -> ScenarioResult:
+    """Runs one turn to completion: install faults/recorder, `runtime.arun()`,
+    restore the tools/resources, return the result. Shared by
+    `ScenarioLab.run()` and `Scenario.turn()` so there is exactly one place
+    that builds a `Runtime`.
+    """
+    recorder = ToolCallRecorder()
+    tracer = _CapturingTracer()
+    runtime = Runtime(
+        context,
+        agents=list(agents),
+        budget=budget,
+        max_concurrency=max_concurrency,
+        tracer=tracer,
+        isolate_errors=isolate_errors,
+    )
+    with (
+        FaultInstaller(agents, faults, recorder),
+        ResourceFaultInstaller(context.resources, resource_faults),
+    ):
+        await runtime.arun(max_iterations=max_iterations)
+
+    return ScenarioResult(
+        context=context,
+        stats=runtime.last_stats,
+        trace=tracer.trace,
+        calls=recorder.calls,
+    )
+
+
+def _combined_trace(traces: list[RunTrace]) -> RunTrace | None:
+    if not traces:
+        return None
+    return RunTrace(spans=[span for trace in traces for span in trace.spans])
+
+
+class Scenario:
+    """A multi-turn scenario: one `Context`/agent set reused across `.turn()`
+    calls (see `ScenarioLab.scenario()`).
+
+    Each `.turn()` seeds new artifacts onto the *same* context (so a later
+    turn sees everything an earlier one produced), runs the agents to
+    completion, and returns that turn's own `ScenarioResult` — faults queued
+    via `lab.fail(...)` are still one-shot, consumed by the next `.turn()`
+    only. `.path`/`.tools`/`.llm`/`.errors` mirror `ScenarioResult`'s, but
+    aggregated over every turn run so far, for conversation-wide assertions
+    (e.g. "the model was never called across the whole exchange").
+    """
+
+    def __init__(
+        self,
+        *,
+        agents: Sequence[Agent],
+        budget: Budget | None,
+        max_concurrency: int | None,
+        isolate_errors: bool,
+        take_faults: Callable[[], list[ToolFault]],
+        take_resource_faults: Callable[[], list[ResourceFault]],
+        context: Context,
+    ) -> None:
+        self._agents = agents
+        self._budget = budget
+        self._max_concurrency = max_concurrency
+        self._isolate_errors = isolate_errors
+        self._take_faults = take_faults
+        self._take_resource_faults = take_resource_faults
+        self.context = context
+        self.all_traces: list[RunTrace] = []
+        self.all_calls: list[ToolCallRecord] = []
+
+    async def turn(self, *seed: Any, max_iterations: int = 100) -> ScenarioResult:
+        for data in seed:
+            self.context.create(data)
+        result = await _execute_turn(
+            agents=self._agents,
+            faults=self._take_faults(),
+            resource_faults=self._take_resource_faults(),
+            budget=self._budget,
+            max_concurrency=self._max_concurrency,
+            isolate_errors=self._isolate_errors,
+            context=self.context,
+            max_iterations=max_iterations,
+        )
+        if result.trace is not None:
+            self.all_traces.append(result.trace)
+        self.all_calls.extend(result.calls)
+        return result
+
+    @property
+    def path(self) -> PathAssertions:
+        """Agent path across every turn run so far, in order."""
+        return PathAssertions(_combined_trace(self.all_traces))
+
+    @property
+    def tools(self) -> ToolAssertions:
+        """Tool calls across every turn run so far."""
+        return ToolAssertions(self.all_calls)
+
+    @property
+    def llm(self) -> LLMAssertions:
+        """LLM usage across every turn run so far."""
+        return LLMAssertions(_combined_trace(self.all_traces))
+
+    @property
+    def errors(self) -> ErrorAssertions:
+        """Isolated agent errors across every turn run so far."""
+        return ErrorAssertions(_combined_trace(self.all_traces), None)

reactifact/testing/mock.py

@@ -0,0 +1,166 @@
+"""Generic resource-level fault injection for `reactifact.testing`.
+
+`fault.py` covers tools (the only thing reactifact keeps in a named registry).
+Everything else an agent depends on lives on `RuntimeResources` — the LLM,
+the embedder, a named `Source`, or an arbitrary object stashed via
+`resources.set(name, ...)` — each with a *different* shape (`llm.complete`/
+`.stream`, `embedder.embed`, `source.search`/`.asearch`/`.resolve`, or
+whatever an app-specific resource exposes). reactifact never does `isinstance`
+checks against its own provider/source ABCs (see `fault.py`'s own docstring
+— everything is duck-typed), so a single reflection-based proxy that
+intercepts calls by name is a safe stand-in for any of them: this is the
+one general "mock this resource, make it fail, watch the honest-fallback
+path (§59) kick in" primitive, for whatever `resources.llm`/`.embedder`/
+`.sources[...]`/`.get(...)` isn't a tool.
+"""
+
+from __future__ import annotations
+
+import inspect
+from collections.abc import Callable
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Any
+
+from .exceptions import ScenarioError
+
+if TYPE_CHECKING:
+    from reactifact.resources import RuntimeResources
+
+
+@dataclass
+class ResourceFault:
+    """A queued fault for one named resource.
+
+    `resource` is `"llm"`, `"embedder"`, a source id (`resources.sources
+    [id]`), or a name set via `resources.set(name, ...)`. `method=None`
+    (default) fails every callable on the resource; naming one (e.g.
+    `"embed"`) faults only that method, leaving the rest of the resource
+    working normally. `times=None` faults every call; `times=N` faults the
+    first `N`, then delegates to the real resource — same shape as
+    `fault.ToolFault`.
+    """
+
+    resource: str
+    error: BaseException | Callable[[], BaseException]
+    method: str | None = None
+    times: int | None = None
+
+
+def _get_resource(resources: RuntimeResources, name: str) -> tuple[Any, bool]:
+    """Returns `(value, found)` — `found=False` means no such resource exists
+    at all (as opposed to existing but being `None`)."""
+    if name == "llm":
+        return resources.llm, True
+    if name == "embedder":
+        return resources.embedder, True
+    if name in resources.sources:
+        return resources.sources[name], True
+    if name in resources.additional:
+        return resources.additional[name], True
+    return None, False
+
+
+def _set_resource(resources: RuntimeResources, name: str, value: Any) -> None:
+    if name == "llm":
+        resources.llm = value
+    elif name == "embedder":
+        resources.embedder = value
+    elif name in resources.sources:
+        resources.sources[name] = value
+    else:
+        resources.set(name, value)
+
+
+class _FailingProxy:
+    """Wraps `inner`, intercepting `fault.method` (or every public callable,
+    if `method=None`) to raise `fault.error` instead of delegating, for the
+    fault's next `times` calls (or forever). Everything else — attributes,
+    other methods — passes straight through to `inner`.
+    """
+
+    def __init__(self, inner: Any, fault: ResourceFault) -> None:
+        object.__setattr__(self, "_inner", inner)
+        object.__setattr__(self, "_fault", fault)
+        object.__setattr__(self, "_remaining", fault.times)
+
+    def _take_error(self, name: str) -> BaseException | None:
+        fault: ResourceFault = object.__getattribute__(self, "_fault")
+        if fault.method is not None and name != fault.method:
+            return None
+        remaining = object.__getattribute__(self, "_remaining")
+        if remaining is not None:
+            if remaining <= 0:
+                return None
+            object.__setattr__(self, "_remaining", remaining - 1)
+        return fault.error() if callable(fault.error) else fault.error
+
+    def __getattr__(self, name: str) -> Any:
+        attr = getattr(object.__getattribute__(self, "_inner"), name)
+        if not callable(attr) or name.startswith("_"):
+            return attr
+
+        if inspect.isasyncgenfunction(attr):
+
+            async def _failing_agen(*args: Any, **kwargs: Any) -> Any:
+                err = self._take_error(name)
+                if err is not None:
+                    raise err
+                async for item in attr(*args, **kwargs):
+                    yield item
+
+            return _failing_agen
+
+        if inspect.iscoroutinefunction(attr):
+
+            async def _failing_coro(*args: Any, **kwargs: Any) -> Any:
+                err = self._take_error(name)
+                if err is not None:
+                    raise err
+                return await attr(*args, **kwargs)
+
+            return _failing_coro
+
+        def _failing_sync(*args: Any, **kwargs: Any) -> Any:
+            err = self._take_error(name)
+            if err is not None:
+                raise err
+            return attr(*args, **kwargs)
+
+        return _failing_sync
+
+
+class ResourceFaultInstaller:
+    """Context manager: wraps the resources named in `faults` for the
+    duration of one turn, restoring the originals in `__exit__` even if the
+    wrapped run raises.
+    """
+
+    def __init__(
+        self, resources: RuntimeResources, faults: list[ResourceFault]
+    ) -> None:
+        self._resources = resources
+        self._faults = {f.resource: f for f in faults}
+        self._originals: list[tuple[str, Any]] = []
+
+    def __enter__(self) -> ResourceFaultInstaller:
+        for name, fault in self._faults.items():
+            original, found = _get_resource(self._resources, name)
+            if not found:
+                raise ScenarioError(
+                    f"fail_resource({name!r}, ...): no such resource — expected "
+                    '"llm", "embedder", a source id, or a name set via '
+                    "resources.set(...)"
+                )
+            if original is None:
+                raise ScenarioError(
+                    f"fail_resource({name!r}, ...): resource is None (not "
+                    "configured for this scenario) — nothing to fail"
+                )
+            self._originals.append((name, original))
+            _set_resource(self._resources, name, _FailingProxy(original, fault))
+        return self
+
+    def __exit__(self, *exc: object) -> None:
+        for name, original in self._originals:
+            _set_resource(self._resources, name, original)
+        self._originals.clear()