reactifact 0.6.0__py3-none-any.whl

reactifact/structured.py

@@ -0,0 +1,245 @@
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+import re
+from collections.abc import Callable
+from typing import Generic, Literal, TypeVar
+
+from pydantic import BaseModel, ValidationError
+
+from .context import Context
+from .providers import LLMRequest, Message
+
+logger = logging.getLogger(__name__)
+
+TModel = TypeVar("TModel", bound=BaseModel)
+
+#: Why `structured_llm` returned None — passed to an `on_error` hook so a
+#: caller can tell "no provider configured" (offline/misconfigured) apart
+#: from "the provider was called and failed" (network/rate-limit/outage) and
+#: "the model replied but not with valid JSON" — all three collapse to the
+#: same `None` return (the honest-fallback contract, §67), but a caller that
+#: needs to alert on real outages can now distinguish them without changing
+#: how it handles the `None`.
+StructuredLLMFailure = Literal["no_provider", "provider_error", "parse_error"]
+OnStructuredError = Callable[[StructuredLLMFailure, BaseException | None], None]
+
+SYSTEM_STRUCTURED = (
+    "You produce structured output. Reply with a single JSON object only, "
+    "no commentary, no code fences."
+)
+
+
+def _extract_json(text: str) -> str | None:
+    """Extracts the first balanced JSON object from the model text.
+
+    Local models often add text/code fences around JSON — a tolerant
+    parser (§67: parsing is not the LLM's job).
+    """
+    text = text.strip()
+    text = re.sub(r"^```(?:json)?", "", text, flags=re.MULTILINE).rstrip("`").strip()
+    try:
+        start = text.index("{")
+    except ValueError:
+        return None
+    depth = 0
+    in_string = False
+    escaped = False
+    for i in range(start, len(text)):
+        ch = text[i]
+        if in_string:
+            if escaped:
+                escaped = False
+            elif ch == "\\":
+                escaped = True
+            elif ch == '"':
+                in_string = False
+            continue
+        if ch == '"':
+            in_string = True
+        elif ch == "{":
+            depth += 1
+        elif ch == "}":
+            depth -= 1
+            if depth == 0:
+                return text[start : i + 1]
+    return None
+
+
+def parse_structured(text: str, schema: type[TModel]) -> TModel | None:
+    """Tolerant parsing of an LLM response into a pydantic schema."""
+    for candidate in (text, _extract_json(text)):
+        if not candidate:
+            continue
+        try:
+            return schema.model_validate_json(candidate)
+        except ValidationError:
+            try:
+                return schema.model_validate(json.loads(candidate))
+            except (ValueError, ValidationError):
+                continue
+    return None
+
+
+async def structured_llm(
+    context: Context,
+    *,
+    schema: type[TModel],
+    system: str = SYSTEM_STRUCTURED,
+    user: str,
+    attempts: int = 2,
+    temperature: float | None = None,
+    max_tokens: int | None = None,
+    on_error: OnStructuredError | None = None,
+) -> TModel | None:
+    """Single LLM call against a schema: JSON + tolerant parse + retry.
+
+    Returns `schema` or None (not enough resources / the model did not return valid JSON).
+    Deterministic logic (JSON, retry) stays in code; the LLM only reasons (§9, §67).
+
+    `temperature`/`max_tokens`: `None` uses the provider default; pass a value
+    for a per-call override.
+
+    `on_error`, if given, is called right before returning None with *why*
+    ("no_provider" | "provider_error" | "parse_error") and the exception when
+    there is one — for callers that want to distinguish "offline" from "the
+    provider is down" (e.g. to alert) without changing how they handle `None`.
+    """
+    llm = context.resources.llm
+    if llm is None:
+        if on_error is not None:
+            on_error("no_provider", None)
+        return None
+    instruction = f"Reply with a single JSON object matching this schema:\n{schema.model_json_schema()}"
+
+    def _request(text: str) -> LLMRequest:
+        return LLMRequest(
+            messages=[
+                Message.system(system),
+                Message.user(text),
+            ],
+            temperature=temperature,
+            response_format={"type": "json_object"},
+            max_tokens=max_tokens,
+        )
+
+    request = _request(f"{instruction}\n\n{user}")
+    total = max(attempts, 1)
+    for attempt in range(total):
+        try:
+            response = await llm.complete(request)
+        except Exception as exc:
+            logger.warning(
+                "structured_llm: provider call failed (attempt %s/%s): %r",
+                attempt + 1,
+                total,
+                exc,
+            )
+            if attempt + 1 < total:
+                await asyncio.sleep(0.4 * (attempt + 1))  # backoff on network failures
+                request = _request(
+                    f"{instruction}\n\n{user}\n\n"
+                    "The previous request failed. Return a single strict JSON object only."
+                )
+                continue
+            if on_error is not None:
+                on_error("provider_error", exc)
+            return None  # provider/network failed — honest fallback
+        parsed = parse_structured(response.text, schema)
+        if parsed is not None:
+            return parsed
+        logger.debug(
+            "structured_llm parse failed (attempt %s): %.160r",
+            attempt + 1,
+            response.text,
+        )
+        if attempt + 1 < total:
+            request = _request(
+                f"{instruction}\n\n{user}\n\n"
+                "Previous reply was not valid JSON. Return a single strict JSON object only."
+            )
+    if on_error is not None:
+        on_error("parse_error", None)
+    return None
+
+
+async def llm_reply(
+    context: Context,
+    *,
+    system: str = "",
+    user: str,
+    attempts: int = 2,
+    temperature: float | None = None,
+    max_tokens: int | None = None,
+    on_error: OnStructuredError | None = None,
+) -> str | None:
+    """A *plain-text* chat completion → `str`, or `None` on an honest failure.
+
+    Convenience over `structured_llm` with a single-text schema: same retries,
+    same tolerant parsing, same `None` fallback (no model / provider failure) —
+    but no need to declare a one-field body model for free-form replies (§67).
+
+    It sends exactly **one** system message (the wire format is not a place for
+    multiple system blocks — extra context belongs in artifacts/views, §28).
+
+    `on_error`: see `structured_llm`.
+    """
+    body = await structured_llm(
+        context,
+        schema=_ReplyBody,
+        system=system,
+        user=user,
+        attempts=attempts,
+        temperature=temperature,
+        max_tokens=max_tokens,
+        on_error=on_error,
+    )
+    return body.text if body is not None else None
+
+
+class _ReplyBody(BaseModel):
+    text: str
+
+
+class StructuredLLM(Generic[TModel]):
+    """A reusable structured call: a fixed schema + system, varying only `user`.
+
+    Build a role once, use it wherever the same struct is needed:
+
+        extract_facts = StructuredLLM(schema=Facts, system=SYSTEM_EXTRACTOR)
+        facts = await extract_facts.call(context, user="Summarize this page: …")
+
+    All deterministic logic (JSON extraction, retries, backoff) is delegated to
+    `structured_llm` (§67); the object only carries the fixed parts.
+    """
+
+    def __init__(
+        self,
+        schema: type[TModel],
+        *,
+        system: str = SYSTEM_STRUCTURED,
+        attempts: int = 2,
+        temperature: float = 0.0,
+        max_tokens: int = 2048,
+        on_error: OnStructuredError | None = None,
+    ):
+        self.schema = schema
+        self.system = system
+        self.attempts = attempts
+        self.temperature = temperature
+        self.max_tokens = max_tokens
+        self.on_error = on_error
+
+    async def call(self, context: Context, user: str) -> TModel | None:
+        return await structured_llm(
+            context,
+            schema=self.schema,
+            system=self.system,
+            user=user,
+            attempts=self.attempts,
+            temperature=self.temperature,
+            max_tokens=self.max_tokens,
+            on_error=self.on_error,
+        )

reactifact/testing/__init__.py

@@ -0,0 +1,48 @@
+"""reactifact.testing — a scenario-testing harness for agent pipelines.
+
+`ScenarioLab` is the entry point: seed some artifacts, run a set of agents,
+assert on what happened (artifacts produced, tools called, agent path,
+LLM usage, isolated errors). `lab.fail(...)` injects tool faults;
+`lab.fail_resource(...)` injects a fault into any other resource (the LLM,
+the embedder, a source); `mode=` on `ScenarioLab` switches the LLM between
+live, record, and replay.
+"""
+
+from __future__ import annotations
+
+from .assertions import (
+    ArtifactAssertions,
+    ErrorAssertions,
+    LLMAssertions,
+    PathAssertions,
+    ToolAssertions,
+)
+from .exceptions import AssertionFailure, ScenarioError, ScenarioSkip
+from .fault import ToolCallRecord, ToolCallRecorder, ToolFault
+from .lab import Scenario, ScenarioLab, ScenarioResult
+from .mock import ResourceFault
+from .record import Mode, mode_from_env
+from .registry import ScenarioCase, collect, scenario
+
+__all__ = [
+    "ArtifactAssertions",
+    "AssertionFailure",
+    "ErrorAssertions",
+    "LLMAssertions",
+    "Mode",
+    "PathAssertions",
+    "ResourceFault",
+    "Scenario",
+    "ScenarioCase",
+    "ScenarioError",
+    "ScenarioLab",
+    "ScenarioResult",
+    "ScenarioSkip",
+    "ToolAssertions",
+    "ToolCallRecord",
+    "ToolCallRecorder",
+    "ToolFault",
+    "collect",
+    "mode_from_env",
+    "scenario",
+]

reactifact/testing/assertions.py

@@ -0,0 +1,326 @@
+"""Assertion objects for `reactifact.testing` scenario results.
+
+Each assertion group is a thin, chainable wrapper over data reactifact already
+computes — `Context.list_artifacts()` for artifacts, `RunTrace` for the agent
+path and LLM calls, and (for tools) the harness's own `ToolCallRecorder`
+(`reactifact.testing.fault`), since `AgentSpan` has no field for raw tool
+invocations. Every failure raises `AssertionFailure` with the actually
+observed data inlined, so a failing scenario test is debuggable straight from
+the pytest output.
+
+Dropped from v1 (no faithful reactifact analog): a LangGraph-style "node status"
+assertion. reactifact's closest concept, `ProgressEvent` from `Runtime.astream()`,
+is a live-streaming concept, not a post-hoc trace field; a `result.events`
+assertion group may be added later if `ScenarioLab` grows a streaming mode.
+"""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Iterable
+from typing import TYPE_CHECKING, Any, Generic, TypeVar
+
+from pydantic import BaseModel
+
+from .exceptions import AssertionFailure
+
+if TYPE_CHECKING:
+    from reactifact.budget import RunStats
+    from reactifact.context import Context
+    from reactifact.tracing.models import AgentSpan, LLMCall, RunTrace
+
+    from .fault import ToolCallRecord
+
+T = TypeVar("T", bound=BaseModel)
+
+
+class ArtifactAssertions(Generic[T]):
+    """Assertions over `context.list_artifacts(artifact_type)`."""
+
+    def __init__(self, context: Context, artifact_type: type[T]) -> None:
+        self._context = context
+        self._type = artifact_type
+
+    def all(self) -> list[T]:
+        return [a.data for a in self._context.list_artifacts(self._type)]
+
+    def exists(self) -> T:
+        """Asserts at least one exists; returns the most recently created."""
+        artifact = self._context.latest(self._type)
+        if artifact is None:
+            raise AssertionFailure(
+                f"expected an artifact of type {self._type.__name__!r}, found none "
+                f"(context has: {self._present_types()})"
+            )
+        return artifact.data
+
+    def none(self) -> None:
+        found = self.all()
+        if found:
+            raise AssertionFailure(
+                f"expected no {self._type.__name__!r} artifacts, found {len(found)}: "
+                f"{found!r}"
+            )
+
+    def count(self, n: int) -> None:
+        found = self.all()
+        if len(found) != n:
+            raise AssertionFailure(
+                f"expected {n} artifact(s) of type {self._type.__name__!r}, "
+                f"found {len(found)}: {found!r}"
+            )
+
+    def latest(self) -> T:
+        return self.exists()
+
+    def matches(self, pattern: str, *, field: str = "text") -> T:
+        """Regex-searches `field` on the latest artifact (default: `.text`)."""
+        data = self.exists()
+        value = str(getattr(data, field, None))
+        if re.search(pattern, value) is None:
+            raise AssertionFailure(
+                f"{self._type.__name__}.{field} = {value!r} does not match "
+                f"pattern {pattern!r}"
+            )
+        return data
+
+    def field_equals(self, field: str, value: Any) -> T:
+        data = self.exists()
+        actual = getattr(data, field, None)
+        if actual != value:
+            raise AssertionFailure(
+                f"{self._type.__name__}.{field} = {actual!r}, expected {value!r}"
+            )
+        return data
+
+    def equals(self, **fields: Any) -> T:
+        """Asserts every given field on the latest artifact matches at once."""
+        data = self.exists()
+        mismatches = {
+            name: (getattr(data, name, None), expected)
+            for name, expected in fields.items()
+            if getattr(data, name, None) != expected
+        }
+        if mismatches:
+            raise AssertionFailure(
+                f"{self._type.__name__} field mismatch(es): "
+                + ", ".join(
+                    f"{name}={actual!r} (expected {expected!r})"
+                    for name, (actual, expected) in mismatches.items()
+                )
+            )
+        return data
+
+    def contains(self, substring: str, *, field: str = "text") -> T:
+        """Plain substring check on `field` of the latest artifact (no regex)."""
+        data = self.exists()
+        value = str(getattr(data, field, None))
+        if substring not in value:
+            raise AssertionFailure(
+                f"{self._type.__name__}.{field} = {value!r} does not contain "
+                f"{substring!r}"
+            )
+        return data
+
+    def field_in(self, field: str, values: Iterable[Any]) -> T:
+        """Asserts `field` on the latest artifact is one of `values`."""
+        data = self.exists()
+        options = list(values)
+        actual = getattr(data, field, None)
+        if actual not in options:
+            raise AssertionFailure(
+                f"{self._type.__name__}.{field} = {actual!r}, expected one of {options!r}"
+            )
+        return data
+
+    def _present_types(self) -> str:
+        names = sorted({type(a.data).__name__ for a in self._context.list_artifacts()})
+        return ", ".join(names) if names else "(none)"
+
+
+class ToolAssertions:
+    """Assertions over recorded tool calls (`ToolCallRecorder`, see `fault.py`).
+
+    Independent of `RunTrace`: `AgentSpan` does not carry raw tool
+    invocations, so this reads the harness's own call log instead.
+    """
+
+    def __init__(self, calls: list[ToolCallRecord]) -> None:
+        self._calls = calls
+
+    def call_order(self) -> list[str]:
+        return [c.tool for c in self._calls]
+
+    def called(self, name: str) -> list[ToolCallRecord]:
+        matches = [c for c in self._calls if c.tool == name]
+        if not matches:
+            raise AssertionFailure(
+                f"expected tool {name!r} to be called, but it never was "
+                f"(called: {self.call_order()})"
+            )
+        return matches
+
+    def never_called(self, name: str) -> None:
+        matches = [c for c in self._calls if c.tool == name]
+        if matches:
+            raise AssertionFailure(
+                f"expected tool {name!r} to never be called, but it was called "
+                f"{len(matches)} time(s) with args {[m.args for m in matches]}"
+            )
+
+    def called_times(self, name: str, n: int) -> None:
+        matches = [c for c in self._calls if c.tool == name]
+        if len(matches) != n:
+            raise AssertionFailure(
+                f"expected tool {name!r} to be called {n} time(s), "
+                f"was called {len(matches)} time(s)"
+            )
+
+    def called_any(self, *names: str) -> ToolCallRecord:
+        """Asserts at least one of `names` was called; returns its first call."""
+        for call in self._calls:
+            if call.tool in names:
+                return call
+        raise AssertionFailure(
+            f"expected at least one of {list(names)} to be called "
+            f"(called: {self.call_order()})"
+        )
+
+    def called_with(self, name: str, **kwargs: Any) -> ToolCallRecord:
+        for call in self._calls:
+            if call.tool != name:
+                continue
+            if all(call.args.get(k) == v for k, v in kwargs.items()):
+                return call
+        raise AssertionFailure(
+            f"no call to tool {name!r} matched args {kwargs!r} "
+            f"(calls: {[(c.tool, c.args) for c in self._calls if c.tool == name]})"
+        )
+
+
+class PathAssertions:
+    """Assertions over the agent execution path (`RunTrace.spans[*].agent`)."""
+
+    def __init__(self, trace: RunTrace | None) -> None:
+        self._trace = trace
+
+    def all(self) -> list[str]:
+        if self._trace is None:
+            return []
+        return [span.agent for span in self._trace.spans]
+
+    def contains(self, agent_name: str) -> None:
+        if agent_name not in self.all():
+            raise AssertionFailure(
+                f"expected agent {agent_name!r} to have run, path was {self.all()}"
+            )
+
+    def not_contains(self, agent_name: str) -> None:
+        if agent_name in self.all():
+            raise AssertionFailure(
+                f"expected agent {agent_name!r} to not run, but path was {self.all()}"
+            )
+
+    def times(self, agent_name: str) -> int:
+        """How many times `agent_name` appears in the path (a measurement,
+        not an assertion — pair it with your own `==` check)."""
+        return self.all().count(agent_name)
+
+    def sequence(self, *names: str) -> None:
+        """Asserts `names` appear, in order, as a (not-necessarily-contiguous) subsequence."""
+        path = self.all()
+        pos = 0
+        for name in names:
+            try:
+                pos = path.index(name, pos) + 1
+            except ValueError as exc:
+                raise AssertionFailure(
+                    f"expected subsequence {list(names)} in path {path}, "
+                    f"but {name!r} was not found after position {pos}"
+                ) from exc
+
+    def exact_sequence(self, *names: str) -> None:
+        path = self.all()
+        if path != list(names):
+            raise AssertionFailure(f"expected path {list(names)}, got {path}")
+
+    def any_of(self, *names: str) -> None:
+        """Asserts at least one of `names` ran (membership over the path)."""
+        path = self.all()
+        if not any(name in path for name in names):
+            raise AssertionFailure(
+                f"expected at least one of {list(names)} in path, got {path}"
+            )
+
+
+class LLMAssertions:
+    """Assertions over `RunTrace.llm_calls` (populated automatically since
+    `ScenarioLab` always attaches a tracer)."""
+
+    def __init__(self, trace: RunTrace | None) -> None:
+        self._trace = trace
+
+    def _calls(self) -> list[LLMCall]:
+        return self._trace.llm_calls if self._trace is not None else []
+
+    @property
+    def calls(self) -> int:
+        return len(self._calls())
+
+    @property
+    def tokens(self) -> int:
+        return sum(c.prompt_tokens + c.completion_tokens for c in self._calls())
+
+    def max_calls(self, n: int) -> None:
+        if self.calls > n:
+            raise AssertionFailure(
+                f"expected at most {n} LLM call(s), got {self.calls}"
+            )
+
+    def max_tokens(self, n: int) -> None:
+        if self.tokens > n:
+            raise AssertionFailure(f"expected at most {n} token(s), got {self.tokens}")
+
+    def by_agent(self, agent_name: str) -> list[LLMCall]:
+        return [c for c in self._calls() if c.agent == agent_name]
+
+
+class ErrorAssertions:
+    """Assertions over isolated agent errors (`Runtime(isolate_errors=True)`).
+
+    Reads `AgentSpan.error` (per-agent) and `RunStats.errors` (count) — both
+    require the scenario's `Runtime` to have been constructed with
+    `isolate_errors=True`; otherwise an agent exception propagates before a
+    trace/stats exist at all.
+    """
+
+    def __init__(self, trace: RunTrace | None, stats: RunStats | None) -> None:
+        self._trace = trace
+        self._stats = stats
+
+    def _errored_spans(self) -> list[AgentSpan]:
+        if self._trace is None:
+            return []
+        return [span for span in self._trace.spans if span.error]
+
+    def none(self) -> None:
+        errored = self._errored_spans()
+        if errored:
+            raise AssertionFailure(
+                f"expected no agent errors, found: "
+                f"{[(s.agent, s.error) for s in errored]}"
+            )
+
+    def count(self) -> int:
+        if self._stats is not None:
+            return self._stats.errors
+        return len(self._errored_spans())
+
+    def expected(self, agent_name: str) -> AgentSpan:
+        for span in self._errored_spans():
+            if span.agent == agent_name:
+                return span
+        raise AssertionFailure(
+            f"expected agent {agent_name!r} to have errored, but no error span "
+            f"for it was found (errored agents: {[s.agent for s in self._errored_spans()]})"
+        )

reactifact/testing/exceptions.py

@@ -0,0 +1,27 @@
+"""Exceptions for `reactifact.testing`."""
+
+from __future__ import annotations
+
+
+class ScenarioError(Exception):
+    """Misuse of the scenario harness itself (not a failed assertion) — e.g.
+    `lab.fail_resource("no_such_name", ...)` naming a resource that doesn't
+    exist on this scenario's `RuntimeResources`.
+    """
+
+
+class ScenarioSkip(Exception):
+    """A scenario opts out of running (e.g. no API key, no recorded fixture).
+
+    Raise this from inside a `@scenario`-decorated function — the `reactifact
+    scenario` CLI reports it as `SKIP` (with this exception's message),
+    distinct from a failed assertion or a crash.
+    """
+
+
+class AssertionFailure(AssertionError):
+    """A scenario assertion did not hold.
+
+    Subclasses `AssertionError` so pytest's assertion introspection/output
+    still applies to it like any other failed `assert`.
+    """