simulhausen 0.1.0__py3-none-any.whl

simulhausen/__init__.py

@@ -0,0 +1,95 @@
+"""simulhausen: local simulation testing for agno agents.
+
+An LLM plays the user and holds a multi-turn dialogue with the agent under
+test, then an LLM judge grades the outcome criterion by criterion (passed,
+failed or inconclusive, each with its reasoning). No external services: the
+helper agents run on whatever agno model you configure.
+
+    import simulhausen as sim
+
+    result = await sim.run(
+        name="refund request",
+        description="The user wants a refund for a duplicated charge.",
+        agents=[
+            sim.agno_adapter(lambda: support_agent),
+            sim.UserSimulatorAgent(),
+            sim.JudgeAgent(criteria=["The agent asks for the order number"]),
+        ],
+        script=[sim.user("I was charged twice"), sim.agent(), sim.proceed(turns=3), sim.judge()],
+    )
+    assert result.success, result.failure_summary()
+
+The API follows LangWatch Scenario (Apache-2.0); see NOTICE.
+"""
+
+from importlib.metadata import PackageNotFoundError, version
+
+from simulhausen.adapters import agno_adapter, function_adapter
+from simulhausen.agents import JudgeAgent, JudgeVerdict, UserSimulatorAgent, UserTurn
+from simulhausen.config import SimulationConfig, configure, get_config
+from simulhausen.exceptions import (
+    SimulationBackendError,
+    SimulationHarnessError,
+    SimulationStructuredOutputError,
+)
+from simulhausen.reporting import collector
+from simulhausen.runner import run
+from simulhausen.script import agent, fail, judge, message, proceed, succeed, user
+from simulhausen.types import (
+    AgentAdapter,
+    AgentInput,
+    AgentReturnTypes,
+    AgentRole,
+    AgentTurn,
+    CriterionStatus,
+    CriterionVerdict,
+    JudgeScore,
+    Message,
+    Role,
+    ScenarioResult,
+    ScenarioState,
+    ScenarioStatus,
+)
+
+try:
+    __version__ = version("simulhausen")
+except PackageNotFoundError:  # pragma: no cover - running from a source tree without metadata
+    __version__ = "0.0.0"
+
+__all__ = [
+    "AgentAdapter",
+    "AgentInput",
+    "AgentReturnTypes",
+    "AgentRole",
+    "AgentTurn",
+    "CriterionStatus",
+    "CriterionVerdict",
+    "JudgeAgent",
+    "JudgeScore",
+    "JudgeVerdict",
+    "Message",
+    "Role",
+    "ScenarioResult",
+    "ScenarioState",
+    "ScenarioStatus",
+    "SimulationBackendError",
+    "SimulationConfig",
+    "SimulationHarnessError",
+    "SimulationStructuredOutputError",
+    "UserSimulatorAgent",
+    "UserTurn",
+    "__version__",
+    "agent",
+    "agno_adapter",
+    "collector",
+    "configure",
+    "fail",
+    "function_adapter",
+    "get_config",
+    "judge",
+    "message",
+    "proceed",
+    "run",
+    "succeed",
+    "user",
+]

simulhausen/adapters.py

@@ -0,0 +1,212 @@
+"""Ready-made adapters: agno Agent/Team/Workflow, and plain functions.
+
+``agno_adapter`` builds the usual adapter (``arun`` in one session, an
+``AgentTurn`` with content, tools and references) in one call, so tests do not
+need a wrapper class each. ``function_adapter`` wraps any sync or async
+callable. Subclass :class:`~simulhausen.types.AgentAdapter` for anything else.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import inspect
+import logging
+from typing import TYPE_CHECKING, Any, Protocol, cast
+
+from simulhausen.exceptions import SimulationBackendError
+from simulhausen.types import AgentAdapter, AgentInput, AgentReturnTypes, AgentTurn
+
+if TYPE_CHECKING:
+    from collections.abc import Awaitable, Callable
+
+    from agno.agent import Agent
+    from agno.team import Team
+    from agno.workflow import Workflow
+
+__all__ = ["agno_adapter", "function_adapter"]
+
+logger = logging.getLogger(__name__)
+
+# Transient backend failures (a 504 and the like) are retried a few times: runs
+# go through a live inference gateway, and one timeout must not sink a scenario.
+DEFAULT_BACKEND_RETRIES = 2
+DEFAULT_BACKEND_RETRY_DELAY_S = 2.0
+
+# agno error types that mean an input/output check refused the request (a
+# guardrail), which is a product answer, not an infrastructure failure. See
+# agno/exceptions.py: InputCheckError.type == "input_check_error".
+_CHECK_ERROR_TYPES = frozenset({"input_check_error", "output_check_error"})
+
+_ERROR_PAGE_MARKERS = ("gateway time-out", "bad gateway", "service unavailable", "gateway timeout")
+
+
+class _Runnable(Protocol):
+    async def arun(self, input: Any, **kwargs: Any) -> Any: ...
+
+
+type AgnoTarget = Agent | Team | Workflow | Callable[[], Agent | Team | Workflow]
+
+
+def _looks_like_error_page(text: str) -> bool:
+    """Whether the content is a gateway's HTML error page rather than a model reply."""
+    head = text.lstrip()[:300].lower()
+    if head.startswith(("<!doctype", "<html", "<head")):
+        return True
+    return any(marker in head for marker in _ERROR_PAGE_MARKERS)
+
+
+def _is_check_refusal(response: Any) -> bool:
+    """Did the run end with an input/output check refusing the request?
+
+    On InputCheckError agno sets RunStatus=error and puts the refusal text in
+    ``content``. Formally an error, but for a scenario it is the product's
+    answer ("Sorry, I can't help with that"), which is exactly what a guardrail
+    scenario wants to check. Without this branch the adapter treated a
+    guardrail as a backend failure, retried, and raised SimulationBackendError.
+    """
+    return any(
+        getattr(event, "error_type", None) in _CHECK_ERROR_TYPES for event in getattr(response, "events", None) or []
+    )
+
+
+def _backend_error_reason(response: Any) -> str | None:
+    """Why the run counts as a backend failure, or ``None`` when it does not.
+
+    Two cases: agno reported RunStatus=error (guardrail refusals excluded, see
+    ``_is_check_refusal``), or the model returned a raw HTML error page from a
+    proxy instead of text or a structured object.
+    """
+    status = getattr(response, "status", None)
+    status_name = getattr(status, "value", None) or getattr(status, "name", None) or str(status)
+    if isinstance(status_name, str) and status_name.lower() == "error":
+        if _is_check_refusal(response):
+            return None
+        return "agno RunStatus=error"
+    content = getattr(response, "content", None)
+    if isinstance(content, str) and _looks_like_error_page(content):
+        return "backend returned a gateway HTML error page: " + " ".join(content.split())[:120]
+    return None
+
+
+def _collect_tools(response: Any) -> list[Any] | None:
+    """Tool executions of the run, team members' included.
+
+    For a Team, ``response.tools`` holds only the leader's calls (delegation),
+    while the domain tools live in ``member_responses[*].tools``. Without
+    merging them a team scenario cannot assert ``state.has_tool_call(...)`` on
+    a member tool. Agents have no ``member_responses``, so nothing changes there.
+    """
+    own = list(getattr(response, "tools", None) or [])
+    members = [
+        tool
+        for member in getattr(response, "member_responses", None) or []
+        for tool in getattr(member, "tools", None) or []
+    ]
+    return (own + members) or None
+
+
+class _AgnoAdapter(AgentAdapter):
+    """Last user message -> ``target.arun`` -> ``AgentTurn``."""
+
+    def __init__(
+        self,
+        target: AgnoTarget,
+        arun_kwargs: dict[str, Any],
+        content_fn: Callable[[Any], str] | None,
+        retries: int,
+        retry_delay: float,
+    ) -> None:
+        self._target = target
+        self._resolved: _Runnable | None = None
+        self._arun_kwargs = arun_kwargs
+        self._content_fn = content_fn
+        self._retries = retries
+        self._retry_delay = retry_delay
+
+    def _resolve(self) -> _Runnable:
+        if self._resolved is None:
+            target = self._target
+            # A factory (callable without arun) is called once and cached.
+            resolved = target() if callable(target) and not hasattr(target, "arun") else target
+            self._resolved = cast("_Runnable", resolved)
+        return self._resolved
+
+    async def call(self, input: AgentInput) -> AgentTurn:
+        target = self._resolve()
+        message = input.last_new_user_message_str()
+        attempts = self._retries + 1
+        reason: str | None = None
+        response: Any = None
+        for attempt in range(1, attempts + 1):
+            response = await target.arun(message, session_id=input.thread_id, **self._arun_kwargs)
+            reason = _backend_error_reason(response)
+            if reason is None:
+                break
+            logger.warning("backend error (attempt %d/%d): %s", attempt, attempts, reason)
+            if attempt < attempts:
+                await asyncio.sleep(self._retry_delay * attempt)
+        else:
+            # Every attempt hit the backend failure: raise a typed error instead
+            # of passing an HTML page on as the reply and failing a tool assertion.
+            raise SimulationBackendError(f"backend failed {attempts} time(s) in a row: {reason}")
+
+        raw = response.content
+        content = self._content_fn(raw) if self._content_fn is not None else str(raw or "")
+        return AgentTurn(
+            content=content,
+            tools=_collect_tools(response),
+            references=getattr(response, "references", None),
+        )
+
+
+def agno_adapter(
+    target: AgnoTarget,
+    *,
+    content_fn: Callable[[Any], str] | None = None,
+    retries: int = DEFAULT_BACKEND_RETRIES,
+    retry_delay: float = DEFAULT_BACKEND_RETRY_DELAY_S,
+    **arun_kwargs: Any,
+) -> AgentAdapter:
+    """Adapter for an agno Agent, Team or Workflow.
+
+    ``target`` is the object itself or a lazy factory (``lambda: my_agent``
+    with the import inside, so collecting tests does not build the agent).
+    Extra kwargs go to ``arun`` unchanged: ``session_state=...``,
+    ``user_id=...`` and so on. ``session_id`` is always the simulation's
+    ``thread_id``, so the whole scenario runs in one session.
+
+    ``content_fn`` renders ``response.content`` into the reply text. Agents
+    with ``output_schema`` need it, since their content is a Pydantic model;
+    otherwise the content goes through ``str(...)``.
+
+    ``retries``/``retry_delay`` control how backend failures (RunStatus=error,
+    a gateway HTML error page) are retried before ``SimulationBackendError``.
+
+    >>> adapter = agno_adapter(lambda: support_agent, session_state={"plan": "pro"})
+    """
+    if retries < 0:
+        raise ValueError("retries must be >= 0")
+    return _AgnoAdapter(target, arun_kwargs, content_fn=content_fn, retries=retries, retry_delay=retry_delay)
+
+
+class _FunctionAdapter(AgentAdapter):
+    def __init__(self, fn: Callable[[AgentInput], AgentReturnTypes | Awaitable[AgentReturnTypes]]) -> None:
+        self._fn = fn
+
+    async def call(self, input: AgentInput) -> AgentReturnTypes:
+        result = self._fn(input)
+        if inspect.isawaitable(result):
+            return await result
+        return result
+
+
+def function_adapter(
+    fn: Callable[[AgentInput], AgentReturnTypes | Awaitable[AgentReturnTypes]],
+) -> AgentAdapter:
+    """Adapter around a sync or async function ``AgentInput -> str | AgentTurn``.
+
+    >>> async def echo(input: AgentInput) -> str:
+    ...     return input.last_new_user_message_str()
+    >>> adapter = function_adapter(echo)
+    """
+    return _FunctionAdapter(fn)

simulhausen/agents.py

@@ -0,0 +1,340 @@
+"""Helper LLM agents: the user simulator and the per-criterion judge.
+
+Both are plain agno agents with an ``output_schema``. Prompts are adapted from
+LangWatch Scenario's simulator and judge (Apache-2.0, see NOTICE).
+"""
+
+from __future__ import annotations
+
+import logging
+import secrets
+from typing import TYPE_CHECKING, Literal
+
+from agno.agent import Agent
+from agno.run.agent import RunOutput
+from agno.scorer.judge import JudgeScorer
+from pydantic import BaseModel, Field
+
+from simulhausen.config import require_model, resolve_parser_model
+from simulhausen.exceptions import SimulationStructuredOutputError
+from simulhausen.prompts import (
+    JUDGE_DECISION_PROMPT,
+    JUDGE_VERDICT_PROMPT,
+    PER_CRITERION_RULES,
+    STRUCTURED_RETRY_NUDGE,
+    USER_SIMULATOR_PROMPT,
+)
+from simulhausen.types import AgentRole, CriterionVerdict, JudgeScore
+
+if TYPE_CHECKING:
+    from collections.abc import Sequence
+
+    from agno.models.base import Model
+
+    from simulhausen.types import ScenarioState
+
+__all__ = ["JudgeAgent", "JudgeVerdict", "UserSimulatorAgent", "UserTurn"]
+
+logger = logging.getLogger(__name__)
+
+# Models now and then answer in free text instead of the schema. A parser model
+# fixes most of it (see SimulationConfig.parser_model); the retry covers what
+# remains of the nondeterminism.
+STRUCTURED_ATTEMPTS = 3
+
+
+def _fence(label: str, text: str) -> str:
+    """Wrap untrusted data in a tag with a random per-call suffix.
+
+    User text, RAG hits and tool results may contain instructions or closing
+    XML tags. A random suffix makes the closing tag unguessable from inside the
+    data, the same idea as agno's hardened judge prompt.
+    """
+    tag = f"untrusted_{label}_{secrets.token_hex(16)}"
+    return f"<{tag}>\n{text}\n</{tag}>"
+
+
+async def _arun_structured[T](agent: Agent, prompt: str, schema: type[T], *, scenario: str, what: str) -> T:
+    """Run the agent and demand a schema object; retry with a format nudge if it is not one."""
+    for attempt in range(1, STRUCTURED_ATTEMPTS + 1):
+        attempt_prompt = prompt if attempt == 1 else prompt + STRUCTURED_RETRY_NUDGE
+        response = await agent.arun(attempt_prompt)
+        content = response.content
+        if isinstance(content, schema):
+            return content
+        logger.warning(
+            "%s returned %s instead of %s (scenario %r, attempt %d/%d)",
+            what,
+            type(content).__name__,
+            schema.__name__,
+            scenario,
+            attempt,
+            STRUCTURED_ATTEMPTS,
+        )
+    raise SimulationStructuredOutputError(
+        f"{what} returned no structured output in {STRUCTURED_ATTEMPTS} attempts (scenario {scenario!r})"
+    )
+
+
+def _numbered(criteria: Sequence[str]) -> str:
+    return "\n".join(f"{index}. {criterion}" for index, criterion in enumerate(criteria, 1))
+
+
+# -- user simulator ------------------------------------------------------------
+
+
+class UserTurn(BaseModel):
+    """One simulated user turn: the message plus a "goal reached" signal."""
+
+    message: str = Field(description="The user's next message, without quotes or commentary")
+    goal_achieved: bool = Field(
+        description="true when the scenario's goal is already reached and the dialogue should stop"
+    )
+
+
+class UserSimulatorAgent:
+    """An LLM playing the user: writes the next user message from the scenario and transcript.
+
+    ``next_turn`` also returns ``goal_achieved``, so unscripted runs stop as
+    soon as the scenario's goal is reached instead of padding the dialogue
+    with polite filler up to ``max_turns``.
+
+    ``system_prompt`` replaces the built-in prompt entirely (the scenario
+    description is then up to you to include); ``persona`` is appended to
+    whichever prompt is used.
+    """
+
+    role: AgentRole = AgentRole.USER
+
+    def __init__(
+        self,
+        model: Model | None = None,
+        *,
+        parser_model: Model | None = None,
+        system_prompt: str | None = None,
+        persona: str | None = None,
+    ) -> None:
+        self._model = model
+        self._parser_model = parser_model
+        self.system_prompt = system_prompt
+        self.persona = persona
+
+    def instructions(self, state: ScenarioState) -> str:
+        prompt = self.system_prompt or USER_SIMULATOR_PROMPT.format(description=state.description)
+        if self.persona:
+            prompt += f"\n<persona>\n{self.persona}\n</persona>\n"
+        return prompt
+
+    async def next_turn(self, state: ScenarioState) -> UserTurn:
+        agent = Agent(
+            model=require_model(self._model),
+            instructions=self.instructions(state),
+            output_schema=UserTurn,
+            parser_model=resolve_parser_model(self._parser_model),
+            telemetry=False,
+        )
+        transcript = state.transcript() or "(the dialogue has not started yet)"
+        turn = await _arun_structured(
+            agent,
+            f"Dialogue so far:\n{transcript}\n\nThe user's next message:",
+            UserTurn,
+            scenario=state.name,
+            what="user simulator",
+        )
+        message = turn.message.strip()
+        if not (message or turn.goal_achieved):
+            raise SimulationStructuredOutputError(f"user simulator returned an empty turn (scenario {state.name!r})")
+        return UserTurn(message=message, goal_achieved=turn.goal_achieved)
+
+    async def next_message(self, state: ScenarioState) -> str:
+        """A message for a script ``user()`` step without content."""
+        turn = await self.next_turn(state)
+        if not turn.message:
+            raise SimulationStructuredOutputError(f"user simulator returned an empty message (scenario {state.name!r})")
+        return turn.message
+
+
+# -- judge ---------------------------------------------------------------------
+
+
+class JudgeVerdict(BaseModel):
+    """The judge's structured answer: one verdict per criterion plus a summary."""
+
+    verdicts: list[CriterionVerdict] = Field(description="Exactly one verdict per criterion, in the given order")
+    summary: str = Field(description="One or two sentences on the overall outcome")
+
+
+class _TurnDecision(BaseModel):
+    """Mid-dialogue decision: keep talking or conclude.
+
+    No reasoning field on purpose: writing one pushes the judge to commit to a
+    pass or fail before the evidence is in, and the text is wasted tokens for a
+    binary transition (the same choice LangWatch Scenario made).
+    """
+
+    decision: Literal["continue", "conclude"] = Field(
+        description="continue: the dialogue needs another user turn; conclude: there is enough to judge"
+    )
+
+
+class JudgeAgent:
+    """LLM judge with per-criterion verdicts and an optional numeric grade.
+
+    Each criterion gets ``passed``, ``failed`` or ``inconclusive`` with its own
+    reasoning. ``criteria`` may be empty when the scenario only uses inline
+    ``judge(criteria=[...])`` checkpoints.
+
+    ``score_threshold`` (1-10) additionally runs agno's ``JudgeScorer`` over the
+    whole dialogue and fails the scenario below the threshold. It is opt-in so a
+    numeric cut-off never changes results before it is calibrated.
+
+    ``system_prompt`` replaces the built-in verdict prompt; the dialogue and the
+    evidence are still sent as the message, fenced as untrusted data.
+    """
+
+    role: AgentRole = AgentRole.JUDGE
+
+    def __init__(
+        self,
+        criteria: Sequence[str] = (),
+        model: Model | None = None,
+        *,
+        parser_model: Model | None = None,
+        system_prompt: str | None = None,
+        score_threshold: int | None = None,
+    ) -> None:
+        if score_threshold is not None and not 1 <= score_threshold <= 10:
+            raise ValueError("score_threshold must be between 1 and 10")
+        self.criteria = list(criteria)
+        self._model = model
+        self._parser_model = parser_model
+        self.system_prompt = system_prompt
+        self.score_threshold = score_threshold
+
+    def _agent(self, instructions: str, schema: type[BaseModel]) -> Agent:
+        return Agent(
+            model=require_model(self._model),
+            instructions=instructions,
+            output_schema=schema,
+            parser_model=resolve_parser_model(self._parser_model),
+            telemetry=False,
+        )
+
+    @staticmethod
+    def evidence(state: ScenarioState, additional_context: str | None = None) -> str:
+        """The dialogue plus what the agent saw and did, each fenced as untrusted data.
+
+        The judge sees the same reference material (RAG) as the agent, so
+        grounding criteria are checked against the real retrieval rather than
+        a hand-written answer key. Tool executions let it tell facts that came
+        from a tool apart from hallucinations.
+        """
+        prompt = "Dialogue:\n" + _fence("transcript", state.transcript() or "(empty)")
+        if references := state.references_text():
+            prompt += "\n\nReference material the agent saw:\n" + _fence("references", references)
+        if tool_evidence := state.tool_evidence_text():
+            prompt += "\n\nTool executions:\n" + _fence("tools", tool_evidence)
+        if additional_context:
+            prompt += "\n\nAdditional context from the test:\n" + _fence("context", additional_context)
+        return prompt
+
+    async def judge(
+        self,
+        state: ScenarioState,
+        criteria: Sequence[str] | None = None,
+        *,
+        additional_context: str | None = None,
+    ) -> JudgeVerdict:
+        """Verdict on ``criteria`` (the judge's own when ``None``), one per criterion.
+
+        ``additional_context`` is extra evidence the dialogue does not show, for
+        example the state of a database or a file the agent should have changed.
+        """
+        criteria = list(self.criteria if criteria is None else criteria)
+        if not criteria:
+            raise ValueError("the judge has no criteria: pass JudgeAgent(criteria=[...]) or judge(criteria=[...])")
+        instructions = self.system_prompt or JUDGE_VERDICT_PROMPT.format(
+            description=state.description,
+            criteria=_numbered(criteria),
+            per_criterion_rules=PER_CRITERION_RULES,
+        )
+        judged = await _arun_structured(
+            self._agent(instructions, JudgeVerdict),
+            self.evidence(state, additional_context),
+            JudgeVerdict,
+            scenario=state.name,
+            what="judge",
+        )
+        return JudgeVerdict(verdicts=self._align(state, criteria, judged.verdicts), summary=judged.summary)
+
+    async def evaluate(
+        self,
+        state: ScenarioState,
+        criteria: Sequence[str] | None = None,
+        *,
+        additional_context: str | None = None,
+    ) -> list[CriterionVerdict]:
+        """Just the per-criterion verdicts of :meth:`judge`."""
+        return (await self.judge(state, criteria, additional_context=additional_context)).verdicts
+
+    @staticmethod
+    def _align(state: ScenarioState, criteria: list[str], verdicts: list[CriterionVerdict]) -> list[CriterionVerdict]:
+        """One verdict per criterion, in order, with the criterion text restored verbatim.
+
+        Models paraphrase, drop or duplicate criteria. With a matching count the
+        order is trusted; otherwise verdicts are matched by exact text and a
+        criterion left without one becomes inconclusive.
+        """
+        if len(verdicts) == len(criteria):
+            return [v.model_copy(update={"criterion": c}) for c, v in zip(criteria, verdicts, strict=True)]
+        logger.warning(
+            "judge returned %d verdicts for %d criteria (scenario %r)", len(verdicts), len(criteria), state.name
+        )
+        by_text = {v.criterion: v for v in verdicts}
+        return [
+            by_text.get(c)
+            or CriterionVerdict(criterion=c, status="inconclusive", reasoning="the judge returned no verdict for it")
+            for c in criteria
+        ]
+
+    async def decide(self, state: ScenarioState) -> bool:
+        """Unscripted runs: ``True`` when the dialogue holds enough for a verdict."""
+        if not self.criteria:
+            return False
+        agent = self._agent(
+            JUDGE_DECISION_PROMPT.format(description=state.description, criteria=_numbered(self.criteria)),
+            _TurnDecision,
+        )
+        response = await agent.arun(self.evidence(state))
+        decision = response.content
+        if not isinstance(decision, _TurnDecision):
+            # Unparsed decision: keep talking until max_turns, the final verdict decides.
+            logger.warning("judge decision could not be parsed (scenario %r)", state.name)
+            return False
+        return decision.decision == "conclude"
+
+    async def score(self, state: ScenarioState) -> JudgeScore | None:
+        """Grade the whole dialogue 1-10 with agno's hardened ``JudgeScorer`` (when enabled)."""
+        if self.score_threshold is None:
+            return None
+        scorer = JudgeScorer(
+            model=require_model(self._model),
+            criteria=(
+                f"Scenario:\n{state.description}\n\n"
+                "The dialogue must satisfy all of the following criteria:\n"
+                f"{_numbered(self.criteria)}"
+            ),
+            mode="numeric",
+            threshold=self.score_threshold,
+        )
+        # agno fences both the run output and the reference answer with nonce
+        # blocks, so dialogue text and RAG data cannot rewrite the judge's task.
+        score = await scorer.ascore(RunOutput(content=state.transcript()), expected=state.references_text() or None)
+        raw_score = int((score.detail or {}).get("raw_score", round(score.value * 9 + 1)))
+        return JudgeScore(
+            value=score.value,
+            raw_score=raw_score,
+            threshold=self.score_threshold,
+            passed=score.passed,
+            reasoning=score.reason,
+        )