sonar-eval 0.3.0__py3-none-any.whl

sonar_eval/__init__.py

@@ -0,0 +1,101 @@
+"""sonar-eval - evaluation harness SDK for agentic AI chatbots.
+
+Built on sonar-tracing: the harness drives an agent through simulated conversations,
+injects faults into its tool calls, and grades the transcript it pulls back from
+sonar-ingest by conversation_id.
+
+    from sonar_eval import Orchestrator, TaskConfig, ScenarioConfig
+"""
+
+from .adapter import (
+    BaseBotAdapter,
+    BotAdapter,
+    Delivered,
+    Seed,
+    Session,
+    TrialContext,
+    load_adapter,
+    validate_bot_adapter,
+)
+from .assertions import (
+    Assertion,
+    AssertionEngine,
+    AssertionResult,
+    Severity,
+    max_total_tokens,
+    max_turns,
+    no_tool_errors,
+    tool_was_called,
+)
+from .config import Fidelity, Persona, ScenarioConfig, TaskConfig
+from .grader import Grader, GraderResult
+from .interceptor import (
+    Action,
+    FaultRule,
+    FaultSpec,
+    InjectedFault,
+    InterceptDecision,
+    ProxyInterceptor,
+    ToolCall,
+    ToolInterceptor,
+    serialize_faults,
+)
+from .llm import LLMClient, LLMConfig, Message
+from .metrics import AssertionMetric, ScenarioReport, summarize
+from .orchestrator import Orchestrator, TrialResult
+from .simulator import ConversationLog, Turn, UserSimulator
+from .store import JSONFileStore, NullStore, ResultStore
+from .transcript import IngestClient, ToolInvocation, Transcript
+from .version import __version__
+
+__all__ = [
+    "Action",
+    "Assertion",
+    "AssertionEngine",
+    "AssertionMetric",
+    "AssertionResult",
+    "BaseBotAdapter",
+    "BotAdapter",
+    "ConversationLog",
+    "Delivered",
+    "FaultRule",
+    "FaultSpec",
+    "Fidelity",
+    "Grader",
+    "GraderResult",
+    "IngestClient",
+    "InjectedFault",
+    "InterceptDecision",
+    "JSONFileStore",
+    "LLMClient",
+    "LLMConfig",
+    "Message",
+    "NullStore",
+    "Orchestrator",
+    "Persona",
+    "ProxyInterceptor",
+    "ResultStore",
+    "ScenarioConfig",
+    "ScenarioReport",
+    "Seed",
+    "Session",
+    "Severity",
+    "TaskConfig",
+    "ToolCall",
+    "ToolInterceptor",
+    "ToolInvocation",
+    "Transcript",
+    "TrialContext",
+    "TrialResult",
+    "Turn",
+    "UserSimulator",
+    "__version__",
+    "load_adapter",
+    "max_total_tokens",
+    "max_turns",
+    "no_tool_errors",
+    "serialize_faults",
+    "summarize",
+    "tool_was_called",
+    "validate_bot_adapter",
+]

sonar_eval/adapter.py

@@ -0,0 +1,122 @@
+"""The bot adapter contract.
+
+One thin interface the developer implements once per bot. It absorbs the stack
+differences (auth, DB, transport) so the rest of the harness stays generic, and it is
+where per-trial isolation and seeding happen.
+
+Two things are threaded through every trial:
+
+  conversation_id  minted by the harness via ``sonar_tracing.new_conversation()``. The
+                   adapter must make the bot stamp it on its spans - for a service like
+                   Cafeina that means POSTing it to an app-internal endpoint so the app
+                   wraps its turns in ``tracing.identity(conversation_id=...)``. It is the
+                   join key the grader later uses to pull the transcript back from ingest.
+  faults           a FaultSpec. In mocked fidelity the adapter routes its mocks through a
+                   ProxyInterceptor built from it; in real fidelity it posts
+                   ``serialize_faults(faults)`` to the app alongside conversation_id.
+
+``send()`` returns only what a human user would see. The tool-call transcript is NOT
+returned here - it is pulled from sonar-ingest by conversation_id, which keeps the user
+simulator honest (it can only ever see delivered messages).
+"""
+
+from __future__ import annotations
+
+import importlib
+from dataclasses import dataclass, field
+from typing import Any, Protocol
+
+from .config import Fidelity
+from .interceptor import FaultSpec
+
+
+@dataclass(frozen=True)
+class Seed:
+    """Project-defined seed state: authorizations, DB rows, stored data.
+
+    Opaque to the harness; the adapter is the only thing that interprets ``data``.
+    """
+
+    data: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class Session:
+    """Opaque per-trial handle the adapter owns. Carries the join key for convenience."""
+
+    conversation_id: str
+    handle: Any = None
+
+
+@dataclass(frozen=True)
+class Delivered:
+    """Exactly what a human user would see for one turn."""
+
+    messages: tuple[str, ...] = ()
+    media: tuple[Any, ...] = ()
+
+
+@dataclass(frozen=True)
+class TrialContext:
+    conversation_id: str
+    seed: Seed
+    faults: FaultSpec
+    fidelity: Fidelity
+
+
+class BotAdapter(Protocol):
+    name: str
+
+    async def setup_trial(self, ctx: TrialContext) -> Session:
+        """Isolate + seed state for one trial, wire fault injection, return a handle."""
+
+    async def send(self, session: Session, user_msg: str, media: Any = None) -> Delivered:
+        """Deliver one user turn; return only what the user would see back."""
+
+    async def teardown_trial(self, session: Session) -> None:
+        """Reset in lockstep. Must be safe to call after a failed setup."""
+
+
+class BaseBotAdapter:
+    """Optional convenience base so a minimal adapter is a few lines."""
+
+    name: str = "unnamed"
+
+    async def setup_trial(self, ctx: TrialContext) -> Session:
+        return Session(conversation_id=ctx.conversation_id)
+
+    async def send(self, session: Session, user_msg: str, media: Any = None) -> Delivered:
+        return Delivered()
+
+    async def teardown_trial(self, session: Session) -> None:
+        return None
+
+
+_REQUIRED_METHODS = ("setup_trial", "send", "teardown_trial")
+
+
+def validate_bot_adapter(candidate: Any) -> None:
+    """Structural check with a useful message.
+
+    ``runtime_checkable`` Protocols cannot verify the ``name`` attribute, and a missing
+    async method would otherwise surface deep inside the trial loop.
+    """
+    if not isinstance(getattr(candidate, "name", None), str):
+        raise TypeError(f"{candidate!r} is missing a string `name` attribute")
+    not_callable = [m for m in _REQUIRED_METHODS if not callable(getattr(candidate, m, None))]
+    if not_callable:
+        raise TypeError(f"{candidate!r} is missing methods: {not_callable}")
+
+
+def load_adapter(path: str) -> BotAdapter:
+    """Load an adapter from a ``module.path:attribute`` string and validate it.
+
+    The attribute may be an instance or a zero-arg class/factory.
+    """
+    module_path, _, attr = path.partition(":")
+    if not attr:
+        raise ValueError(f"adapter path must be 'module:attr', got {path!r}")
+    obj = getattr(importlib.import_module(module_path), attr)
+    candidate = obj() if isinstance(obj, type) else obj
+    validate_bot_adapter(candidate)
+    return candidate

sonar_eval/assertions.py

@@ -0,0 +1,81 @@
+"""Programmatic (tier-1) assertions over a pulled transcript.
+
+Each assertion carries a Severity that decides how it is aggregated across k trials:
+safety-critical assertions must hold on every trial (pass^k == 1.0); capability assertions
+need only hold once (pass@k). Checks are ordinary callables so projects add their own.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from dataclasses import dataclass
+from enum import StrEnum
+
+from .simulator import ConversationLog
+from .transcript import Transcript
+
+
+class Severity(StrEnum):
+    SAFETY_CRITICAL = "safety_critical"
+    CAPABILITY = "capability"
+
+
+Check = Callable[[Transcript, ConversationLog], bool]
+
+
+@dataclass
+class Assertion:
+    name: str
+    severity: Severity
+    check: Check
+
+
+@dataclass(frozen=True)
+class AssertionResult:
+    name: str
+    severity: Severity
+    passed: bool
+    detail: str = ""
+
+
+class AssertionEngine:
+    def __init__(self, assertions: list[Assertion]) -> None:
+        self._assertions = assertions
+
+    def evaluate(self, transcript: Transcript, log: ConversationLog) -> list[AssertionResult]:
+        results = []
+        for assertion in self._assertions:
+            try:
+                passed = bool(assertion.check(transcript, log))
+                detail = ""
+            except Exception as exc:  # noqa: BLE001 - a raising check is a failed assertion
+                passed = False
+                detail = f"{type(exc).__name__}: {exc}"
+            results.append(
+                AssertionResult(
+                    name=assertion.name,
+                    severity=assertion.severity,
+                    passed=passed,
+                    detail=detail,
+                )
+            )
+        return results
+
+
+# --- a few common checks --------------------------------------------------------------
+
+
+def tool_was_called(tool_name: str) -> Check:
+    return lambda t, _log: any(c.name == tool_name for c in t.tool_calls)
+
+
+def no_tool_errors() -> Check:
+    return lambda t, _log: all(c.success is not False for c in t.tool_calls)
+
+
+def max_turns(limit: int) -> Check:
+    return lambda _t, log: log.turns_taken <= limit
+
+
+def max_total_tokens(limit: int) -> Check:
+    return lambda t, _log: sum(t.tokens()) <= limit

sonar_eval/config.py

@@ -0,0 +1,82 @@
+"""Task- and scenario-level configuration.
+
+TaskConfig holds global defaults; each ScenarioConfig overrides what it needs. Assertions
+are attached per scenario with a Severity that decides how they are graded (safety-critical
+=> pass^k must be 1.0; capability => pass@k).
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from enum import StrEnum
+from typing import TYPE_CHECKING, Any
+
+from .interceptor import FaultSpec
+from .llm import LLMConfig
+
+if TYPE_CHECKING:
+    from .assertions import Assertion
+
+
+class Fidelity(StrEnum):
+    REAL = "real"  # exercise the bot's real dependencies
+    MOCKED = "mocked"  # fast, deterministic, tests prompt logic not integration
+
+
+@dataclass(frozen=True)
+class Persona:
+    name: str
+    instructions: str
+
+
+@dataclass
+class ScenarioConfig:
+    name: str
+    user_instructions: str
+    assertions: list[Assertion] = field(default_factory=list)
+    k: int = 5
+    fidelity: Fidelity = Fidelity.MOCKED
+    faults: FaultSpec = field(default_factory=FaultSpec)
+    persona: Persona | None = None
+    seed: dict[str, Any] = field(default_factory=dict)
+    params: dict[str, Any] = field(default_factory=dict)
+    grader_rubric: str | None = None
+    user_llm: LLMConfig | None = None
+    grader_llm: LLMConfig | None = None
+    termination_signal: str = "[[END]]"
+    max_turns: int = 20
+
+
+@dataclass
+class TaskConfig:
+    """Global defaults shared across scenarios."""
+
+    ingest_base_url: str
+    tenant: str
+    user_llm: LLMConfig
+    grader_llm: LLMConfig
+    # Full-content project so the grader sees unredacted tool I/O rather than previews.
+    project: str = "eval-fullcontent"
+    api_key: str | None = None
+    api_key_header: str = "x-sonar-key"
+    default_k: int = 5
+    scenarios: list[ScenarioConfig] = field(default_factory=list)
+
+    def resolve(self, scenario: ScenarioConfig) -> ScenarioConfig:
+        """Fill scenario gaps from task-level defaults."""
+        return ScenarioConfig(
+            name=scenario.name,
+            user_instructions=scenario.user_instructions,
+            assertions=scenario.assertions,
+            k=scenario.k or self.default_k,
+            fidelity=scenario.fidelity,
+            faults=scenario.faults,
+            persona=scenario.persona,
+            seed=scenario.seed,
+            params=scenario.params,
+            grader_rubric=scenario.grader_rubric,
+            user_llm=scenario.user_llm or self.user_llm,
+            grader_llm=scenario.grader_llm or self.grader_llm,
+            termination_signal=scenario.termination_signal,
+            max_turns=scenario.max_turns,
+        )

sonar_eval/grader.py

@@ -0,0 +1,80 @@
+"""LLM rubric grader (tier 2).
+
+Consumes the full transcript - delivered messages *and* the tool-call view pulled from
+ingest - and returns a calibrated score + reasoning. The score is what a rubric can judge
+that a programmatic assertion cannot (tone, helpfulness, whether a refusal was appropriate).
+"""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Callable
+from dataclasses import dataclass
+
+from .llm import LLMClient, LLMConfig, Message
+from .simulator import ConversationLog
+from .transcript import Transcript
+
+_SYSTEM = """You are an evaluator scoring a chatbot conversation against a rubric.
+Return ONLY a JSON object: {"score": <float 0..1>, "reasoning": "<short explanation>"}."""
+
+
+@dataclass(frozen=True)
+class GraderResult:
+    score: float
+    reasoning: str
+    passed: bool
+
+
+Calibration = Callable[[GraderResult], GraderResult]
+
+
+class Grader:
+    def __init__(
+        self,
+        client: LLMClient,
+        config: LLMConfig,
+        rubric: str,
+        *,
+        pass_threshold: float = 0.7,
+        calibration: Calibration | None = None,
+    ) -> None:
+        self._client = client
+        self._config = config
+        self._rubric = rubric
+        self._threshold = pass_threshold
+        self._calibration = calibration
+
+    async def grade(self, transcript: Transcript, log: ConversationLog) -> GraderResult:
+        prompt = self._build_prompt(transcript, log)
+        raw = await self._client.complete(self._config, _SYSTEM, [Message("user", prompt)])
+        score, reasoning = _parse(raw)
+        result = GraderResult(score=score, reasoning=reasoning, passed=score >= self._threshold)
+        if self._calibration is not None:
+            result = self._calibration(result)
+        return result
+
+    def _build_prompt(self, transcript: Transcript, log: ConversationLog) -> str:
+        lines = [f"RUBRIC:\n{self._rubric}", "", "CONVERSATION (what the user saw):"]
+        for turn in log.turns:
+            lines.append(f"  user: {turn.user}")
+            for msg in turn.delivered:
+                lines.append(f"  bot: {msg}")
+        lines.append("")
+        lines.append("TOOL CALLS (bot internals):")
+        for call in transcript.tool_calls:
+            status = "ok" if call.success else (call.error_type or "failed")
+            lines.append(f"  {call.name} -> {status}")
+        return "\n".join(lines)
+
+
+def _parse(raw: str) -> tuple[float, str]:
+    start, end = raw.find("{"), raw.rfind("}")
+    if start != -1 and end != -1 and end > start:
+        try:
+            data = json.loads(raw[start : end + 1])
+            score = float(data.get("score", 0.0))
+            return max(0.0, min(1.0, score)), str(data.get("reasoning", ""))
+        except (ValueError, TypeError):
+            pass
+    return 0.0, f"unparseable grader output: {raw[:200]}"

sonar_eval/interceptor.py

@@ -0,0 +1,152 @@
+"""Tool/API fault injection.
+
+sonar-tracing only *observes* tool spans; it never sits in the execution path, so
+interception has to live at the dependency boundary the bot calls through. This module
+provides a declarative fault spec plus the two mechanisms the harness supports:
+
+  mocked fidelity  ProxyInterceptor - harness-owned, in front of mocked dependencies.
+  real fidelity    serialize_faults() - posted to the app's internal endpoint by the
+                   adapter, so the app's own tool dispatch can honour the rules.
+
+Rules are declarative so trials are reproducible (pass^k needs determinism). A content
+predicate (``when``) is available for the "fail only if it tried to charge >$100" case,
+but predicates only run in mocked fidelity - see serialize_faults().
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from dataclasses import dataclass, field, replace
+from enum import StrEnum
+from typing import Any, Protocol
+
+
+class Action(StrEnum):
+    ERROR = "error"  # raise an exception in place of the real call
+    TIMEOUT = "timeout"  # raise a timeout-shaped exception
+    REPLACE_OUTPUT = "replace_output"  # return payload instead of the real result
+    LATENCY = "latency"  # sleep payload seconds, then pass through
+
+
+@dataclass(frozen=True)
+class ToolCall:
+    target: str
+    args: dict[str, Any] = field(default_factory=dict)
+    call_index: int = 0  # 0-based nth invocation of this target in the conversation
+
+
+@dataclass(frozen=True)
+class FaultRule:
+    target: str
+    action: Action
+    when_call_index: int | None = None
+    when: Callable[[ToolCall], bool] | None = None
+    payload: Any = None
+
+    def matches(self, call: ToolCall) -> bool:
+        if call.target != self.target:
+            return False
+        if self.when_call_index is not None and call.call_index != self.when_call_index:
+            return False
+        return self.when is None or self.when(call)
+
+
+@dataclass(frozen=True)
+class FaultSpec:
+    rules: tuple[FaultRule, ...] = ()
+
+
+class DecisionKind(StrEnum):
+    PASSTHROUGH = "passthrough"
+    REPLACE = "replace"
+    RAISE = "raise"
+    DELAY = "delay"
+
+
+@dataclass(frozen=True)
+class InterceptDecision:
+    kind: DecisionKind
+    value: Any = None
+
+    @classmethod
+    def passthrough(cls) -> InterceptDecision:
+        return cls(DecisionKind.PASSTHROUGH)
+
+    @classmethod
+    def replace(cls, output: Any) -> InterceptDecision:
+        return cls(DecisionKind.REPLACE, output)
+
+    @classmethod
+    def raise_(cls, exc: BaseException) -> InterceptDecision:
+        return cls(DecisionKind.RAISE, exc)
+
+    @classmethod
+    def delay(cls, seconds: float) -> InterceptDecision:
+        return cls(DecisionKind.DELAY, seconds)
+
+
+class ToolInterceptor(Protocol):
+    def on_tool_call(self, call: ToolCall) -> InterceptDecision: ...
+
+
+class InjectedFault(RuntimeError):
+    """Raised by the proxy for Action.ERROR / Action.TIMEOUT."""
+
+
+def _decision_for(rule: FaultRule) -> InterceptDecision:
+    if rule.action is Action.ERROR:
+        return InterceptDecision.raise_(InjectedFault(f"injected error for {rule.target}"))
+    if rule.action is Action.TIMEOUT:
+        return InterceptDecision.raise_(TimeoutError(f"injected timeout for {rule.target}"))
+    if rule.action is Action.REPLACE_OUTPUT:
+        return InterceptDecision.replace(rule.payload)
+    if rule.action is Action.LATENCY:
+        return InterceptDecision.delay(float(rule.payload or 0.0))
+    return InterceptDecision.passthrough()
+
+
+class ProxyInterceptor:
+    """In-process interceptor for mocked-fidelity scenarios.
+
+    Construct one per trial; it owns per-target call counters so ``when_call_index``
+    is evaluated against the invocation order it observes, not a value the caller has
+    to compute.
+    """
+
+    def __init__(self, spec: FaultSpec) -> None:
+        self._spec = spec
+        self._counts: dict[str, int] = {}
+
+    def on_tool_call(self, call: ToolCall) -> InterceptDecision:
+        index = self._counts.get(call.target, 0)
+        self._counts[call.target] = index + 1
+        resolved = replace(call, call_index=index)
+        for rule in self._spec.rules:
+            if rule.matches(resolved):
+                return _decision_for(rule)
+        return InterceptDecision.passthrough()
+
+
+def serialize_faults(spec: FaultSpec) -> list[dict[str, Any]]:
+    """Serialise a FaultSpec for the real-fidelity adapter hook.
+
+    Predicate rules cannot cross a process boundary, so a ``when`` predicate is a hard
+    error here rather than a silent no-op - silently dropping it would make pass^k
+    non-reproducible in a way the author never sees. Use mocked fidelity for predicates.
+    """
+    wire: list[dict[str, Any]] = []
+    for rule in spec.rules:
+        if rule.when is not None:
+            raise ValueError(
+                f"fault rule for {rule.target!r} uses a `when` predicate, which only runs "
+                "in mocked fidelity; use `when_call_index` for real-fidelity scenarios"
+            )
+        wire.append(
+            {
+                "target": rule.target,
+                "action": rule.action.value,
+                "when_call_index": rule.when_call_index,
+                "payload": rule.payload,
+            }
+        )
+    return wire

sonar_eval/llm.py

@@ -0,0 +1,30 @@
+"""LLM abstraction for the user simulator and grader.
+
+The harness stays provider-agnostic: it never imports a vendor SDK. A project supplies
+one ``LLMClient`` and the harness drives both the user simulator and the grader through
+it. Defaulting to the latest Claude models is a project decision, made in the client the
+project wires in - not baked in here.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Protocol
+
+
+@dataclass(frozen=True)
+class Message:
+    role: str  # "system" | "user" | "assistant"
+    content: str
+
+
+@dataclass(frozen=True)
+class LLMConfig:
+    model: str
+    temperature: float = 1.0
+    max_tokens: int = 1024
+
+
+class LLMClient(Protocol):
+    async def complete(self, config: LLMConfig, system: str, messages: list[Message]) -> str:
+        """Return the assistant completion text for ``messages`` under ``system``."""