agent-learning-kit 0.1.0__py3-none-any.whl

fi/simulate/simulation/matrix.py

@@ -0,0 +1,170 @@
+"""Matrix runner: sweep a scenario across provider/channel legs.
+
+Composes an SDK simulation manifest with per-leg overrides so the same
+scenario can be executed against ``{Vapi, Retell, LiveKit}`` in
+``{webrtc, sip_outbound, sip_inbound}`` shapes. Legs that cannot run
+against a provider (e.g. Retell over PSTN outbound) are declared with
+``skip_reason`` and reported as ``UNSUPPORTED`` in the canonical output
+instead of being silently omitted.
+"""
+
+from __future__ import annotations
+
+import copy
+import logging
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Literal, Mapping
+
+from pydantic import BaseModel, Field
+
+from fi.simulate.runtime import (
+    FailureStage,
+    SimulationFailure,
+    TestCaseStatus,
+)
+
+logger = logging.getLogger(__name__)
+
+Channel = Literal["webrtc", "sip_outbound", "sip_inbound"]
+
+
+class MatrixLeg(BaseModel):
+    """One column of the provider × channel matrix."""
+
+    provider: str
+    channel: Channel
+    agent_definition_overrides: dict[str, Any] = Field(default_factory=dict)
+    provider_evidence_overrides: dict[str, Any] | None = None
+    skip_reason: str | None = None
+    max_concurrent_calls: int | None = None
+
+    @property
+    def label(self) -> str:
+        return f"{self.provider}:{self.channel}"
+
+
+@dataclass
+class MatrixLegResult:
+    leg: MatrixLeg
+    manifest: dict[str, Any]
+    started_at: datetime
+    ended_at: datetime | None = None
+    report: Any = None
+    error: str | None = None
+    skipped: bool = False
+    skipped_cases: list[dict[str, Any]] = field(default_factory=list)
+
+
+def _leg_manifest(base: Mapping[str, Any], leg: MatrixLeg) -> dict[str, Any]:
+    manifest = copy.deepcopy(dict(base))
+    agent_definition = dict(manifest.get("agent_definition") or {})
+    agent_definition.update(leg.agent_definition_overrides)
+    transport = dict(agent_definition.get("transport") or {})
+    transport["kind"] = leg.channel
+    if leg.channel == "webrtc":
+        for legacy in ("sip_trunk_id", "sip_number", "sip_call_to", "dispatch_rule_name"):
+            transport.pop(legacy, None)
+    agent_definition["transport"] = transport
+    if leg.provider_evidence_overrides is not None:
+        agent_definition["provider_evidence"] = leg.provider_evidence_overrides
+    manifest["agent_definition"] = agent_definition
+    simulation = dict(manifest.get("simulation") or {})
+    simulation["engine"] = simulation.get("engine", "livekit")
+    manifest["simulation"] = simulation
+    manifest["name"] = f"{manifest.get('name', 'matrix-run')}:{leg.label}"
+    return manifest
+
+
+def build_skipped_report(
+    leg: MatrixLeg,
+    base: Mapping[str, Any],
+    *,
+    now: datetime | None = None,
+) -> MatrixLegResult:
+    manifest = _leg_manifest(base, leg)
+    started = now or datetime.now(timezone.utc)
+    scenario = manifest.get("scenario") or {}
+    dataset = scenario.get("dataset") or []
+    skipped_cases = [
+        {
+            "index": index,
+            "persona": (row.get("persona") if isinstance(row, dict) else None),
+            "status": TestCaseStatus.UNSUPPORTED.value,
+            "failure": SimulationFailure(
+                stage=FailureStage.PREPARING,
+                code="provider_channel_unsupported",
+                message=leg.skip_reason or "provider_channel_unsupported",
+                retryable=False,
+                provider=leg.provider,
+                details={"channel": leg.channel},
+            ).model_dump(mode="json", exclude_none=True),
+        }
+        for index, row in enumerate(dataset)
+    ]
+    return MatrixLegResult(
+        leg=leg,
+        manifest=manifest,
+        started_at=started,
+        ended_at=started,
+        skipped=True,
+        skipped_cases=skipped_cases,
+    )
+
+
+async def run_matrix(
+    base_manifest: Mapping[str, Any],
+    legs: list[MatrixLeg],
+    *,
+    manifest_path: Path,
+    max_concurrent_calls: int = 1,
+    run_leg,
+) -> list[MatrixLegResult]:
+    """Sweep ``legs`` against ``base_manifest``.
+
+    ``run_leg`` is an async callable ``(manifest, manifest_path, leg)``
+    that executes one leg and returns the SDK report. It is injected
+    rather than imported here so this module remains framework-lean and
+    testable without a live LiveKit backend.
+    """
+
+    results: list[MatrixLegResult] = []
+    for leg in legs:
+        if leg.skip_reason:
+            skipped = build_skipped_report(leg, base_manifest)
+            logger.info(
+                "matrix_leg_skipped",
+                extra={"leg": leg.label, "reason": leg.skip_reason},
+            )
+            results.append(skipped)
+            continue
+        started = datetime.now(timezone.utc)
+        manifest = _leg_manifest(base_manifest, leg)
+        result = MatrixLegResult(leg=leg, manifest=manifest, started_at=started)
+        try:
+            result.report = await run_leg(
+                manifest=manifest,
+                manifest_path=manifest_path,
+                leg=leg,
+                max_concurrent_calls=leg.max_concurrent_calls or max_concurrent_calls,
+            )
+        except Exception as exc:  # noqa: BLE001
+            logger.warning(
+                "matrix_leg_error",
+                extra={"leg": leg.label, "error": type(exc).__name__},
+            )
+            result.error = f"{type(exc).__name__}: {exc}"
+        finally:
+            result.ended_at = datetime.now(timezone.utc)
+        results.append(result)
+    return results
+
+
+__all__ = [
+    "Channel",
+    "MatrixLeg",
+    "MatrixLegResult",
+    "build_skipped_report",
+    "run_matrix",
+]

fi/simulate/simulation/models.py

@@ -0,0 +1,279 @@
+from typing import List, Dict, Any, Union, Optional, Literal
+from pydantic import BaseModel, Field, validator, model_validator
+import hashlib
+import pandas as pd
+import json
+from fi.simulate.agent.wrapper import SimulationArtifact, SimulationEvent
+
+PERSONA_TEMPERAMENT_AXES = ("rajas", "sattva", "tamas")
+# Byte-equal to fi.opt.optimizers.council.GUNA_AXES (council.py:40) — pinned by
+# a cross-equality unit test, never imported (fi.simulate must not depend on
+# fi.opt). Scholarly design device used as deterministic engineering metadata —
+# same framing as council.py:36-38; the four honest limits of Phase-7
+# RESEARCH §3.4 are binding (design metaphor, not a psychometric claim about
+# simulated users; axes ship ONLY with a transcript-observable realization
+# metric — see behavior_policy.py).
+
+PERSONA_EVIDENCE_CLASSES = (
+    "hand_written", "schema_sampled", "policy_evolved",
+    "trace_mined", "cloud_downloaded", "legacy",
+)
+SCENARIO_KINDS = ("task", "adversarial", "regression", "perturbation", "composed")
+
+
+class PersonaIdentity(BaseModel):
+    """Layer 1 — minimal, behavioral-first. Demographics optional, non-default,
+    lint-flagged (P7-D4; SCOPE: demographics explain ~1.5% of behavioral
+    variance). NO field here ever backs a realism claim."""
+    name: Optional[str] = None
+    role: Optional[str] = None
+    summary: Optional[str] = None
+    language: Optional[str] = None                  # locale for bias-lint re-runs
+    demographics: Dict[str, Any] = Field(default_factory=dict)  # ALWAYS lint-flagged
+    style_notes: List[str] = Field(default_factory=list)        # verbatim vendor/platform text
+
+
+class PersonaTemperament(BaseModel):
+    """Layer 2 — continuous axes that COMPILE into layer 3 (behavior_policy.py);
+    never prose adjectives in a prompt (PPol / 2604.00026)."""
+    rajas: float = Field(0.5, ge=0.0, le=1.0)    # activation/urgency dial
+    sattva: float = Field(0.5, ge=0.0, le=1.0)   # clarity/cooperation dial
+    tamas: float = Field(0.5, ge=0.0, le=1.0)    # inertia/withdrawal dial
+
+
+class BehaviorPolicy(BaseModel):
+    """Layer 3 — the executable, searchable representation (PPol). The six
+    parameters map 1:1 onto the canon axes V1_PERSONA_BEHAVIOR_AXES
+    ["patience", "disclosure", "interruption", "escalation", "cooperation",
+    "repair"] (ARCH §4), each paired with its transcript-observable
+    realization metric (behavior_policy.py); a parameter without one DOES NOT
+    SHIP (R§3.4 limit 4). verbosity/tempo dials are POST-v1.x — they exceed
+    the closed axis set and ship no realization metric in v1 (ARCH Decision 4)."""
+    patience_curve: List[float] = Field(default_factory=lambda: [1.0])  # axis: patience — per-turn patience 0..1
+    disclosure_policy: float = Field(0.7, ge=0.0, le=1.0)   # axis: disclosure — fraction of known facts volunteered
+    interruption_propensity: float = Field(0.1, ge=0.0, le=1.0)  # axis: interruption
+    escalation_schedule: List[float] = Field(default_factory=lambda: [0.0])  # axis: escalation — per-turn pressure 0..1
+    cooperation_bounds: float = Field(0.8, ge=0.0, le=1.0)  # axis: cooperation — ceiling on helpfulness (anti-cooperative-bias)
+    repair_propensity: float = Field(0.5, ge=0.0, le=1.0)   # axis: repair — good-faith repair probability after misunderstanding
+
+
+class PersonaFact(BaseModel):
+    """Layer 4 — retrievable knowledge store (2603.19313: retrieved, not
+    prompt-stuffed). Goals are NOT here: the Scenario owns the task
+    (2601.15290 separation)."""
+    key: str
+    value: str
+    disclosure: Literal["volunteer", "on_request", "withhold"] = "on_request"
+
+
+class AttackConditioning(BaseModel):
+    """Optional red-team conditioning (PCAP). Values must be members of the
+    gate-enforced 10x6 taxonomy — membership is asserted FACADE-side
+    (studio.validate_persona) and by the gate, not here (fi.simulate must not
+    import fi.alk.trinity)."""
+    strategies: List[str] = Field(default_factory=list)   # ⊆ V1_REDTEAM_RESEARCH_ATTACK_TYPES
+    surfaces: List[str] = Field(default_factory=list)     # ⊆ V1_REDTEAM_RESEARCH_SURFACES
+    in_character_floor: float = Field(0.6, ge=0.0, le=1.0)
+
+
+class PersonaProvenance(BaseModel):
+    """Layer 5 — how the persona was made + what it is calibrated FOR.
+    No class ever claims population representativeness (2602.18462 hard
+    limit — stated here in the schema, not just docs)."""
+    evidence_class: Literal[
+        "hand_written", "schema_sampled", "policy_evolved",
+        "trace_mined", "cloud_downloaded", "legacy",
+    ] = "legacy"
+    calibrated: bool = False
+    calibration_ref: Optional[str] = None         # content hash of the calibration artifact
+    source_format: Optional[str] = None           # "vapi" | "retell" | "futureagi" | None
+    raw: Optional[str] = None                     # verbatim vendor/source text (ARCH Decision 8 losslessness)
+    pin: Dict[str, Any] = Field(default_factory=dict)   # download pin block (studio/_download.py)
+    representativeness_claim: Literal["none"] = "none"  # frozen; the schema-level hard limit
+
+
+class Persona(BaseModel):
+    """
+    A single test case defining a customer persona, situation, and desired outcome.
+    """
+    persona: Dict[str, Any] = Field(..., description="Characteristics of the simulated customer (e.g., name, age, communication_style).")
+    situation: str = Field(..., description="The context or reason for the customer's call.")
+    outcome: str = Field(..., description="The desired goal or resolution for the conversation.")
+    # ---- Phase 7 typed layers (ALL optional => full back-compat) ----------
+    identity: Optional[PersonaIdentity] = None
+    temperament: Optional[PersonaTemperament] = None
+    behavior_policy: Optional[BehaviorPolicy] = None
+    knowledge: List[PersonaFact] = Field(default_factory=list)
+    attack: Optional[AttackConditioning] = None
+    provenance: Optional[PersonaProvenance] = None
+    version: Optional[str] = None                 # content address, ARCH §2d
+
+    @property
+    def is_typed(self) -> bool:
+        """True when the persona carries an executable layer-3 policy —
+        the precondition for fidelity measurement (fidelity.py)."""
+        return self.behavior_policy is not None
+
+    def content_hash(self) -> str:
+        payload = self.model_dump(exclude={"version"}, exclude_none=True)
+        canonical = json.dumps(payload, sort_keys=True, separators=(",", ":"), default=str)
+        return "sha256:" + hashlib.sha256(canonical.encode("utf-8")).hexdigest()
+
+    @model_validator(mode="after")
+    def _stamp_version(self) -> "Persona":
+        if self.version is None and self.is_typed:
+            object.__setattr__(self, "version", self.content_hash())
+        return self
+
+
+class ScenarioGoal(BaseModel):
+    """Goal/state progression — the task-state half of the 2601.15290 split."""
+    states: List[str] = Field(default_factory=list)          # ordered milestone names
+    success_state: Optional[str] = None
+    failure_states: List[str] = Field(default_factory=list)
+
+
+class VerificationSpec(BaseModel):
+    checks: List[Dict[str, Any]] = Field(default_factory=list)  # eval-template refs / predicates
+    threshold: float = Field(0.7, ge=0.0, le=1.0)
+
+
+class CoverageDeclaration(BaseModel):
+    """Declared coverage axes (2605.26521 obligations, not counts)."""
+    intents: List[str] = Field(default_factory=list)
+    personas: List[str] = Field(default_factory=list)            # persona version hashes
+    perturbations: List[str] = Field(default_factory=list)
+    tool_obligations: List[str] = Field(default_factory=list)    # "allow:<tool>" / "deny:<tool>"
+    delegation_obligations: List[str] = Field(default_factory=list)
+
+
+class ScenarioConstraints(BaseModel):
+    """The tau^2 move: the scenario BOUNDS persona freedom so fidelity is
+    checkable — declared tools, observable state, goal machine."""
+    declared_tools: List[str] = Field(default_factory=list)
+    observable_state: Dict[str, Any] = Field(default_factory=dict)
+    max_user_knowledge: List[str] = Field(default_factory=list)  # PersonaFact keys usable here
+
+
+class EscalationStep(BaseModel):
+    turn: int = Field(..., ge=1)
+    pressure: float = Field(..., ge=0.0, le=1.0)
+    tactic: str                                    # free label, e.g. "reframe", "urgency", "authority"
+
+
+class EscalationArc(BaseModel):
+    """Turn-wise in-character escalation (Crescendo finding, R§1 2605.04019)."""
+    steps: List[EscalationStep]
+    hold_character: bool = True
+
+
+class Scenario(BaseModel):
+    """
+    Defines a collection of test cases for a simulation.
+    """
+    name: str = Field(..., description="A unique name for the scenario.")
+    description: Optional[str] = Field(None, description="A brief description of what this scenario tests.")
+    dataset: List[Persona] = Field(..., description="A list of personas defining the test cases.")
+    # ---- Phase 7 typing (ALL optional; kind=None == legacy untyped — NEVER
+    # silently retyped; studio-created scenarios must carry an explicit kind;
+    # the gate requires kinds only on studio-library scenarios — ARCH §2a) ----
+    kind: Optional[Literal["task", "adversarial", "regression", "perturbation", "composed"]] = None
+    goal: Optional[ScenarioGoal] = None
+    verification: Optional[VerificationSpec] = None
+    coverage: Optional[CoverageDeclaration] = None
+    constraints: Optional[ScenarioConstraints] = None
+    escalation: Optional[EscalationArc] = None
+    attack_type: Optional[str] = None              # adversarial kind: required
+    attack_surface: Optional[str] = None           # adversarial kind: required
+    version: Optional[str] = None                  # content address, ARCH §2d
+    parent_version: Optional[str] = None           # expansion lineage (studio/_coverage.py)
+
+    @validator('dataset', pre=True)
+    def load_dataset(cls, v: Union[List[Dict], str]) -> List[Dict]:
+        if isinstance(v, str):
+            if v.endswith('.csv'):
+                return pd.read_csv(v).to_dict('records')
+            elif v.endswith('.json'):
+                with open(v, 'r') as f:
+                    return json.load(f)
+            else:
+                raise ValueError("Unsupported file type for dataset. Please use .csv or .json.")
+        return v
+
+    def content_hash(self) -> str:                 # same canonicalization as Persona
+        payload = self.model_dump(exclude={"version"}, exclude_none=True)
+        canonical = json.dumps(payload, sort_keys=True, separators=(",", ":"), default=str)
+        return "sha256:" + hashlib.sha256(canonical.encode("utf-8")).hexdigest()
+
+    @model_validator(mode="after")
+    def _kind_contract(self) -> "Scenario":
+        if self.kind == "adversarial":
+            if not (self.attack_type and self.attack_surface and self.escalation):
+                raise ValueError(
+                    "adversarial scenarios must declare attack_type, "
+                    "attack_surface, and an escalation arc"
+                )
+        if self.kind is not None and self.version is None:
+            object.__setattr__(self, "version", self.content_hash())
+        return self
+
+class TestCaseResult(BaseModel):
+    """
+    Represents the result of a single test case.
+    """
+    persona: Persona = Field(..., description="The original persona that was run.")
+    transcript: str = Field(..., description="The full transcript of the conversation.")
+    messages: List[Dict[str, Any]] = Field(
+        default_factory=list,
+        description="Normalized message trajectory including user, assistant, and tool turns.",
+    )
+    tool_calls: List[Dict[str, Any]] = Field(
+        default_factory=list,
+        description="Tool calls observed during the run, when wrappers expose them.",
+    )
+    artifacts: List[SimulationArtifact] = Field(
+        default_factory=list,
+        description="Multimodal artifacts observed during the run, such as audio, images, screenshots, files, and traces.",
+    )
+    events: List[SimulationEvent] = Field(
+        default_factory=list,
+        description="Normalized simulation events, including tools, memory, voice, browser/CUA, and framework spans.",
+    )
+    metadata: Dict[str, Any] = Field(
+        default_factory=dict,
+        description="Engine, scenario, timing, stop reason, and other run metadata.",
+    )
+    evaluation: dict | None = Field(
+        default=None,
+        description="Optional evaluation results (scores, reasons) keyed by template.",
+    )
+    audio_input_path: str | None = Field(
+        default=None,
+        description="Optional path to recorded customer (input) audio for this test.",
+    )
+    audio_output_path: str | None = Field(
+        default=None,
+        description="Optional path to recorded agent (output) audio for this test.",
+    )
+    audio_combined_path: str | None = Field(
+        default=None,
+        description="Optional path to a single WAV containing the mixed conversation.",
+    )
+    audio_stereo_path: str | None = Field(
+        default=None,
+        description="Optional path to a 2-channel WAV: ch0 customer, ch1 assistant.",
+    )
+
+class TestReport(BaseModel):
+    """
+    A comprehensive report aggregating the results of all test cases in a scenario.
+    """
+    results: List[TestCaseResult] = Field(default_factory=list, description="A list of results for each test case.")
+
+    def admissible_results(self) -> List[TestCaseResult]:
+        return [r for r in self.results
+                if r.metadata.get("admission", {}).get("admissible", True)]
+
+    def inconclusive_results(self) -> List[TestCaseResult]:
+        return [r for r in self.results
+                if r.metadata.get("admission", {}).get("verdict") == "inconclusive"]

fi/simulate/simulation/runner.py

@@ -0,0 +1,153 @@
+from typing import Optional, Callable
+import os
+
+from fi.simulate.agent.definition import (
+    AgentDefinition,
+    LiveKitSimulatorRuntime,
+    SimulatorAgentDefinition,
+)
+from fi.simulate.simulation.models import Scenario, TestReport
+from fi.simulate.simulation.engines import (
+    LiveKitEngine,
+    BaseEngine,
+    CloudEngine,
+    LocalTextEngine,
+)
+
+
+class TestRunner:
+    """
+    Main entry point for running agent simulations.
+
+    Supports three execution modes:
+    1. Local mode (LiveKit): Uses LiveKit to connect to deployed agents
+    2. Cloud mode (Backend API): Uses Future AGI backend for orchestrated testing
+    3. Local text mode: Runs a self-contained text simulator against an agent callback
+
+    The mode is automatically determined based on the arguments provided.
+    """
+
+    def __init__(
+        self,
+        api_key: Optional[str] = None,
+        secret_key: Optional[str] = None,
+        api_url: Optional[str] = None,
+    ):
+        """
+        Initialize the TestRunner.
+
+        Args:
+            api_key: Optional API key for cloud mode. If not provided, will check FI_API_KEY env var.
+            secret_key: Optional Secret key for cloud mode. If not provided, will check FI_SECRET_KEY env var.
+            api_url: Optional API URL for cloud mode. If not provided, will check FI_BASE_URL env var.
+        """
+        self.api_key = api_key or os.environ.get("FI_API_KEY")
+        self.secret_key = secret_key or os.environ.get("FI_SECRET_KEY")
+        self.api_url = api_url or os.environ.get("FI_BASE_URL")
+        self._engine: Optional[BaseEngine] = None
+
+    async def run_test(
+        self,
+        # --- Local Mode Arguments (LiveKit) ---
+        agent_definition: Optional[AgentDefinition] = None,
+        livekit_runtime: Optional[LiveKitSimulatorRuntime] = None,
+        scenario: Optional[Scenario] = None,
+        simulator: Optional[SimulatorAgentDefinition] = None,
+        # --- Cloud Mode Arguments (Backend API) ---
+        run_id: Optional[str] = None,
+        run_test_name: Optional[str] = None,
+        agent_callback: Optional[Callable] = None,
+        # --- Shared Arguments ---
+        num_scenarios: int = 1,
+        topic: Optional[str] = None,
+        simulation_run_id: Optional[str] = None,
+        record_audio: bool = False,
+        recorder_sample_rate: int = 8000,
+        recorder_join_delay: float = 0.2,
+        min_turn_messages: int = 8,
+        max_seconds: float = 45.0,
+        **kwargs,
+    ) -> TestReport:
+        """
+        Run a test simulation.
+
+        Mode is determined by arguments:
+        - If `run_id` or `run_test_name` is provided → Cloud mode (Backend API)
+        - If `agent_definition` is provided → Local mode (LiveKit)
+
+        Args:
+            agent_definition: Target-agent configuration for local mode
+            livekit_runtime: FutureAGI LiveKit simulator runtime
+            scenario: Test scenario for local mode
+            simulator: Simulator configuration for local mode
+            run_id: Run ID from platform for cloud mode
+            run_test_name: Run test name (alternative to run_id) - will fetch ID from backend
+            agent_callback: User's agent function to wrap for cloud mode
+            num_scenarios: Number of scenarios to generate (local mode only)
+            topic: Topic for scenario generation (local mode only)
+            simulation_run_id: Optional stable run ID for local LiveKit mode
+            record_audio: Whether to record audio
+            recorder_sample_rate: Audio sample rate
+            recorder_join_delay: Delay before recorder joins
+            min_turn_messages: Minimum turn messages
+            max_seconds: Maximum test duration
+            **kwargs: Additional arguments passed to engine
+
+        Returns:
+            TestReport with results from all test cases
+        """
+        # Dispatch to appropriate engine
+        if run_id is not None or run_test_name is not None:
+            # Cloud mode - Use CloudEngine
+            timeout = kwargs.pop("timeout", 120.0)  # Default 120s for LLM operations
+            engine = CloudEngine(
+                self.api_key, self.secret_key, self.api_url, timeout=timeout
+            )
+            return await engine.run(
+                run_id=run_id,
+                run_test_name=run_test_name,
+                agent_callback=agent_callback,
+                **kwargs,
+            )
+        elif agent_callback is not None:
+            # Local text mode - no backend, LiveKit, or model dependency required
+            engine = LocalTextEngine()
+            return await engine.run(
+                scenario=scenario,
+                agent_callback=agent_callback,
+                num_scenarios=num_scenarios,
+                topic=topic,
+                run_id=simulation_run_id,
+                **kwargs,
+            )
+
+        elif agent_definition is not None:
+            # Local mode - use LiveKit engine
+            if LiveKitEngine is None:
+                raise ImportError(
+                    "LiveKit mode requires the LiveKit dependency, but it is not available in this environment. "
+                    "Install it (and compatible plugins) then retry. Cloud mode (run_id/run_test_name) does not "
+                    "require LiveKit."
+                )
+            engine = LiveKitEngine()
+            return await engine.run(
+                agent_definition=agent_definition,
+                livekit_runtime=livekit_runtime,
+                scenario=scenario,
+                simulator=simulator,
+                num_scenarios=num_scenarios,
+                topic=topic,
+                record_audio=record_audio,
+                recorder_sample_rate=recorder_sample_rate,
+                recorder_join_delay=recorder_join_delay,
+                min_turn_messages=min_turn_messages,
+                max_seconds=max_seconds,
+                run_id=simulation_run_id,
+                **kwargs,
+            )
+        else:
+            raise ValueError(
+                "Must provide either 'agent_definition' (Local/LiveKit mode) "
+                "'agent_callback' with a scenario/topic (Local text mode), "
+                "or 'run_id'/'run_test_name' (Cloud/Backend API mode)."
+            )