agent-learning-kit 0.1.0__py3-none-any.whl

fi/simulate/environments/voice.py

@@ -0,0 +1,212 @@
+"""Voice environment plugin (canonical plan §3/§7.2-7.4).
+
+This is the registry-facing wrapper that finally routes voice through the same
+``SimulationRunner`` spine as chat. It does *not* reimplement the voice engine —
+it hydrates the typed inputs from ``spec.environment.config`` and drives the
+existing, working ``run_voice_simulation`` (LiveKit engine), returning the legacy
+``TestReport`` the runner converts uniformly.
+
+The voice config is secret-free by construction: providers are referenced by
+``api_key_env`` / ``api_secret_env`` (env var *names*), never raw values, so the
+whole config embeds inside the validated ``SimulationSpec`` without tripping
+``_reject_resolved_secrets``. Secrets reach the child process through the
+environment, resolved by the runner activity.
+
+Distinct from :class:`fi.simulate.environment.VoiceEnvironment`, which is the
+deterministic *replay* adapter (a sync ``EnvironmentAdapter`` fixture) — a
+different role, kept under its existing public name.
+"""
+
+from __future__ import annotations
+
+import inspect
+import logging
+
+from fi.simulate.environments.base import EnvironmentManifest
+from fi.simulate.registry import register_environment
+from fi.simulate.runtime.capabilities import EndpointCapabilities
+from fi.simulate.runtime.failures import FailureStage, SimulationFailure
+from fi.simulate.runtime.report import SimulationReport
+from fi.simulate.runtime.run import RunStatus, TestCaseStatus
+from fi.simulate.simulation.models import TestReport
+
+_logger = logging.getLogger(__name__)
+
+# Kwargs VoiceEnvironmentPlugin.run already binds explicitly when it calls
+# run_voice_simulation (see below) — reserved so a same-named key surviving in
+# hosted voice.params cannot collide with them at the call site.
+_PLUGIN_OWNED = frozenset(
+    {
+        "agent_definition",
+        "livekit_runtime",
+        "scenario",
+        "simulator",
+        "simulation_run_id",
+        "on_case_complete",
+        "on_case_start",
+    }
+)
+
+
+def _filter_hosted_voice_params(params: dict, run_voice_simulation) -> dict:
+    """Drop platform-sent keys run_voice_simulation can't bind: it is
+    keyword-only with a closed parameter list, so one stray key would raise
+    TypeError at hydration and kill the whole hosted run. (A **kwargs sink, if
+    one is ever added, accepts everything, so nothing is dropped.)"""
+    parameters = inspect.signature(run_voice_simulation).parameters
+    if any(p.kind is inspect.Parameter.VAR_KEYWORD for p in parameters.values()):
+        return dict(params)
+    accepted = {
+        name
+        for name, parameter in parameters.items()
+        if parameter.kind is not inspect.Parameter.VAR_KEYWORD
+    } - _PLUGIN_OWNED
+    dropped = set(params) - accepted
+    if dropped:
+        _logger.warning("hosted_voice_params_ignored", extra={"keys": sorted(dropped)})
+    return {key: value for key, value in params.items() if key in accepted}
+
+
+@register_environment("voice")
+class VoiceEnvironmentPlugin:
+    manifest = EnvironmentManifest(
+        name="voice",
+        world_kinds=["voice_telephony", "voice"],
+        capabilities=EndpointCapabilities(
+            audio=True,
+            streaming=True,
+            interruption=True,
+            recording=True,
+            transcript_events=True,
+            web_rtc=True,
+        ),
+    )
+
+    async def run(
+        self,
+        spec,
+        *,
+        target=None,
+        artifacts=None,
+        events=None,
+        environment=None,
+        auto_execute_tools: bool = True,
+        stop_when=None,
+        agent_wrapper_kwargs=None,
+        on_case_complete=None,
+        on_case_start=None,
+    ) -> TestReport:
+        from fi.simulate import voice as voice_api
+        from fi.simulate.agent.definition import (
+            AgentDefinition,
+            LiveKitSimulatorRuntime,
+            SimulatorAgentDefinition,
+        )
+
+        config = dict(spec.environment.config or {})
+        agent_definition = AgentDefinition.model_validate(config["agent_definition"])
+        livekit_runtime = (
+            LiveKitSimulatorRuntime.model_validate(config["livekit_runtime"])
+            if config.get("livekit_runtime")
+            else None
+        )
+        simulator = (
+            SimulatorAgentDefinition.model_validate(config["simulator"])
+            if config.get("simulator")
+            else None
+        )
+        params = _filter_hosted_voice_params(
+            dict(config.get("params") or {}), voice_api.run_voice_simulation
+        )
+
+        # When streaming, score the case's goal (if any) BEFORE the case is
+        # submitted, so the streamed payload carries ``goal_machine`` metadata
+        # identically to the post-run report — then forward to the runner's sink
+        # callback. Absent streaming, the report-level pass below is authoritative.
+        streamed_callback = None
+        if on_case_complete is not None:
+            scenario = spec.scenario
+
+            async def streamed_callback(index, case):  # noqa: ANN001
+                self._attach_goal_to_case(scenario, case)
+                await on_case_complete(index, case)
+
+        report = await voice_api.run_voice_simulation(
+            agent_definition=agent_definition,
+            livekit_runtime=livekit_runtime,
+            scenario=spec.scenario,
+            simulator=simulator,
+            simulation_run_id=spec.run_id,
+            on_case_complete=streamed_callback,
+            on_case_start=on_case_start,
+            **params,
+        )
+        self._attach_goal_machine(spec.scenario, report)
+        return report
+
+    @classmethod
+    def _attach_goal_machine(cls, scenario, report: TestReport) -> None:
+        """Voice world-contract (plan §1.9, settle-only). A declared
+        ``scenario.goal`` is scored over each case transcript at episode end and
+        attached as metadata — the same idiom chat uses. No declared goal ⇒ no-op
+        (byte-identical). Voice has no per-turn hook, so this scores the run; it
+        does not (and must not) early-stop a live call or fail the run.
+
+        Idempotent: overwrites ``goal_machine`` with the same value the streaming
+        path already wrote, so streamed and reconciled cases stay identical.
+        """
+        for case in report.results:
+            cls._attach_goal_to_case(scenario, case)
+
+    @staticmethod
+    def _attach_goal_to_case(scenario, case) -> None:
+        """Score ``scenario.goal`` over one case's transcript, in place."""
+        goal = getattr(scenario, "goal", None)
+        if goal is None:
+            return
+        from fi.simulate.simulation import goal_machine
+
+        verification = getattr(scenario, "verification", None)
+        settle = goal_machine.evaluate_settle(
+            goal,
+            verification,
+            environment_state={},
+            world_status={},
+            messages=getattr(case, "messages", None) or [],
+        )
+        case.metadata["goal_machine"] = {
+            "states_reached": settle.get("states_reached", []),
+            "stop_reason": None,
+            "checks": settle.get("checks", []),
+        }
+
+    def finalize_run_status(self, report: SimulationReport) -> SimulationReport:
+        """A voice run whose only case(s) failed is a failed job, not COMPLETED.
+
+        ``from_legacy`` carries per-case status but the overall status is the
+        environment's to decide (plan §3: environments enforce terminal
+        conditions). Chat keeps COMPLETED; only voice downgrades.
+        """
+        failed = [
+            case
+            for case in report.test_cases
+            if case.status is not TestCaseStatus.COMPLETED
+        ]
+        if not report.test_cases or len(failed) != len(report.test_cases):
+            return report
+        failure = failed[0].failure or SimulationFailure(
+            stage=FailureStage.RUNNING,
+            code="voice_run_failed",
+            message="voice simulation failed",
+            retryable=False,
+        )
+        return SimulationReport.model_validate(
+            {
+                **report.model_dump(exclude={"report_hash"}),
+                "status": RunStatus.FAILED.value,
+                "failure": failure.model_dump(),
+            }
+        )
+
+
+__all__ = ["VoiceEnvironmentPlugin"]

fi/simulate/evaluation/__init__.py

@@ -0,0 +1,4 @@
+from .ai_eval import evaluate_agent_report, evaluate_report
+
+__all__ = ["evaluate_agent_report", "evaluate_report"]
+

fi/simulate/evaluation/ai_eval.py

@@ -0,0 +1,227 @@
+from __future__ import annotations
+
+from typing import Any, Iterable, Mapping, Sequence
+import os
+import base64
+import json
+
+from fi.simulate.simulation.models import TestReport
+
+
+def evaluate_agent_report(
+    report: TestReport,
+    *,
+    config: Mapping[str, Any] | None = None,
+    threshold: float = 0.7,
+    attach: bool = True,
+) -> Any:
+    """
+    Evaluate a simulation report locally with Agent Learning eval metrics.
+
+    This is the no-cloud path for the trinity loop:
+    Agent Learning simulation captures messages/tool calls/events/artifacts,
+    Agent Learning evals score the agent trajectory and pentest signals, and
+    Agent Learning optimization can optimize against the attached numeric
+    scores.
+
+    Returns the AgentReportEvaluation object. When `attach=True`,
+    each TestCaseResult receives an `evaluation["agent_report"]` payload and
+    aggregate summary is copied into `result.metadata["agent_report_summary"]`.
+    """
+
+    try:
+        from fi.evals.metrics.agents import evaluate_agent_report as evaluate
+    except Exception as e:  # pragma: no cover - import error clarity
+        raise RuntimeError(
+            "Agent Learning Kit eval metrics are required. "
+            "Reinstall `agent-learning-kit`."
+        ) from e
+
+    evaluation = evaluate(report, config=dict(config or {}), threshold=threshold)
+    if attach:
+        _attach_agent_report_evaluation(report, evaluation)
+    return evaluation
+
+
+def evaluate_report(
+    report: TestReport,
+    *,
+    eval_templates: Iterable[str] | None = ("task_completion", "tone", "is_helpful"),
+    eval_specs: Sequence[dict] | None = None,
+    model_name: str = "turing_flash",
+    api_key: str | None = None,
+    secret_key: str | None = None,
+    extra_inputs: Mapping[str, str] | None = None,
+) -> TestReport:
+    """
+    Evaluate each test case transcript using Agent Learning evals.
+
+    - Templates like "task_completion" will receive input and output fields
+      mapped from persona and transcript.
+    - "tone" will receive the whole transcript as input.
+
+    Docs: https://docs.futureagi.com/future-agi/get-started/evaluation/running-your-first-eval#evaluate-using-sdk
+    """
+
+    try:
+        from fi.evals import Evaluator
+    except Exception as e:  # pragma: no cover - import error clarity
+        raise RuntimeError(
+            "Agent Learning Kit eval engine is required. Reinstall `agent-learning-kit`."
+        ) from e
+
+    evaluator = Evaluator(fi_api_key=api_key, fi_secret_key=secret_key)
+
+    for result in report.results:
+        persona = result.persona
+        transcript = result.transcript
+
+        scores: dict[str, dict] = {}
+
+        def resolve_source(key: str) -> str | None:
+            if key == "transcript":
+                return transcript
+            if key == "messages":
+                return json.dumps(result.messages, default=str)
+            if key == "tool_calls":
+                return json.dumps(result.tool_calls, default=str)
+            if key == "artifacts":
+                return json.dumps([_model_to_dict(item) for item in result.artifacts], default=str)
+            if key == "events":
+                return json.dumps([_model_to_dict(item) for item in result.events], default=str)
+            if key == "metadata":
+                return json.dumps(result.metadata, default=str)
+            if key == "persona":
+                return json.dumps(persona.persona, default=str)
+            if key == "persona.situation":
+                return persona.situation
+            if key == "persona.outcome":
+                return persona.outcome
+            if key == "audio_input_path":
+                val = getattr(result, "audio_input_path", None)
+                return os.path.abspath(val) if val and os.path.exists(val) else val
+            if key == "audio_output_path":
+                val = getattr(result, "audio_output_path", None)
+                return os.path.abspath(val) if val and os.path.exists(val) else val
+            if key == "audio_combined_path":
+                val = getattr(result, "audio_combined_path", None)
+                return os.path.abspath(val) if val and os.path.exists(val) else val
+            return None
+
+        def _encode_audio_inputs(inputs: dict[str, str]) -> dict[str, str]:
+            """Strict encoding: if a value is a local audio file path, replace that value with base64.
+
+            - Never rename keys or add aliases.
+            - Do not add extra fields (no audio_mime or data URI).
+            """
+            audio_exts = {".wav", ".ogg", ".mp3", ".m4a", ".flac", ".aac"}
+            for k, v in list(inputs.items()):
+                if isinstance(v, str) and os.path.exists(v):
+                    _, ext = os.path.splitext(v.lower())
+                    if ext in audio_exts:
+                        try:
+                            with open(v, "rb") as f:
+                                data = f.read()
+                            inputs[k] = base64.b64encode(data).decode("ascii")
+                        except Exception:
+                            # Leave as-is on read failure
+                            pass
+            return inputs
+
+        # If eval_specs provided, use explicit mappings per template
+        if eval_specs:
+            for spec in eval_specs:
+                template = spec.get("template")
+                mapping: Mapping[str, str] = spec.get("map", {})  # desired_input_key -> source_key
+                if not template:
+                    continue
+                inputs: dict[str, str] = {}
+                for dest, source in mapping.items():
+                    val = resolve_source(source)
+                    if val is not None:
+                        inputs[dest] = val
+                if extra_inputs:
+                    inputs.update(extra_inputs)
+                inputs = _encode_audio_inputs(inputs)
+                try:
+                    ev = evaluator.evaluate(eval_templates=template, inputs=inputs, model_name=model_name)
+                    item = ev.eval_results[0] if ev and getattr(ev, "eval_results", None) else None
+                    scores[template] = {
+                        "output": getattr(item, "output", None),
+                        "reason": getattr(item, "reason", None),
+                        "score": getattr(item, "score", None),
+                    }
+                except Exception as e:
+                    scores[template] = {"error": str(e), "inputs": inputs}
+        else:
+            # Fallback: simple built-ins by template name
+            for template in (eval_templates or []):
+                inputs: dict[str, str] = {}
+                if template == "tone":
+                    inputs = {"input": transcript}
+                elif template == "task_completion":
+                    inputs = {"input": persona.situation, "output": transcript}
+                elif template == "is_helpful":
+                    inputs = {"input": transcript}
+                else:
+                    inputs = {"input": transcript}
+
+                if extra_inputs:
+                    inputs.update(extra_inputs)
+                inputs = _encode_audio_inputs(inputs)
+
+                try:
+                    ev = evaluator.evaluate(eval_templates=template, inputs=inputs, model_name=model_name)
+                    item = ev.eval_results[0] if ev and getattr(ev, "eval_results", None) else None
+                    scores[template] = {
+                        "output": getattr(item, "output", None),
+                        "reason": getattr(item, "reason", None),
+                        "score": getattr(item, "score", None),
+                    }
+                except Exception as e:
+                    scores[template] = {"error": str(e)}
+
+        result.evaluation = scores
+
+    return report
+
+
+def _model_to_dict(value):
+    if hasattr(value, "model_dump"):
+        return value.model_dump()
+    if hasattr(value, "dict"):
+        return value.dict()
+    if isinstance(value, list):
+        return [_model_to_dict(item) for item in value]
+    if isinstance(value, tuple):
+        return [_model_to_dict(item) for item in value]
+    if isinstance(value, dict):
+        return {key: _model_to_dict(item) for key, item in value.items()}
+    if hasattr(value, "__dict__"):
+        return {key: _model_to_dict(item) for key, item in vars(value).items()}
+    return value
+
+
+def _attach_agent_report_evaluation(report: TestReport, evaluation: Any) -> None:
+    cases = getattr(evaluation, "cases", []) or []
+    summary = _model_to_dict(getattr(evaluation, "summary", {}))
+    aggregate = {
+        "score": getattr(evaluation, "score", None),
+        "passed": getattr(evaluation, "passed", None),
+        "threshold": getattr(evaluation, "threshold", None),
+        "summary": summary,
+    }
+    for index, result in enumerate(report.results):
+        case = cases[index] if index < len(cases) else None
+        metrics = getattr(case, "metrics", []) if case is not None else []
+        payload = {
+            **aggregate,
+            "case_score": getattr(case, "score", None) if case is not None else None,
+            "case_passed": getattr(case, "passed", None) if case is not None else None,
+            "metrics": [_model_to_dict(metric) for metric in metrics],
+            "findings": _model_to_dict(getattr(case, "findings", [])) if case is not None else [],
+        }
+        result.evaluation = dict(result.evaluation or {})
+        result.evaluation["agent_report"] = payload
+        result.metadata = dict(result.metadata or {})
+        result.metadata["agent_report_summary"] = aggregate

fi/simulate/evidence/__init__.py

@@ -0,0 +1,35 @@
+from .base import (
+    AgentEvidenceSource,
+    EvidenceCapabilities,
+    EvidenceClass,
+    EvidenceSourceSpec,
+    EvidenceSourceSummary,
+)
+from .caller_observed import CallerObservedEvidenceSource
+from .livekit_instrumentation import LiveKitAgentInstrumentationSource
+from .livekit_room import LiveKitRoomDataEvidenceSource
+from .otel import OpenTelemetryEvidenceSource
+from .providers import (
+    EvidenceContext,
+    ProviderConfigError,
+    ProviderFetchResult,
+    RetellEvidenceSource,
+    VapiEvidenceSource,
+)
+
+__all__ = [
+    "AgentEvidenceSource",
+    "CallerObservedEvidenceSource",
+    "EvidenceCapabilities",
+    "EvidenceClass",
+    "EvidenceContext",
+    "EvidenceSourceSpec",
+    "EvidenceSourceSummary",
+    "LiveKitAgentInstrumentationSource",
+    "LiveKitRoomDataEvidenceSource",
+    "OpenTelemetryEvidenceSource",
+    "ProviderConfigError",
+    "ProviderFetchResult",
+    "RetellEvidenceSource",
+    "VapiEvidenceSource",
+]

fi/simulate/evidence/base.py

@@ -0,0 +1,59 @@
+from __future__ import annotations
+
+from enum import Enum
+from typing import Protocol
+
+from pydantic import BaseModel, Field, JsonValue
+
+
+class EvidenceClass(str, Enum):
+    CALLER_OBSERVED = "caller_observed"
+    PROVIDER_REPORTED = "provider_reported"
+    AGENT_INSTRUMENTED = "agent_instrumented"
+    PLATFORM_VERIFIED = "platform_verified"
+
+
+class EvidenceCapabilities(BaseModel):
+    transcript: bool = False
+    audio: bool = False
+    tool_calls: bool = False
+    tool_results: bool = False
+    usage: bool = False
+    internal_latency: bool = False
+    configuration_snapshot: bool = False
+
+    def supported(self) -> set[str]:
+        return {
+            name
+            for name, enabled in self.model_dump().items()
+            if enabled
+        }
+
+
+class EvidenceSourceSpec(BaseModel):
+    source_id: str
+    adapter: str
+    adapter_version: str = "1"
+    evidence_class: EvidenceClass
+    capabilities: EvidenceCapabilities = Field(default_factory=EvidenceCapabilities)
+    config: dict[str, JsonValue] = Field(default_factory=dict)
+
+
+class EvidenceSourceSummary(BaseModel):
+    source_id: str
+    adapter: str
+    evidence_class: EvidenceClass
+    capabilities: EvidenceCapabilities = Field(default_factory=EvidenceCapabilities)
+    available: bool = True
+    redactions: list[str] = Field(default_factory=list)
+    metadata: dict[str, JsonValue] = Field(default_factory=dict)
+
+
+class AgentEvidenceSource(Protocol):
+    capabilities: EvidenceCapabilities
+
+    async def connect(self, context: object) -> None: ...
+
+    async def fetch_final(self) -> object: ...
+
+    async def close(self) -> None: ...

fi/simulate/evidence/caller_observed.py

@@ -0,0 +1,50 @@
+"""CallerObservedEvidenceSource skeleton (plan §6).
+
+Reports whatever the SDK's own recorder + transcript captured for the
+simulator leg (audio, transcript, timing) without consulting provider
+APIs or agent instrumentation. Real per-track summary derivation lands
+alongside the media router in §5.
+"""
+
+from __future__ import annotations
+
+import uuid
+
+from .base import EvidenceCapabilities, EvidenceClass, EvidenceSourceSummary
+from .providers.base import EvidenceContext, ProviderFetchResult
+
+
+class CallerObservedEvidenceSource:
+    capabilities = EvidenceCapabilities(
+        transcript=True,
+        audio=True,
+        tool_calls=False,
+        tool_results=False,
+        usage=False,
+        internal_latency=False,
+        configuration_snapshot=False,
+    )
+
+    def __init__(self) -> None:
+        self._source_id = f"caller_observed:{uuid.uuid4().hex[:12]}"
+        self._context: EvidenceContext | None = None
+
+    async def connect(self, context: EvidenceContext) -> None:
+        self._context = context
+
+    async def fetch_final(self) -> ProviderFetchResult:
+        summary = EvidenceSourceSummary(
+            source_id=self._source_id,
+            adapter="caller_observed",
+            evidence_class=EvidenceClass.CALLER_OBSERVED,
+            capabilities=self.capabilities,
+            available=False,
+            metadata={"reason": "not_implemented"},
+        )
+        return ProviderFetchResult(summary=summary, artifacts=[])
+
+    async def close(self) -> None:
+        return None
+
+
+__all__ = ["CallerObservedEvidenceSource"]

fi/simulate/evidence/livekit_instrumentation.py

@@ -0,0 +1,51 @@
+"""LiveKitAgentInstrumentationSource skeleton (plan §6.6).
+
+Consumes canonical events emitted by
+``fi.simulate.instrumentation.livekit.FutureAGIObserver`` and surfaces
+them as agent-instrumented evidence. Bridging the observer's in-process
+event stream into a case-scoped ``EvidenceSourceSummary`` lands with
+Stage 7.
+"""
+
+from __future__ import annotations
+
+import uuid
+
+from .base import EvidenceCapabilities, EvidenceClass, EvidenceSourceSummary
+from .providers.base import EvidenceContext, ProviderFetchResult
+
+
+class LiveKitAgentInstrumentationSource:
+    capabilities = EvidenceCapabilities(
+        transcript=True,
+        audio=False,
+        tool_calls=True,
+        tool_results=True,
+        usage=True,
+        internal_latency=True,
+        configuration_snapshot=True,
+    )
+
+    def __init__(self) -> None:
+        self._source_id = f"livekit_instrumentation:{uuid.uuid4().hex[:12]}"
+        self._context: EvidenceContext | None = None
+
+    async def connect(self, context: EvidenceContext) -> None:
+        self._context = context
+
+    async def fetch_final(self) -> ProviderFetchResult:
+        summary = EvidenceSourceSummary(
+            source_id=self._source_id,
+            adapter="livekit_instrumentation",
+            evidence_class=EvidenceClass.AGENT_INSTRUMENTED,
+            capabilities=self.capabilities,
+            available=False,
+            metadata={"reason": "not_implemented"},
+        )
+        return ProviderFetchResult(summary=summary, artifacts=[])
+
+    async def close(self) -> None:
+        return None
+
+
+__all__ = ["LiveKitAgentInstrumentationSource"]

fi/simulate/evidence/livekit_room.py

@@ -0,0 +1,50 @@
+"""LiveKitRoomDataEvidenceSource skeleton (plan §6.5).
+
+Streams participant events, tracks, and data-channel messages via the
+LiveKit room API. This skeleton establishes the seam for Stage 7 and
+declares the capabilities LiveKit-native evidence carries even before
+the collector is wired up.
+"""
+
+from __future__ import annotations
+
+import uuid
+
+from .base import EvidenceCapabilities, EvidenceClass, EvidenceSourceSummary
+from .providers.base import EvidenceContext, ProviderFetchResult
+
+
+class LiveKitRoomDataEvidenceSource:
+    capabilities = EvidenceCapabilities(
+        transcript=False,
+        audio=True,
+        tool_calls=False,
+        tool_results=False,
+        usage=False,
+        internal_latency=False,
+        configuration_snapshot=True,
+    )
+
+    def __init__(self) -> None:
+        self._source_id = f"livekit_room:{uuid.uuid4().hex[:12]}"
+        self._context: EvidenceContext | None = None
+
+    async def connect(self, context: EvidenceContext) -> None:
+        self._context = context
+
+    async def fetch_final(self) -> ProviderFetchResult:
+        summary = EvidenceSourceSummary(
+            source_id=self._source_id,
+            adapter="livekit_room",
+            evidence_class=EvidenceClass.PLATFORM_VERIFIED,
+            capabilities=self.capabilities,
+            available=False,
+            metadata={"reason": "not_implemented"},
+        )
+        return ProviderFetchResult(summary=summary, artifacts=[])
+
+    async def close(self) -> None:
+        return None
+
+
+__all__ = ["LiveKitRoomDataEvidenceSource"]