agent-learning-kit 0.1.0__py3-none-any.whl

fi/simulate/utils/routes.py

@@ -0,0 +1,164 @@
+import httpx
+from typing import Dict, Any, List, Optional
+
+class APIRoutes:
+    """
+    Handles API interactions with the Future AGI backend.
+    """
+    def __init__(self, api_key: str, secret_key: str, base_url: str, timeout: float = 120.0):
+        """
+        Args:
+            api_key: API key for authentication
+            secret_key: Secret key for authentication
+            base_url: Base URL of the backend API
+            timeout: Request timeout in seconds (default: 120s for LLM operations)
+        """
+        self.api_key = api_key
+        self.secret_key = secret_key
+        self.base_url = base_url.rstrip("/")
+        self.headers = {
+            "x-api-key": self.api_key,
+            "x-secret-key": self.secret_key,
+            "Content-Type": "application/json"
+        }
+        # Using a single client for connection pooling
+        # Increased timeout for chat operations which may involve LLM calls
+        self.client = httpx.AsyncClient(
+            base_url=self.base_url,
+            headers=self.headers,
+            timeout=timeout
+        )
+
+    async def close(self):
+        await self.client.aclose()
+    
+    async def get_run_test_id_by_name(self, run_test_name: str) -> Dict[str, Any]:
+        """
+        GET /simulate/run-tests/get-id-by-name/{run_test_name}/
+        Gets the run_test_id by run_test_name.
+        """
+        url = f"/simulate/run-tests/get-id-by-name/{run_test_name}/"
+        response = await self.client.get(url)
+        if response.is_error:
+            self._handle_error(response, f"Failed to get run_test_id for name '{run_test_name}'")
+        return response.json()
+    
+    def _handle_error(self, response: httpx.Response, operation: str) -> None:
+        """
+        Raises a more informative error with backend response details.
+        """
+        try:
+            error_body = response.json()
+            # Extract common error message patterns from backend
+            if isinstance(error_body, dict):
+                # Try to extract the most relevant error message
+                error_msg_parts = []
+                if "result" in error_body:
+                    error_msg_parts.append(str(error_body["result"]))
+                if "error" in error_body:
+                    error_msg_parts.append(str(error_body["error"]))
+                if "message" in error_body:
+                    error_msg_parts.append(str(error_body["message"]))
+                if "detail" in error_body:
+                    error_msg_parts.append(str(error_body["detail"]))
+                
+                if error_msg_parts:
+                    backend_error = " | ".join(error_msg_parts)
+                else:
+                    backend_error = str(error_body)
+            else:
+                backend_error = str(error_body)
+        except Exception:
+            error_body = response.text or f"<No response body (status {response.status_code})>"
+            backend_error = error_body
+        
+        # Simple error message with just status code and backend error
+        error_msg = f"{response.status_code}: {backend_error}"
+        
+        raise httpx.HTTPStatusError(
+            error_msg,
+            request=response.request,
+            response=response
+        )
+
+    async def start_test_execution(self, run_test_id: str) -> Dict[str, Any]:
+        """
+        POST /simulate/run-tests/{run_test_id}/chat-execute/
+        Starts a test execution and returns the execution ID.
+        Note: The backend uses scenarios associated with the run_test_id.
+        """
+        url = f"/simulate/run-tests/{run_test_id}/chat-execute/"
+        # Empty body - backend uses scenarios from run_test
+        response = await self.client.post(url, json={})
+        if response.is_error:
+            self._handle_error(response, f"Failed to start test execution for run_id '{run_test_id}'")
+        return response.json()
+
+    async def fetch_execution_batch(
+        self, 
+        test_execution_id: str
+    ) -> Dict[str, Any]:
+        """
+        POST /simulate/test-executions/{test_execution_id}/chat/call-executions/batch/
+        Creates a batch of call execution IDs (has side effects - creates CallExecution records).
+        Returns: {"call_execution_ids": [...], "has_more": bool, "batched_scenarios": [...]}
+        """
+        url = f"/simulate/test-executions/{test_execution_id}/chat/call-executions/batch/"
+        
+        response = await self.client.post(url, json={})
+        if response.is_error:
+            self._handle_error(response, f"Failed to fetch execution batch for test_execution_id '{test_execution_id}'")
+        return response.json()
+
+    async def send_chat_message(
+        self, 
+        call_execution_id: str, 
+        messages: List[Dict[str, str]] = None,
+        metrics: Dict[str, float | int] = None,
+        initiate_chat: bool = False
+    ) -> Dict[str, Any]:
+        """
+        POST /simulate/call-executions/{call_execution_id}/chat/send-message/
+        Sends a message to a chat execution.
+        """
+        url = f"/simulate/call-executions/{call_execution_id}/chat/send-message/"
+        
+        payload = {
+            "messages": messages,
+            "metrics": metrics,
+            "initiate_chat": initiate_chat
+        }
+        # Filter None values (but keep False for booleans if needed, though backend defaults to False)
+        # We explicitly keep initiate_chat if it's True
+        payload = {k: v for k, v in payload.items() if v is not None}
+        
+        response = await self.client.post(url, json=payload)
+        if response.is_error:
+            self._handle_error(response, f"Failed to send chat message for call_execution_id '{call_execution_id}'")
+        return response.json()
+
+    async def update_call_execution_status(
+        self,
+        call_execution_id: str,
+        status: str,
+        ended_reason: Optional[str] = None
+    ) -> Dict[str, Any]:
+        """
+        PATCH /simulate/call-executions/{call_execution_id}/
+        Updates the status of a call execution.
+        
+        Args:
+            call_execution_id: The ID of the call execution to update
+            status: The new status (must be a valid CallStatus choice, e.g., "FAILED", "CANCELLED")
+            ended_reason: Optional reason for ending the call execution
+        """
+        url = f"/simulate/call-executions/{call_execution_id}/"
+        payload = {"status": status}
+        if ended_reason is not None:
+            payload["ended_reason"] = ended_reason
+        
+        response = await self.client.patch(url, json=payload)
+        if response.is_error:
+            self._handle_error(response, f"Failed to update call execution status for call_execution_id '{call_execution_id}'")
+        return response.json()
+

fi/simulate/voice.py

@@ -0,0 +1,225 @@
+from __future__ import annotations
+
+import asyncio
+from collections.abc import Awaitable, Callable, Mapping, Sequence
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from fi.alk.studio import GeneratedScenario
+
+from fi.simulate.agent.definition import (
+    AgentDefinition,
+    LiveKitSimulatorRuntime,
+    SimulatorAgentDefinition,
+)
+from fi.simulate.simulation.models import Scenario, TestReport
+from fi.simulate.simulation.runner import TestRunner
+
+_RUN_VERSION = "agent-learning.run.v1"
+
+
+async def run_voice_simulation(
+    *,
+    agent_definition: AgentDefinition,
+    livekit_runtime: LiveKitSimulatorRuntime | None = None,
+    scenario: Scenario | None = None,
+    simulator: SimulatorAgentDefinition | None = None,
+    topic: str | None = None,
+    num_scenarios: int = 1,
+    simulation_run_id: str | None = None,
+    record_audio: bool = False,
+    recording_root: str | Path = "recordings",
+    recording_case_directory: str | Path | None = None,
+    recorder_sample_rate: int = 8000,
+    recorder_join_delay: float = 0.2,
+    min_turn_messages: int = 8,
+    max_seconds: float = 45.0,
+    connect_timeout: float = 15.0,
+    readiness_timeout: float = 30.0,
+    cleanup_timeout: float = 30.0,
+    conversation_direction: str = "simulator_first",
+    agent_first_silence_timeout_seconds: float = 30.0,
+    max_concurrency: int = 1,
+    on_case_complete: Callable[[int, Any], Awaitable[None]] | None = None,
+    on_case_start: Callable[[int], Awaitable[None]] | None = None,
+) -> TestReport:
+    """Run a LiveKit voice simulation directly from typed SDK objects."""
+
+    if scenario is not None and topic is not None:
+        raise ValueError("scenario and topic are mutually exclusive")
+    if scenario is None and not topic:
+        raise ValueError("provide scenario or topic for scenario generation")
+    return await TestRunner().run_test(
+        agent_definition=agent_definition,
+        livekit_runtime=livekit_runtime,
+        scenario=scenario,
+        simulator=simulator,
+        topic=topic,
+        num_scenarios=num_scenarios,
+        simulation_run_id=simulation_run_id,
+        record_audio=record_audio,
+        recording_root=recording_root,
+        recording_case_directory=recording_case_directory,
+        recorder_sample_rate=recorder_sample_rate,
+        recorder_join_delay=recorder_join_delay,
+        min_turn_messages=min_turn_messages,
+        max_seconds=max_seconds,
+        connect_timeout=connect_timeout,
+        readiness_timeout=readiness_timeout,
+        cleanup_timeout=cleanup_timeout,
+        conversation_direction=conversation_direction,
+        agent_first_silence_timeout_seconds=agent_first_silence_timeout_seconds,
+        max_concurrency=max_concurrency,
+        on_case_complete=on_case_complete,
+        on_case_start=on_case_start,
+    )
+
+
+async def generate_platform_voice_scenario(
+    *,
+    agent_definition: AgentDefinition,
+    name: str,
+    description: str | None = None,
+    custom_instruction: str | None = None,
+    no_of_rows: int = 10,
+    poll_interval_seconds: float = 2.0,
+    timeout_seconds: float = 900.0,
+    config: Any | None = None,
+) -> "GeneratedScenario":
+    """Create a platform Agent Definition and generate its typed Scenario."""
+
+    from fi.alk import studio
+
+    request = studio.PlatformScenarioRequest(
+        name=name,
+        agent_definition=agent_definition,
+        description=description,
+        custom_instruction=custom_instruction,
+        no_of_rows=no_of_rows,
+        poll_interval_seconds=poll_interval_seconds,
+        timeout_seconds=timeout_seconds,
+    )
+    return await asyncio.to_thread(studio.generate_scenario, request, config=config)
+
+
+def build_voice_run_manifest(
+    *,
+    agent_definition: AgentDefinition,
+    scenario: Scenario,
+    simulator: SimulatorAgentDefinition | None = None,
+    livekit_runtime: LiveKitSimulatorRuntime | None = None,
+    name: str | None = None,
+    required_env: Sequence[str] = (),
+    simulation_run_id: str | None = None,
+    record_audio: bool = False,
+    recording_root: str | Path = "recordings",
+    recording_case_directory: str | Path | None = None,
+    recorder_sample_rate: int = 8000,
+    recorder_join_delay: float = 0.2,
+    min_turn_messages: int = 8,
+    max_seconds: float = 45.0,
+    connect_timeout: float = 15.0,
+    readiness_timeout: float = 30.0,
+    cleanup_timeout: float = 30.0,
+    conversation_direction: str = "simulator_first",
+    agent_first_silence_timeout_seconds: float = 30.0,
+    evaluation_enabled: bool = True,
+    evaluation_config: Mapping[str, Any] | None = None,
+    threshold: float = 0.7,
+) -> dict[str, Any]:
+    """Build the portable manifest for a typed LiveKit voice simulation."""
+
+    agent = AgentDefinition.model_validate(agent_definition)
+    typed_scenario = Scenario.model_validate(scenario)
+    typed_simulator = (
+        SimulatorAgentDefinition.model_validate(simulator)
+        if simulator is not None
+        else None
+    )
+    typed_runtime = (
+        LiveKitSimulatorRuntime.model_validate(livekit_runtime)
+        if livekit_runtime is not None
+        else None
+    )
+    simulation: dict[str, Any] = {
+        "engine": "livekit",
+        "modality": "voice",
+        "record_audio": record_audio,
+        "recording_root": str(recording_root),
+        "recorder_sample_rate": recorder_sample_rate,
+        "recorder_join_delay": recorder_join_delay,
+        "min_turn_messages": min_turn_messages,
+        "max_seconds": max_seconds,
+        "connect_timeout": connect_timeout,
+        "readiness_timeout": readiness_timeout,
+        "cleanup_timeout": cleanup_timeout,
+        "conversation_direction": conversation_direction,
+        "agent_first_silence_timeout_seconds": agent_first_silence_timeout_seconds,
+    }
+    if recording_case_directory is not None:
+        simulation["recording_case_directory"] = str(recording_case_directory)
+    if typed_runtime is not None:
+        simulation["livekit_runtime"] = typed_runtime.model_dump(
+            mode="json", exclude_none=True
+        )
+    if simulation_run_id:
+        simulation["run_id"] = simulation_run_id
+    manifest: dict[str, Any] = {
+        "version": _RUN_VERSION,
+        "name": name or f"{agent.name}-voice-simulation",
+        "required_env": _voice_required_env(
+            agent,
+            typed_runtime,
+            required_env,
+        ),
+        "agent_definition": agent.model_dump(
+            mode="json",
+            exclude_none=True,
+            exclude_unset=True,
+        ),
+        "scenario": typed_scenario.model_dump(mode="json", exclude_none=True),
+        "simulation": simulation,
+        "evaluation": {
+            "enabled": evaluation_enabled,
+            "agent_report": {
+                "config": dict(evaluation_config or {}),
+                "threshold": threshold,
+            },
+        },
+    }
+    if typed_simulator is not None:
+        manifest["simulator"] = typed_simulator.model_dump(
+            mode="json", exclude_none=True
+        )
+    return manifest
+
+
+def _voice_required_env(
+    agent_definition: AgentDefinition,
+    livekit_runtime: LiveKitSimulatorRuntime | None,
+    required_env: Sequence[str],
+) -> list[str]:
+    names = [
+        livekit_runtime.api_key_env if livekit_runtime else "LIVEKIT_API_KEY",
+        livekit_runtime.api_secret_env if livekit_runtime else "LIVEKIT_API_SECRET",
+        *required_env,
+    ]
+    transport = agent_definition.transport
+    target = agent_definition.target
+    if target is not None:
+        names.append(target.api_key_env)
+    if transport is not None:
+        from fi.simulate.endpoints.profiles import get_profile
+
+        profile = get_profile(transport.kind)
+        if profile is not None:
+            names.extend(profile.required_env(agent_definition))
+    return list(dict.fromkeys(str(name) for name in names if str(name).strip()))
+
+
+__all__ = [
+    "build_voice_run_manifest",
+    "generate_platform_voice_scenario",
+    "run_voice_simulation",
+]

fi/simulate/voice_cli.py

@@ -0,0 +1,182 @@
+from __future__ import annotations
+
+import argparse
+import time
+from pathlib import Path
+from typing import Any, Callable, Mapping
+
+from pydantic import ValidationError
+
+from fi.simulate.agent.definition import (
+    AgentDefinition,
+    LiveKitSimulatorRuntime,
+    SimulatorAgentDefinition,
+)
+from fi.simulate.manifest import ManifestError, validate_manifest_env
+from fi.simulate.simulation.models import Scenario
+from fi.simulate.voice import build_voice_run_manifest, run_voice_simulation
+
+
+def add_voice_arguments(parser: argparse.ArgumentParser) -> None:
+    parser.add_argument(
+        "--agent-definition",
+        required=True,
+        help="Path to an AgentDefinition JSON/YAML file.",
+    )
+    scenarios = parser.add_mutually_exclusive_group(required=True)
+    scenarios.add_argument("--scenario", help="Path to a Scenario JSON/YAML file.")
+    scenarios.add_argument("--topic", help="Generate scenarios from this topic.")
+    parser.add_argument(
+        "--simulator", help="Optional SimulatorAgentDefinition JSON/YAML file."
+    )
+    parser.add_argument(
+        "--livekit-runtime",
+        help="FutureAGI LiveKitSimulatorRuntime JSON/YAML file.",
+    )
+    parser.add_argument("--num-scenarios", type=int, default=1)
+    parser.add_argument("--run-id", default=None)
+    parser.add_argument("--name", default=None)
+    parser.add_argument("--record-audio", action="store_true")
+    parser.add_argument("--recording-root", default="recordings")
+    parser.add_argument("--recorder-sample-rate", type=int, default=8000)
+    parser.add_argument("--recorder-join-delay", type=float, default=0.2)
+    parser.add_argument("--min-turn-messages", type=int, default=8)
+    parser.add_argument("--max-seconds", type=float, default=45.0)
+    parser.add_argument("--connect-timeout", type=float, default=15.0)
+    parser.add_argument("--readiness-timeout", type=float, default=30.0)
+    parser.add_argument("--cleanup-timeout", type=float, default=30.0)
+    parser.add_argument(
+        "--conversation-direction",
+        choices=["simulator_first", "agent_first"],
+        default="simulator_first",
+    )
+    parser.add_argument(
+        "--agent-first-silence-timeout-seconds",
+        type=float,
+        default=30.0,
+    )
+    parser.add_argument(
+        "--write-manifest",
+        help="Write a portable manifest; requires --scenario.",
+    )
+    parser.add_argument("-o", "--output", action="append", default=[])
+    parser.add_argument("--junit", action="append", default=[])
+    parser.add_argument("--threshold", type=float, default=None)
+    parser.add_argument("--no-eval", action="store_true")
+    parser.add_argument("--dry-run", action="store_true")
+    parser.add_argument("--quiet", action="store_true")
+
+
+async def run_voice_command(
+    args: argparse.Namespace,
+    *,
+    load_object: Callable[[Path], dict[str, Any]],
+    write_manifest: Callable[[Mapping[str, Any], str | Path], Path],
+    evaluate_report: Callable[[Mapping[str, Any], Any], Any],
+    result_builder: Callable[..., dict[str, Any]],
+    write_outputs: Callable[
+        [dict[str, Any], Mapping[str, Any], argparse.Namespace, Path], dict[str, Any]
+    ],
+) -> dict[str, Any]:
+    agent_path = Path(args.agent_definition).expanduser().resolve()
+    try:
+        agent_definition = AgentDefinition(**load_object(agent_path))
+        scenario = (
+            Scenario(**load_object(Path(args.scenario).expanduser().resolve()))
+            if args.scenario
+            else None
+        )
+        simulator = (
+            SimulatorAgentDefinition(
+                **load_object(Path(args.simulator).expanduser().resolve())
+            )
+            if args.simulator
+            else None
+        )
+        livekit_runtime = (
+            LiveKitSimulatorRuntime(
+                **load_object(Path(args.livekit_runtime).expanduser().resolve())
+            )
+            if args.livekit_runtime
+            else None
+        )
+    except ValidationError as exc:
+        raise ManifestError(f"invalid typed voice input: {exc}") from exc
+    if args.write_manifest and scenario is None:
+        raise ManifestError("--write-manifest requires --scenario")
+
+    manifest = (
+        build_voice_run_manifest(
+            agent_definition=agent_definition,
+            scenario=scenario,
+            simulator=simulator,
+            livekit_runtime=livekit_runtime,
+            name=args.name,
+            simulation_run_id=args.run_id,
+            record_audio=args.record_audio,
+            recording_root=args.recording_root,
+            recorder_sample_rate=args.recorder_sample_rate,
+            recorder_join_delay=args.recorder_join_delay,
+            min_turn_messages=args.min_turn_messages,
+            max_seconds=args.max_seconds,
+            connect_timeout=args.connect_timeout,
+            readiness_timeout=args.readiness_timeout,
+            cleanup_timeout=args.cleanup_timeout,
+            conversation_direction=args.conversation_direction,
+            agent_first_silence_timeout_seconds=(
+                args.agent_first_silence_timeout_seconds
+            ),
+            evaluation_enabled=not args.no_eval,
+            threshold=args.threshold if args.threshold is not None else 0.7,
+        )
+        if scenario is not None
+        else {"name": args.name or f"{agent_definition.name}-voice-simulation"}
+    )
+    if args.write_manifest:
+        write_manifest(manifest, args.write_manifest)
+    if args.dry_run:
+        if scenario is None:
+            raise ManifestError("--dry-run requires --scenario")
+        validate_manifest_env(manifest)
+        result = {
+            "schema_version": "agent-simulate.cli.v1",
+            "name": manifest["name"],
+            "status": "passed",
+            "exit_code": 0,
+            "dry_run": True,
+            "summary": {"scenario_cases": len(scenario.dataset)},
+            "duration_seconds": 0.0,
+        }
+        return write_outputs(result, manifest, args, agent_path)
+
+    started = time.monotonic()
+    report = await run_voice_simulation(
+        agent_definition=agent_definition,
+        scenario=scenario,
+        simulator=simulator,
+        livekit_runtime=livekit_runtime,
+        topic=args.topic,
+        num_scenarios=args.num_scenarios,
+        simulation_run_id=args.run_id,
+        record_audio=args.record_audio,
+        recording_root=args.recording_root,
+        recorder_sample_rate=args.recorder_sample_rate,
+        recorder_join_delay=args.recorder_join_delay,
+        min_turn_messages=args.min_turn_messages,
+        max_seconds=args.max_seconds,
+        connect_timeout=args.connect_timeout,
+        readiness_timeout=args.readiness_timeout,
+        cleanup_timeout=args.cleanup_timeout,
+        conversation_direction=args.conversation_direction,
+        agent_first_silence_timeout_seconds=(
+            args.agent_first_silence_timeout_seconds
+        ),
+    )
+    evaluation = None if args.no_eval else evaluate_report(manifest, report)
+    result = result_builder(
+        manifest=manifest,
+        report=report,
+        evaluation=evaluation,
+        duration_seconds=round(time.monotonic() - started, 4),
+    )
+    return write_outputs(result, manifest, args, agent_path)

fi/utils/__init__.py

@@ -0,0 +1 @@
+"""Shared utilities — HTTP auth, routes, error types, base URL helpers."""

fi/utils/constants.py

@@ -0,0 +1,14 @@
+import os
+
+SECRET_KEY_ENVVAR_NAME = "FI_SECRET_KEY"
+API_KEY_ENVVAR_NAME = "FI_API_KEY"
+
+
+def get_base_url() -> str:
+    """Return the api base URL from the environment, falling back to prod."""
+    return os.getenv("FI_BASE_URL", "https://api.futureagi.com")
+
+
+DEFAULT_TIMEOUT = 200
+DEFAULT_MAX_WORKERS = 8
+DEFAULT_MAX_QUEUE = 5000