agent-learning-kit 0.1.0__py3-none-any.whl

fi/simulate/hosted/child_entrypoint.py

@@ -0,0 +1,306 @@
+"""Child process a ``simulation-runner`` worker spawns per hosted job.
+
+    python -m fi.simulate.hosted.child_entrypoint <job.json> [--status-file PATH]
+
+It runs the released SDK for the job's mode and submits results through
+``FutureAGIResultSink``. It is the only place hosted execution differs from a
+local run — the simulation itself is the same ``SimulationRunner``/engine code.
+
+Lifecycle is reported as newline-delimited JSON ``RunnerJobStatus`` objects, both
+to stdout (the worker tails these for Temporal heartbeats) and to an optional
+status file. SIGTERM triggers a graceful cancel + cleanup.
+
+Slice 1 wires the chat mode only; the voice modes raise until their slices land.
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import json
+import logging
+import os
+import signal
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+from fi.simulate.results.futureagi import FutureAGIResultSink
+from fi.simulate.runtime.report import SimulationReport
+from fi.simulate.runtime.run import RunStatus
+from fi.simulate.runtime.runner import SimulationRunner
+
+from .job import RunnerJobPhase, RunnerJobStatus, RunnerMode, StartRunnerJob
+from .targets import resolve_chat_target
+
+if TYPE_CHECKING:
+    from fi.simulate.runtime.spec import SimulationSpec
+
+_HEARTBEAT_INTERVAL_SECONDS = 10.0
+_CANCEL_GRACE_SECONDS = 30.0
+
+logger = logging.getLogger("fi.simulate.hosted.runner")
+
+
+def _job_log_fields(job: StartRunnerJob) -> dict[str, Any]:
+    fields: dict[str, Any] = {"job_id": job.job_id, "mode": job.mode.value}
+    if job.voice is not None:
+        target = dict(job.voice.agent_definition or {}).get("target") or {}
+        fields["provider"] = target.get("provider")
+        dataset = dict(job.voice.scenario or {}).get("dataset") or []
+        fields["cases"] = len(dataset)
+    if job.sink is not None:
+        fields["run_test_id"] = job.sink.run_test_id
+        fields["test_execution_id"] = job.sink.test_execution_id
+    return fields
+
+
+class _StatusReporter:
+    def __init__(self, job_id: str, status_file: Path | None) -> None:
+        self._job_id = job_id
+        self._status_file = status_file
+
+    def emit(
+        self,
+        phase: RunnerJobPhase,
+        *,
+        detail: str | None = None,
+        report_hash: str | None = None,
+        submission_status: str | None = None,
+    ) -> None:
+        status = RunnerJobStatus(
+            job_id=self._job_id,
+            phase=phase,
+            detail=detail,
+            report_hash=report_hash,
+            submission_status=submission_status,
+            updated_at=datetime.now(timezone.utc),
+        )
+        line = status.model_dump_json()
+        print(line, flush=True)
+        if self._status_file is not None:
+            with self._status_file.open("a", encoding="utf-8") as handle:
+                handle.write(line + "\n")
+
+
+def _load_job(path: Path) -> StartRunnerJob:
+    return StartRunnerJob.model_validate_json(path.read_text(encoding="utf-8"))
+
+
+def _build_sink(job: StartRunnerJob) -> FutureAGIResultSink:
+    root = job.sink.root_directory or os.environ.get("FI_RUN_ROOT") or ".fagi/runs"
+    return FutureAGIResultSink(
+        root=root,
+        api_url=job.sink.api_url,
+        run_test_id=job.sink.run_test_id,
+        test_execution_id=job.sink.test_execution_id,
+    )
+
+
+def _read_submission(run_directory: Path | None) -> dict[str, Any]:
+    if run_directory is None:
+        return {}
+    submission_path = run_directory / "submission.json"
+    if not submission_path.exists():
+        return {}
+    try:
+        return json.loads(submission_path.read_text(encoding="utf-8"))
+    except (ValueError, OSError):
+        return {}
+
+
+def _build_voice_spec(job: StartRunnerJob) -> "SimulationSpec":
+    """Translate a voice job into a ``SimulationSpec`` so the voice run flows
+    through the same ``SimulationRunner`` spine as chat (plan §3). The typed voice
+    inputs ride in ``environment.config`` — secret-free, since providers are
+    referenced by ``*_env`` name, never raw values. ``transport.kind`` selects the
+    target adapter. The DID pool is leased by the runner activity (telephone
+    only), not here — the leased number arrives via the agent definition / params.
+    """
+    from fi.simulate.runtime import new_run_id
+    from fi.simulate.runtime.spec import (
+        AgentEndpointSpec,
+        EnvironmentSpec,
+        EvidencePolicy,
+        ExecutionPolicy,
+        SimulationSpec,
+        SimulatorPolicySpec,
+        TimeoutPolicy,
+    )
+    from fi.simulate.simulation.models import Scenario
+
+    cfg = job.voice
+    run_id = str((job.spec.run_id if job.spec else None) or new_run_id())
+    params = dict(cfg.params or {})
+    transport = (dict(cfg.agent_definition or {}).get("transport") or {})
+    transport_kind = transport.get("kind") or "livekit"
+
+    # The runner's outer deadline must clear the voice call's own budget.
+    run_seconds = max(
+        300.0,
+        float(params.get("max_seconds", 45.0))
+        + float(params.get("connect_timeout", 15.0))
+        + float(params.get("readiness_timeout", 30.0))
+        + float(params.get("cleanup_timeout", 30.0))
+        + 60.0,
+    )
+
+    return SimulationSpec(
+        run_id=run_id,
+        environment=EnvironmentSpec(
+            adapter="voice",
+            world_kind="voice",
+            config={
+                "agent_definition": cfg.agent_definition,
+                "livekit_runtime": cfg.livekit_runtime,
+                "simulator": cfg.simulator,
+                "params": cfg.params,
+            },
+        ),
+        target=AgentEndpointSpec(adapter=transport_kind),
+        simulator=SimulatorPolicySpec(adapter="livekit_simulator"),
+        scenario=Scenario.model_validate(cfg.scenario),
+        execution=ExecutionPolicy(timeout=TimeoutPolicy(run_seconds=run_seconds)),
+        evidence=EvidencePolicy(),
+    )
+
+
+async def _heartbeat(reporter: _StatusReporter) -> None:
+    while True:
+        await asyncio.sleep(_HEARTBEAT_INTERVAL_SECONDS)
+        reporter.emit(RunnerJobPhase.RUNNING, detail="heartbeat")
+
+
+async def _execute(job: StartRunnerJob, reporter: _StatusReporter) -> int:
+    reporter.emit(RunnerJobPhase.PREPARING)
+    logger.info("hosted job start", extra=_job_log_fields(job))
+    sink = _build_sink(job)
+
+    if job.mode is RunnerMode.CHAT:
+        target = resolve_chat_target(job.spec)
+        run_coro = SimulationRunner().run(job.spec, target=target, result_sink=sink)
+    elif job.mode.is_voice:
+        run_coro = SimulationRunner().run(_build_voice_spec(job), result_sink=sink)
+    else:
+        raise NotImplementedError(f"runner mode not wired: {job.mode.value}")
+
+    run_task = asyncio.ensure_future(run_coro)
+    heartbeat_task = asyncio.ensure_future(_heartbeat(reporter))
+    reporter.emit(RunnerJobPhase.RUNNING)
+    try:
+        report: SimulationReport = await run_task
+    except asyncio.CancelledError:
+        reporter.emit(RunnerJobPhase.CANCELED, detail="cancelled")
+        logger.warning("hosted job cancelled", extra={"job_id": job.job_id})
+        # Cancelling this coroutine does not cancel ``run_task``; without an
+        # explicit cancel ``asyncio.run`` shutdown waits on it forever and the
+        # child leaks past SIGTERM.
+        run_task.cancel()
+        try:
+            await asyncio.wait({run_task}, timeout=_CANCEL_GRACE_SECONDS)
+        except asyncio.CancelledError:
+            pass
+        if not run_task.done():
+            os._exit(2)
+        raise
+    finally:
+        heartbeat_task.cancel()
+
+    reporter.emit(RunnerJobPhase.FINALIZING)
+    submission = _read_submission(sink.run_directory)
+    submission_status = submission.get("status")
+    run_completed = report.status is RunStatus.COMPLETED
+    # When a submission target is configured (a hosted run), a failed or omitted
+    # submission is a job failure — otherwise a broken upload reports as green.
+    submission_expected = bool(job.sink.run_test_id)
+    submission_ok = (not submission_expected) or submission_status == "submitted"
+    completed = run_completed and submission_ok
+    if completed:
+        detail = None
+    elif not run_completed:
+        detail = report.failure.code if report.failure else "run_failed"
+    else:
+        detail = f"submission_{submission_status or 'missing'}"
+    outcome_fields = {
+        **_job_log_fields(job),
+        "run_status": getattr(report.status, "value", str(report.status)),
+        "submission_status": submission_status,
+        "report_hash": report.report_hash,
+        "detail": detail,
+    }
+    if completed:
+        logger.info("hosted job completed", extra=outcome_fields)
+    else:
+        logger.error("hosted job failed", extra=outcome_fields)
+    reporter.emit(
+        RunnerJobPhase.COMPLETED if completed else RunnerJobPhase.FAILED,
+        detail=detail,
+        report_hash=report.report_hash,
+        submission_status=submission_status,
+    )
+    return 0 if completed else 1
+
+
+def _install_cancellation(run_task_holder: dict[str, asyncio.Task[int]]) -> None:
+    loop = asyncio.get_running_loop()
+
+    def _cancel() -> None:
+        task = run_task_holder.get("task")
+        if task is not None and not task.done():
+            task.cancel()
+
+    for sig in (signal.SIGTERM, signal.SIGINT):
+        try:
+            loop.add_signal_handler(sig, _cancel)
+        except (NotImplementedError, ValueError):
+            pass
+
+
+async def _main_async(job: StartRunnerJob, reporter: _StatusReporter) -> int:
+    holder: dict[str, asyncio.Task[int]] = {}
+    _install_cancellation(holder)
+    task = asyncio.ensure_future(_execute(job, reporter))
+    holder["task"] = task
+    try:
+        return await task
+    except asyncio.CancelledError:
+        return 2
+
+
+def _configure_logging() -> None:
+    """The child runs with no logging config, so INFO seams (job start/outcome,
+    engine dispatch/join/stop_reason) were silently dropped by the WARNING-level
+    lastResort handler and never reached the runner's log capture."""
+    root = logging.getLogger()
+    if not root.handlers:
+        handler = logging.StreamHandler(sys.stderr)
+        handler.setFormatter(logging.Formatter("%(levelname)s:%(name)s:%(message)s"))
+        root.addHandler(handler)
+        root.setLevel(logging.WARNING)
+    logging.getLogger("fi.simulate").setLevel(logging.INFO)
+
+
+def main(argv: list[str] | None = None) -> int:
+    _configure_logging()
+    parser = argparse.ArgumentParser(prog="fi.simulate.hosted.child_entrypoint")
+    parser.add_argument("job", help="path to the StartRunnerJob JSON file")
+    parser.add_argument("--status-file", default=None)
+    args = parser.parse_args(argv)
+
+    job = _load_job(Path(args.job))
+    status_file = Path(args.status_file) if args.status_file else None
+    reporter = _StatusReporter(job.job_id, status_file)
+
+    try:
+        return asyncio.run(_main_async(job, reporter))
+    except Exception as exc:  # noqa: BLE001
+        logger.exception("hosted job crashed", extra={"job_id": job.job_id})
+        reporter.emit(
+            RunnerJobPhase.FAILED, detail=f"{type(exc).__name__}: {exc}"
+        )
+        return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

fi/simulate/hosted/job.py

@@ -0,0 +1,150 @@
+"""Hosted-runner job contracts (plan §9.1).
+
+A ``StartRunnerJob`` is the serializable unit the platform hands to a
+``simulation-runner`` worker. It embeds an immutable ``SimulationSpec`` plus the
+result-sink target; it carries only ``SecretRef``s, never resolved secrets (the
+runner resolves those into the child process environment). The child process
+(``fi.simulate.hosted.child_entrypoint``) consumes exactly this model.
+"""
+
+from __future__ import annotations
+
+from datetime import datetime
+from enum import Enum
+from typing import Protocol
+
+from pydantic import BaseModel, Field, JsonValue, model_validator
+
+from fi.simulate.runtime.spec import SecretRef, SimulationSpec
+
+RUNNER_JOB_SCHEMA_VERSION = "futureagi.runner-job.v1"
+
+
+class RunnerMode(str, Enum):
+    """Execution mode the child selects an engine for. Only the SIP mode
+    leases a phone-number slot; chat and WebRTC never touch the pool."""
+
+    CHAT = "chat"
+    VOICE_WEBRTC = "voice_webrtc"
+    VOICE_SIP = "voice_sip"
+
+    @property
+    def needs_phone(self) -> bool:
+        return self is RunnerMode.VOICE_SIP
+
+    @property
+    def is_voice(self) -> bool:
+        return self in {RunnerMode.VOICE_WEBRTC, RunnerMode.VOICE_SIP}
+
+
+class ResultSinkConfig(BaseModel):
+    """Where the child submits results. ``test_execution_id`` is set for hosted
+    runs (the platform pre-creates the execution); leaving it unset preserves
+    the local create-then-submit behavior."""
+
+    api_url: str | None = None
+    run_test_id: str | None = None
+    test_execution_id: str | None = None
+    root_directory: str | None = None
+    secret_refs: dict[str, SecretRef] = Field(default_factory=dict)
+
+
+class VoiceRunConfig(BaseModel):
+    """Voice runs use ``run_voice_simulation`` (LiveKit), not the chat
+    ``SimulationRunner``. This carries the typed inputs as JSON-round-trippable
+    dicts the child hydrates into ``AgentDefinition`` / ``LiveKitSimulatorRuntime``
+    / ``Scenario`` / ``SimulatorAgentDefinition``. ``transport.kind`` on the
+    agent definition selects webrtc vs sip."""
+
+    agent_definition: dict[str, JsonValue]
+    scenario: dict[str, JsonValue]
+    livekit_runtime: dict[str, JsonValue] | None = None
+    simulator: dict[str, JsonValue] | None = None
+    params: dict[str, JsonValue] = Field(default_factory=dict)
+
+
+class StartRunnerJob(BaseModel):
+    schema_version: str = RUNNER_JOB_SCHEMA_VERSION
+    job_id: str
+    mode: RunnerMode = RunnerMode.CHAT
+    spec: SimulationSpec | None = None
+    voice: VoiceRunConfig | None = None
+    sink: ResultSinkConfig = Field(default_factory=ResultSinkConfig)
+    job_token_env: str | None = None
+    metadata: dict[str, JsonValue] = Field(default_factory=dict)
+
+    @model_validator(mode="after")
+    def _validate(self) -> "StartRunnerJob":
+        if self.schema_version != RUNNER_JOB_SCHEMA_VERSION:
+            raise ValueError(f"runner_job_version_unsupported: {self.schema_version}")
+        if self.mode is RunnerMode.CHAT and self.spec is None:
+            raise ValueError("chat runner job requires a spec")
+        if self.mode.is_voice and self.voice is None:
+            raise ValueError(f"{self.mode.value} runner job requires a voice config")
+        return self
+
+
+class RunnerJobPhase(str, Enum):
+    PENDING = "pending"
+    PREPARING = "preparing"
+    RUNNING = "running"
+    FINALIZING = "finalizing"
+    COMPLETED = "completed"
+    FAILED = "failed"
+    CANCELED = "canceled"
+
+    @property
+    def terminal(self) -> bool:
+        return self in {
+            RunnerJobPhase.COMPLETED,
+            RunnerJobPhase.FAILED,
+            RunnerJobPhase.CANCELED,
+        }
+
+
+class RunnerJobHandle(BaseModel):
+    job_id: str
+    run_id: str
+    pid: int | None = None
+    run_directory: str | None = None
+    metadata: dict[str, JsonValue] = Field(default_factory=dict)
+
+
+class RunnerJobStatus(BaseModel):
+    job_id: str
+    phase: RunnerJobPhase
+    detail: str | None = None
+    report_hash: str | None = None
+    submission_status: str | None = None
+    updated_at: datetime
+
+
+class RunnerReconcileResult(BaseModel):
+    reconciled: bool
+    orphan_ids: list[str] = Field(default_factory=list)
+    metadata: dict[str, JsonValue] = Field(default_factory=dict)
+
+
+class HostedRunnerPort(Protocol):
+    """Scheduler-neutral port Temporal invokes (plan §9.1)."""
+
+    async def start(self, request: StartRunnerJob) -> RunnerJobHandle: ...
+
+    async def status(self, handle: RunnerJobHandle) -> RunnerJobStatus: ...
+
+    async def cancel(self, handle: RunnerJobHandle) -> None: ...
+
+    async def reconcile(self, handle: RunnerJobHandle) -> RunnerReconcileResult: ...
+
+
+__all__ = [
+    "RUNNER_JOB_SCHEMA_VERSION",
+    "HostedRunnerPort",
+    "ResultSinkConfig",
+    "RunnerJobHandle",
+    "RunnerJobPhase",
+    "RunnerJobStatus",
+    "RunnerMode",
+    "RunnerReconcileResult",
+    "StartRunnerJob",
+]

fi/simulate/hosted/targets.py

@@ -0,0 +1,53 @@
+"""Resolve the agent-under-test target from a job's ``SimulationSpec.target``.
+
+The runner never runs the target agent itself — the target is the customer's
+deployed (or supplied) agent. For chat runs the target is a turn-based surface,
+resolved through the one endpoint registry: ``spec.target.adapter`` names the
+actor-source kind (``callable`` / ``python_callable`` / ``import_object`` /
+``factory`` / ``framework`` / ``system_prompt`` / ``http`` / …) and each
+registered ``EndpointProfile`` carries the resolver. Adding a target kind is one
+profile entry, no edits here (plan §4.1).
+
+This is a HOSTED execution path (the runner runs it on our infra), so target
+kinds that execute caller-supplied Python in-process (``callable`` /
+``python_callable`` / ``import_object`` / ``factory`` / ``framework``) are
+**rejected here** — deny-by-default via ``EndpointProfile.runs_caller_code``.
+Hosted runs must reach the agent as a deployed endpoint (``http`` / ``websocket``)
+or through the sandboxed runtime. The only in-process escape is a trusted
+operator-configured default target, opted in explicitly with
+``ALK_UNSAFE_INPROCESS_CODE_ACTORS`` — never set in prod for untrusted jobs.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from typing import Any
+
+from fi.simulate.agent.wrapper import AgentWrapper
+from fi.simulate.runtime.spec import SimulationSpec
+
+
+def resolve_chat_target(spec: SimulationSpec) -> Callable[..., Any] | AgentWrapper:
+    from fi.simulate.endpoints.actor_sources import (
+        ActorSourceError,
+        inprocess_code_allowed,
+    )
+    from fi.simulate.endpoints.profiles import get_profile
+
+    adapter = (spec.target.adapter or "").lower()
+    profile = get_profile(adapter)
+    if profile is None or not profile.is_turn_based_target:
+        raise ValueError(f"unsupported_chat_target_adapter: {spec.target.adapter}")
+    if profile.runs_caller_code and not inprocess_code_allowed():
+        raise ActorSourceError(
+            f"code_actor_denied_in_hosted: target {adapter!r} would run "
+            f"caller-supplied code in the runner process. Hosted runs must use a "
+            f"deployed endpoint (http/websocket) or the sandboxed runtime; "
+            f"in-process code is developer/local only."
+        )
+    return profile.resolve_target(
+        dict(spec.target.config or {}), spec.target.secret_refs, hosted=True
+    )
+
+
+__all__ = ["resolve_chat_target"]

fi/simulate/instrumentation/__init__.py

@@ -0,0 +1,5 @@
+"""Optional SDK instrumentation hooks (plan §6.6)."""
+
+from __future__ import annotations
+
+__all__: list[str] = []

fi/simulate/instrumentation/livekit/__init__.py

@@ -0,0 +1,122 @@
+"""FutureAGIObserver — LiveKit AgentSession event tap (plan §6.6 skeleton).
+
+Attach an observer to a ``livekit.agents.AgentSession`` and it will
+subscribe to a documented list of session events, translate each into
+a ``CanonicalEvent`` from ``fi.simulate.runtime``, and hand it to a
+pluggable sink (defaulting to an in-memory list so tests can assert
+against emitted events). The real OTLP wiring lands with
+``OpenTelemetryEvidenceSource``; this observer only owns the SDK-side
+event capture.
+"""
+
+from __future__ import annotations
+
+import logging
+from collections.abc import Callable
+from typing import Any
+
+from fi.simulate.runtime import CanonicalEvent, EventReliability
+
+_SESSION_EVENT_MAP: dict[str, str] = {
+    "conversation_item_added": "transcript.final",
+    "user_state_changed": "speech.started",
+    "agent_state_changed": "session.ready",
+    "function_tool_execution_started": "tool.started",
+    "function_tool_execution_completed": "tool.completed",
+    "function_tool_execution_failed": "tool.failed",
+    "session_usage_updated": "usage.updated",
+    "close": "session.ended",
+    "error": "session.error",
+}
+
+logger = logging.getLogger(__name__)
+
+EventSink = Callable[[CanonicalEvent], None]
+
+
+class FutureAGIObserver:
+    def __init__(
+        self,
+        *,
+        run_id: str,
+        test_case_id: str,
+        sink: EventSink,
+        source: str = "livekit-observer",
+    ) -> None:
+        self._run_id = run_id
+        self._test_case_id = test_case_id
+        self._sink = sink
+        self._source = source
+        self._sequence = 0
+        self._attached = False
+
+    def attach(self, session: Any) -> "FutureAGIObserver":
+        if self._attached:
+            raise RuntimeError("observer_already_attached")
+        if not hasattr(session, "on"):
+            raise TypeError("session_incompatible: object has no on(event, callback)")
+        for session_event, canonical_type in _SESSION_EVENT_MAP.items():
+            handler = self._handler_for(session_event, canonical_type)
+            try:
+                session.on(session_event, handler)
+            except (AttributeError, ValueError):
+                logger.debug(
+                    "livekit observer: session does not expose event",
+                    extra={"event": session_event},
+                )
+        self._attached = True
+        return self
+
+    def emit(
+        self,
+        event_type: str,
+        payload: dict[str, Any] | None = None,
+        *,
+        reliability: EventReliability = EventReliability.RELIABLE,
+    ) -> CanonicalEvent:
+        self._sequence += 1
+        event = CanonicalEvent.create(
+            run_id=self._run_id,
+            test_case_id=self._test_case_id,
+            event_type=event_type,
+            source=self._source,
+            sequence=self._sequence,
+            reliability=reliability,
+            payload=payload or {},
+        )
+        self._sink(event)
+        return event
+
+    def _handler_for(self, session_event: str, canonical_type: str) -> Callable[..., None]:
+        def handler(*args: Any, **kwargs: Any) -> None:
+            payload = _summarize_payload(session_event, args, kwargs)
+            self.emit(canonical_type, payload)
+
+        return handler
+
+
+def _summarize_payload(
+    session_event: str,
+    args: tuple[Any, ...],
+    kwargs: dict[str, Any],
+) -> dict[str, Any]:
+    def _describe(value: Any) -> Any:
+        if hasattr(value, "model_dump"):
+            try:
+                return value.model_dump(mode="json", exclude_none=True)
+            except Exception:  # noqa: BLE001
+                pass
+        if hasattr(value, "__dict__"):
+            return {"repr": type(value).__name__}
+        if isinstance(value, (str, int, float, bool)) or value is None:
+            return value
+        return type(value).__name__
+
+    return {
+        "session_event": session_event,
+        "args": [_describe(item) for item in args],
+        "kwargs": {key: _describe(value) for key, value in kwargs.items()},
+    }
+
+
+__all__ = ["FutureAGIObserver"]