agent-learning-kit 0.1.0__py3-none-any.whl

fi/simulate/runtime/__init__.py

@@ -0,0 +1,85 @@
+from .capabilities import CapabilitySet, EndpointCapabilities
+from .events import CanonicalEvent, EventReliability
+from .failures import FailureStage, SimulationFailure
+from .ids import (
+    derive_artifact_id,
+    derive_event_id,
+    derive_test_case_id,
+    new_plan_id,
+    new_run_id,
+)
+from .plan import (
+    SIMULATION_PLAN_VERSION,
+    AdapterRef,
+    ArtifactPlan,
+    EvidencePlan,
+    SimulationPlan,
+)
+from .report import (
+    SIMULATION_REPORT_SCHEMA_VERSION,
+    SimulationReport,
+    SimulationTestCaseResult,
+)
+from .run import CleanupStatus, RunStatus, SimulationRun, TestCaseStatus
+from .spec import (
+    SIMULATION_SPEC_SCHEMA_VERSION,
+    AdapterSpec,
+    AgentEndpointSpec,
+    ArtifactPolicy,
+    CleanupPolicy,
+    ConversationDirection,
+    EnvironmentSpec,
+    EvaluationRef,
+    EvidencePolicy,
+    ExecutionPolicy,
+    RetryPolicy,
+    RuntimeIsolation,
+    RuntimeRequirements,
+    SecretRef,
+    SimulationSpec,
+    SimulatorPolicySpec,
+    TimeoutPolicy,
+)
+
+__all__ = [
+    "SIMULATION_PLAN_VERSION",
+    "SIMULATION_REPORT_SCHEMA_VERSION",
+    "SIMULATION_SPEC_SCHEMA_VERSION",
+    "AdapterRef",
+    "AdapterSpec",
+    "AgentEndpointSpec",
+    "ArtifactPlan",
+    "ArtifactPolicy",
+    "CanonicalEvent",
+    "CapabilitySet",
+    "CleanupPolicy",
+    "CleanupStatus",
+    "ConversationDirection",
+    "EndpointCapabilities",
+    "EnvironmentSpec",
+    "EvaluationRef",
+    "EventReliability",
+    "EvidencePlan",
+    "EvidencePolicy",
+    "ExecutionPolicy",
+    "FailureStage",
+    "RetryPolicy",
+    "RunStatus",
+    "RuntimeIsolation",
+    "RuntimeRequirements",
+    "SecretRef",
+    "SimulationFailure",
+    "SimulationPlan",
+    "SimulationReport",
+    "SimulationRun",
+    "SimulationSpec",
+    "SimulationTestCaseResult",
+    "SimulatorPolicySpec",
+    "TestCaseStatus",
+    "TimeoutPolicy",
+    "derive_artifact_id",
+    "derive_event_id",
+    "derive_test_case_id",
+    "new_plan_id",
+    "new_run_id",
+]

fi/simulate/runtime/capabilities.py

@@ -0,0 +1,40 @@
+from __future__ import annotations
+
+from pydantic import BaseModel, Field, field_validator
+
+
+class EndpointCapabilities(BaseModel):
+    audio: bool = False
+    text: bool = False
+    streaming: bool = False
+    interruption: bool = False
+    dtmf: bool = False
+    transfer: bool = False
+    transcript_events: bool = False
+    tool_events: bool = False
+    usage_events: bool = False
+    internal_metrics: bool = False
+    recording: bool = False
+    web_rtc: bool = False
+    sip: bool = False
+
+    def supported(self) -> set[str]:
+        return {
+            name
+            for name, enabled in self.model_dump().items()
+            if enabled
+        }
+
+
+class CapabilitySet(BaseModel):
+    required: list[str] = Field(default_factory=list)
+    supported: list[str] = Field(default_factory=list)
+    degraded: list[str] = Field(default_factory=list)
+
+    @field_validator("required", "supported", "degraded")
+    @classmethod
+    def _normalize(cls, values: list[str]) -> list[str]:
+        return sorted(set(values))
+
+    def missing(self) -> list[str]:
+        return sorted(set(self.required) - set(self.supported))

fi/simulate/runtime/events.py

@@ -0,0 +1,63 @@
+from __future__ import annotations
+
+import time
+from datetime import datetime, timezone
+from enum import Enum
+
+from pydantic import BaseModel, Field, JsonValue
+
+from .ids import derive_event_id
+
+EVENT_SCHEMA_VERSION = "futureagi.simulation-event.v1"
+
+
+class EventReliability(str, Enum):
+    RELIABLE = "reliable"
+    BEST_EFFORT = "best_effort"
+    MEDIA = "media"
+
+
+class CanonicalEvent(BaseModel):
+    schema_version: str = EVENT_SCHEMA_VERSION
+    event_id: str
+    run_id: str
+    test_case_id: str
+    session_id: str | None = None
+    type: str
+    source: str
+    provider: str | None = None
+    wall_time: datetime
+    monotonic_ns: int
+    reliability: EventReliability = EventReliability.RELIABLE
+    payload: dict[str, JsonValue] = Field(default_factory=dict)
+    trace_id: str | None = None
+    correlation_id: str | None = None
+    provider_raw_ref: str | None = None
+
+    @classmethod
+    def create(
+        cls,
+        *,
+        run_id: str,
+        test_case_id: str,
+        event_type: str,
+        source: str,
+        sequence: int,
+        provider: str | None = None,
+        session_id: str | None = None,
+        reliability: EventReliability = EventReliability.RELIABLE,
+        payload: dict[str, JsonValue] | None = None,
+    ) -> "CanonicalEvent":
+        return cls(
+            event_id=derive_event_id(test_case_id, source, sequence),
+            run_id=run_id,
+            test_case_id=test_case_id,
+            session_id=session_id,
+            type=event_type,
+            source=source,
+            provider=provider,
+            wall_time=datetime.now(timezone.utc),
+            monotonic_ns=time.monotonic_ns(),
+            reliability=reliability,
+            payload=payload or {},
+        )

fi/simulate/runtime/failures.py

@@ -0,0 +1,25 @@
+from __future__ import annotations
+
+from enum import Enum
+
+from pydantic import BaseModel, Field, JsonValue
+
+
+class FailureStage(str, Enum):
+    PLANNING = "planning"
+    PREPARING = "preparing"
+    READINESS = "readiness"
+    RUNNING = "running"
+    FINALIZING = "finalizing"
+    SUBMITTING = "submitting"
+    CLEANUP = "cleanup"
+
+
+class SimulationFailure(BaseModel):
+    stage: FailureStage
+    code: str
+    message: str
+    retryable: bool = False
+    provider: str | None = None
+    external_ref: str | None = None
+    details: dict[str, JsonValue] = Field(default_factory=dict)

fi/simulate/runtime/ids.py

@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+from uuid import UUID, uuid4, uuid5
+
+_ID_NAMESPACE = UUID("79626429-64cd-4ec6-88e8-97b61bb649f3")
+
+
+def new_run_id() -> str:
+    return f"run_{uuid4().hex}"
+
+
+def new_plan_id() -> str:
+    return f"plan_{uuid4().hex}"
+
+
+def derive_test_case_id(run_id: str, persona_ref: str, index: int) -> str:
+    return _stable_id("case", run_id, persona_ref, str(index))
+
+
+def derive_event_id(test_case_id: str, source: str, sequence: int) -> str:
+    return _stable_id("event", test_case_id, source, str(sequence))
+
+
+def derive_artifact_id(
+    test_case_id: str,
+    logical_name: str,
+    checksum: str | None = None,
+) -> str:
+    return _stable_id("artifact", test_case_id, logical_name, checksum or "pending")
+
+
+def _stable_id(prefix: str, *parts: str) -> str:
+    value = "\x1f".join((prefix, *parts))
+    return f"{prefix}_{uuid5(_ID_NAMESPACE, value).hex}"

fi/simulate/runtime/plan.py

@@ -0,0 +1,70 @@
+from __future__ import annotations
+
+from pydantic import BaseModel, ConfigDict, Field, JsonValue, model_validator
+
+from .capabilities import CapabilitySet
+from fi.simulate._hashing import content_hash
+from .spec import CleanupPolicy, RetryPolicy, RuntimeRequirements, TimeoutPolicy
+
+SIMULATION_PLAN_VERSION = "futureagi.simulation-plan.v1"
+
+
+class AdapterRef(BaseModel):
+    name: str
+    version: str
+    config: dict[str, JsonValue] = Field(default_factory=dict)
+
+
+class EvidencePlan(BaseModel):
+    source_ids: list[str] = Field(default_factory=list)
+    required_capabilities: list[str] = Field(default_factory=list)
+
+
+class ArtifactPlan(BaseModel):
+    enabled: bool = True
+    root_directory: str
+    record_audio: bool = False
+    required_types: list[str] = Field(default_factory=list)
+
+
+class SimulationPlan(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    plan_version: str = SIMULATION_PLAN_VERSION
+    plan_id: str
+    run_id: str
+    spec_hash: str
+    environment_adapter: AdapterRef
+    target_adapter: AdapterRef
+    simulator_adapter: AdapterRef
+    transport_adapter: AdapterRef | None = None
+    negotiated_capabilities: CapabilitySet = Field(default_factory=CapabilitySet)
+    runtime_requirements: RuntimeRequirements = Field(default_factory=RuntimeRequirements)
+    timeout_policy: TimeoutPolicy = Field(default_factory=TimeoutPolicy)
+    retry_policy: RetryPolicy = Field(default_factory=RetryPolicy)
+    cleanup_policy: CleanupPolicy = Field(default_factory=CleanupPolicy)
+    evidence_plan: EvidencePlan = Field(default_factory=EvidencePlan)
+    artifact_plan: ArtifactPlan
+    plan_hash: str | None = None
+
+    def content_hash(self) -> str:
+        return content_hash(
+            self.model_dump(exclude={"plan_hash"}, exclude_none=True)
+        )
+
+    @model_validator(mode="after")
+    def _validate_and_stamp(self) -> "SimulationPlan":
+        if self.plan_version != SIMULATION_PLAN_VERSION:
+            raise ValueError(
+                f"simulation_plan_version_unsupported: {self.plan_version}"
+            )
+        missing = self.negotiated_capabilities.missing()
+        if missing:
+            raise ValueError(
+                "simulation_capabilities_missing: " + ", ".join(missing)
+            )
+        expected = self.content_hash()
+        if self.plan_hash is not None and self.plan_hash != expected:
+            raise ValueError("simulation_plan_hash_mismatch")
+        object.__setattr__(self, "plan_hash", expected)
+        return self

fi/simulate/runtime/planner.py

@@ -0,0 +1,102 @@
+from __future__ import annotations
+
+from fi.simulate.runtime.capabilities import CapabilitySet
+from fi.simulate.runtime.ids import new_plan_id
+from fi.simulate.runtime.plan import (
+    AdapterRef,
+    ArtifactPlan,
+    EvidencePlan,
+    SimulationPlan,
+)
+from fi.simulate.runtime.spec import SimulationSpec
+
+# Import for the registration side effect: builtin endpoint profiles populate
+# the endpoint_registry that build_plan reads capabilities from, and builtin
+# simulator descriptors populate the simulator_registry that build_plan validates.
+from fi.simulate.endpoints import profiles as _endpoint_profiles  # noqa: F401
+from fi.simulate.simulator import builtins as _simulator_builtins  # noqa: F401
+from fi.simulate.registry import (
+    AdapterNotFound,
+    endpoint_registry,
+    environment_registry,
+    simulator_registry,
+)
+
+
+class UnsupportedWorldKind(ValueError):
+    """Raised when a spec's ``world_kind`` isn't one the environment declares."""
+
+    def __init__(self, adapter: str, world_kind: str, supported: list[str]) -> None:
+        self.adapter = adapter
+        self.world_kind = world_kind
+        self.supported = sorted(supported)
+        super().__init__(
+            f"world_kind_unsupported: {world_kind!r} is not supported by "
+            f"environment {adapter!r}; supported: {self.supported}"
+        )
+
+
+def build_plan(spec: SimulationSpec) -> SimulationPlan:
+    # Validate the simulator adapter against the registry (typo → clear error).
+    # Only enforce when the registry is populated with named builtins, so a
+    # third-party simulator registered by an integrator still passes.
+    if simulator_registry.get_or_none(spec.simulator.adapter) is None:
+        known = simulator_registry.names()
+        if known:
+            raise AdapterNotFound("simulator", spec.simulator.adapter, known)
+    # Validate world_kind against what the environment plugin declares. Empty
+    # declaration = unrestricted (third-party plugins that don't declare stay
+    # unbroken); a declared, non-empty list is enforced (typo → clear error).
+    env_factory = environment_registry.get_or_none(spec.environment.adapter)
+    supported_world_kinds = list(
+        getattr(getattr(env_factory, "manifest", None), "world_kinds", []) or []
+    )
+    if supported_world_kinds and spec.environment.world_kind not in supported_world_kinds:
+        raise UnsupportedWorldKind(
+            spec.environment.adapter,
+            spec.environment.world_kind,
+            supported_world_kinds,
+        )
+    profile = endpoint_registry.get_or_none(spec.target.adapter)
+    supported = (
+        sorted(profile.manifest.capabilities.supported()) if profile else []
+    )
+    root_directory = spec.artifacts.root_directory or f".fagi/runs/{spec.run_id}"
+    return SimulationPlan(
+        plan_id=new_plan_id(),
+        run_id=spec.run_id,
+        spec_hash=spec.spec_hash or spec.content_hash(),
+        environment_adapter=AdapterRef(
+            name=spec.environment.adapter,
+            version=spec.environment.adapter_version,
+            config=spec.environment.config,
+        ),
+        target_adapter=AdapterRef(
+            name=spec.target.adapter,
+            version=spec.target.adapter_version,
+            config=spec.target.config,
+        ),
+        simulator_adapter=AdapterRef(
+            name=spec.simulator.adapter,
+            version=spec.simulator.adapter_version,
+            config=spec.simulator.config,
+        ),
+        negotiated_capabilities=CapabilitySet(
+            required=spec.target.required_capabilities,
+            supported=supported,
+        ),
+        runtime_requirements=spec.execution.runtime,
+        timeout_policy=spec.execution.timeout,
+        retry_policy=spec.execution.retry,
+        cleanup_policy=spec.execution.cleanup,
+        evidence_plan=EvidencePlan(
+            source_ids=[source.source_id for source in spec.evidence.sources],
+            required_capabilities=spec.evidence.required_capabilities,
+        ),
+        artifact_plan=ArtifactPlan(
+            enabled=spec.artifacts.enabled,
+            root_directory=root_directory,
+            record_audio=spec.artifacts.record_audio,
+            required_types=spec.artifacts.required_types,
+        ),
+    )

fi/simulate/runtime/report.py

@@ -0,0 +1,174 @@
+from __future__ import annotations
+
+from collections.abc import Mapping
+from datetime import datetime, timezone
+
+from pydantic import BaseModel, Field, JsonValue, model_validator
+
+from fi.simulate.artifacts import ArtifactManifest
+from fi.simulate.evidence import EvidenceSourceSummary
+from fi.simulate.simulation.models import Persona, TestCaseResult, TestReport
+
+from .failures import SimulationFailure
+from fi.simulate._hashing import content_hash
+from .ids import derive_test_case_id
+from .run import CleanupStatus, RunStatus, TestCaseStatus
+
+SIMULATION_REPORT_SCHEMA_VERSION = "futureagi.simulation-report.v1"
+
+
+class SimulationTestCaseResult(BaseModel):
+    test_case_id: str
+    status: TestCaseStatus
+    persona: Persona
+    result: TestCaseResult | None = None
+    failure: SimulationFailure | None = None
+    evidence: list[EvidenceSourceSummary] = Field(default_factory=list)
+    artifact_ids: list[str] = Field(default_factory=list)
+    started_at: datetime | None = None
+    ended_at: datetime | None = None
+    metadata: dict[str, JsonValue] = Field(default_factory=dict)
+
+    @classmethod
+    def from_legacy_case(
+        cls,
+        result: TestCaseResult,
+        *,
+        index: int,
+        run_id: str,
+        evidence: list[EvidenceSourceSummary] | None = None,
+    ) -> "SimulationTestCaseResult":
+        """Convert one legacy ``TestCaseResult`` into its canonical form.
+
+        Shared by ``SimulationReport.from_legacy`` (report-level) and the hosted
+        runner's per-case streaming callback, so a case streamed mid-run is
+        byte-identical to the one the finalized report carries.
+        """
+        persona_ref = result.persona.version or result.persona.content_hash()
+        test_case_id = str(
+            result.metadata.get("test_case_id")
+            or derive_test_case_id(run_id, persona_ref, index)
+        )
+        case_status = TestCaseStatus(
+            result.metadata.get("status", TestCaseStatus.COMPLETED.value)
+        )
+        raw_failure = result.metadata.get("failure")
+        failure = (
+            SimulationFailure.model_validate(raw_failure)
+            if isinstance(raw_failure, Mapping)
+            else None
+        )
+        return cls(
+            test_case_id=test_case_id,
+            status=case_status,
+            persona=result.persona,
+            result=result,
+            failure=failure,
+            evidence=[item.model_copy(deep=True) for item in evidence or []],
+        )
+
+    @model_validator(mode="after")
+    def _validate_outcome(self) -> "SimulationTestCaseResult":
+        if self.status == TestCaseStatus.COMPLETED and self.result is None:
+            raise ValueError("test_case_result_missing: completed case requires result")
+        failure_statuses = {
+            TestCaseStatus.FAILED,
+            TestCaseStatus.TIMED_OUT,
+            TestCaseStatus.AGENT_UNAVAILABLE,
+        }
+        if self.status in failure_statuses and self.failure is None:
+            raise ValueError("test_case_failure_missing: failed case requires failure")
+        return self
+
+
+class SimulationReport(BaseModel):
+    schema_version: str = SIMULATION_REPORT_SCHEMA_VERSION
+    run_id: str
+    plan_id: str | None = None
+    spec_hash: str
+    status: RunStatus
+    cleanup_status: CleanupStatus = CleanupStatus.PENDING
+    started_at: datetime
+    ended_at: datetime | None = None
+    test_cases: list[SimulationTestCaseResult] = Field(default_factory=list)
+    artifacts: ArtifactManifest
+    failure: SimulationFailure | None = None
+    cleanup_failure: SimulationFailure | None = None
+    metadata: dict[str, JsonValue] = Field(default_factory=dict)
+    report_hash: str | None = None
+
+    def content_hash(self) -> str:
+        payload = self.model_dump(exclude={"report_hash"}, exclude_none=True)
+        payload["test_cases"] = sorted(
+            payload["test_cases"], key=lambda item: item["test_case_id"]
+        )
+        return content_hash(payload)
+
+    def to_legacy(self, *, include_runtime_metadata: bool = True) -> TestReport:
+        results = []
+        for case in self.test_cases:
+            if case.result is not None:
+                result = case.result.model_copy(deep=True)
+            else:
+                result = TestCaseResult(persona=case.persona, transcript="")
+            if include_runtime_metadata:
+                result.metadata.update(
+                    {
+                        "run_id": self.run_id,
+                        "test_case_id": case.test_case_id,
+                        "status": case.status.value,
+                    }
+                )
+            if case.failure is not None:
+                result.metadata["failure"] = case.failure.model_dump(
+                    mode="json", exclude_none=True
+                )
+            results.append(result)
+        return TestReport(results=results)
+
+    @classmethod
+    def from_legacy(
+        cls,
+        report: TestReport,
+        *,
+        run_id: str,
+        spec_hash: str,
+        status: RunStatus = RunStatus.COMPLETED,
+        started_at: datetime | None = None,
+        ended_at: datetime | None = None,
+        plan_id: str | None = None,
+        artifacts: ArtifactManifest | None = None,
+        evidence: list[EvidenceSourceSummary] | None = None,
+    ) -> "SimulationReport":
+        cases = [
+            SimulationTestCaseResult.from_legacy_case(
+                result, index=index, run_id=run_id, evidence=evidence
+            )
+            for index, result in enumerate(report.results)
+        ]
+        return cls(
+            run_id=run_id,
+            plan_id=plan_id,
+            spec_hash=spec_hash,
+            status=status,
+            started_at=started_at or datetime.now(timezone.utc),
+            ended_at=ended_at,
+            test_cases=cases,
+            artifacts=artifacts or ArtifactManifest(run_id=run_id),
+        )
+
+    @model_validator(mode="after")
+    def _validate_and_stamp(self) -> "SimulationReport":
+        if self.schema_version != SIMULATION_REPORT_SCHEMA_VERSION:
+            raise ValueError(
+                f"simulation_report_version_unsupported: {self.schema_version}"
+            )
+        if self.artifacts.run_id != self.run_id:
+            raise ValueError("simulation_report_artifact_run_mismatch")
+        if self.status == RunStatus.FAILED and self.failure is None:
+            raise ValueError("simulation_report_failure_missing")
+        expected = self.content_hash()
+        if self.report_hash is not None and self.report_hash != expected:
+            raise ValueError("simulation_report_hash_mismatch")
+        object.__setattr__(self, "report_hash", expected)
+        return self

fi/simulate/runtime/run.py

@@ -0,0 +1,75 @@
+from __future__ import annotations
+
+from datetime import datetime
+from enum import Enum
+
+from pydantic import BaseModel, Field, JsonValue
+
+from .failures import SimulationFailure
+
+
+class RunStatus(str, Enum):
+    CREATED = "created"
+    PLANNING = "planning"
+    PREPARING = "preparing"
+    READY = "ready"
+    RUNNING = "running"
+    FINALIZING = "finalizing"
+    SUBMITTING = "submitting"
+    COMPLETED = "completed"
+    FAILED = "failed"
+    CANCELED = "canceled"
+    TIMED_OUT = "timed_out"
+
+    @property
+    def terminal(self) -> bool:
+        return self in {
+            RunStatus.COMPLETED,
+            RunStatus.FAILED,
+            RunStatus.CANCELED,
+            RunStatus.TIMED_OUT,
+        }
+
+
+class TestCaseStatus(str, Enum):
+    CREATED = "created"
+    PREPARING = "preparing"
+    READY = "ready"
+    RUNNING = "running"
+    COMPLETED = "completed"
+    FAILED = "failed"
+    CANCELED = "canceled"
+    TIMED_OUT = "timed_out"
+    AGENT_UNAVAILABLE = "agent_unavailable"
+    UNSUPPORTED = "unsupported"
+    INCONCLUSIVE = "inconclusive"
+
+    @property
+    def terminal(self) -> bool:
+        return self not in {
+            TestCaseStatus.CREATED,
+            TestCaseStatus.PREPARING,
+            TestCaseStatus.READY,
+            TestCaseStatus.RUNNING,
+        }
+
+
+class CleanupStatus(str, Enum):
+    PENDING = "pending"
+    RUNNING = "running"
+    COMPLETED = "completed"
+    FAILED = "failed"
+
+
+class SimulationRun(BaseModel):
+    run_id: str
+    plan_id: str | None = None
+    spec_hash: str
+    status: RunStatus = RunStatus.CREATED
+    cleanup_status: CleanupStatus = CleanupStatus.PENDING
+    created_at: datetime
+    started_at: datetime | None = None
+    ended_at: datetime | None = None
+    failure: SimulationFailure | None = None
+    cleanup_failure: SimulationFailure | None = None
+    metadata: dict[str, JsonValue] = Field(default_factory=dict)