ase-python 0.1.0__py3-none-any.whl

ase/core/engine.py

@@ -0,0 +1,348 @@
+"""SimulationEngine — the main orchestration loop for a scenario run.
+
+Ties together: environments, proxy, recorder, and evaluation.
+The engine is the only component that knows how all the pieces connect.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import os
+from typing import TYPE_CHECKING, Any
+
+import structlog
+from ase.core.recorder import Recorder
+from ase.core.resolver import Resolver
+from ase.core.runtime_modes import run_direct_runtime
+
+from ase.errors import ASEError
+from ase.scenario.model import (
+    AgentRuntimeMode,
+    APISeed,
+    EnvironmentKind,
+    ScenarioConfig,
+)
+from ase.trace.model import Trace, TraceStatus
+
+if TYPE_CHECKING:
+    pass
+
+log = structlog.get_logger(__name__)
+
+_POLL_INTERVAL_S = 0.1
+
+# Hosts that must bypass the ASE proxy — LLM providers and auth endpoints.
+# ASE intercepts tool calls (database, APIs, email), never model inference calls.
+_PROXY_BYPASS_HOSTS = [
+    "api.openai.com",
+    "api.anthropic.com",
+    "generativelanguage.googleapis.com",  # Gemini
+    "api.mistral.ai",
+    "api.cohere.com",
+    "bedrock-runtime.amazonaws.com",
+    "oauth2.googleapis.com",
+    "auth.openai.com",
+]
+
+
+def _build_no_proxy(existing: str) -> str:
+    """Merge existing NO_PROXY with ASE's LLM bypass list, deduplicating."""
+    current = {h.strip() for h in existing.split(",") if h.strip()}
+    merged = sorted(current | set(_PROXY_BYPASS_HOSTS))
+    return ",".join(merged)
+
+
+class RunResult:
+    """The outcome of a single scenario execution."""
+
+    def __init__(self, trace: Trace, environments: dict[str, Any]) -> None:
+        self.trace = trace
+        # Expose environments so evaluators can inspect post-run state
+        self.database: Any | None = environments.get("database")
+        self.api: Any | None = environments.get("api")
+        self.email: Any | None = environments.get("email")
+        self.filesystem: Any | None = environments.get("filesystem")
+        self.queue: Any | None = environments.get("queue")
+
+
+class SimulationEngine:
+    """Orchestrates a full scenario run.
+
+    For each scenario:
+    1. Build and set up environments from scenario config
+    2. Start the HTTP proxy (if using HTTP interception)
+    3. Launch the agent subprocess
+    4. Wait for the agent to finish (or timeout)
+    5. Tear down environments
+    6. Return a RunResult containing the Trace
+    """
+
+    def __init__(self, proxy_host: str = "127.0.0.1", proxy_port: int = 0) -> None:
+        self._proxy_host = proxy_host
+        self._proxy_port = proxy_port  # 0 = auto-allocate per run
+
+    async def run(
+        self, scenario: ScenarioConfig, debug: bool = False
+    ) -> RunResult:
+        """Execute a scenario and return the RunResult.
+
+        Never raises — errors are recorded in the trace with status=ERROR.
+        Each call creates its own proxy with a fresh port, so concurrent
+        calls to run() on the same engine instance are safe.
+        """
+        recorder = Recorder(
+            scenario_id=scenario.scenario_id,
+            scenario_name=scenario.name,
+            tags=scenario.tags,
+        )
+        self._seed_trace(recorder, scenario)
+        if scenario.runtime_mode != AgentRuntimeMode.PROXY:
+            try:
+                trace = await run_direct_runtime(scenario, debug=debug)
+            except ASEError as exc:
+                log.error("engine_run_failed", scenario=scenario.scenario_id, error=str(exc))
+                trace = recorder.finish(status=TraceStatus.ERROR, error_message=str(exc))
+            except Exception as exc:
+                log.error("engine_run_unexpected", scenario=scenario.scenario_id, error=str(exc))
+                trace = recorder.finish(
+                    status=TraceStatus.ERROR,
+                    error_message=f"unexpected error: {exc}",
+                )
+            return RunResult(trace=trace, environments={})
+
+        resolver = Resolver()
+        environments: dict[str, Any] = {}
+
+        # Proxy is created per-run so each concurrent scenario gets its own port
+        from ase.core.proxy import HTTPProxy
+
+        proxy = HTTPProxy(
+            resolver=resolver,
+            recorder=recorder,
+            host=self._proxy_host,
+            port=self._proxy_port,
+        )
+        try:
+            environments = await self._setup_environments(scenario, resolver)
+            await proxy.start()
+            trace = await self._execute_agent(
+                scenario, recorder, resolver, proxy.address, debug=debug
+            )
+        except ASEError as exc:
+            log.error("engine_run_failed", scenario=scenario.scenario_id, error=str(exc))
+            trace = recorder.finish(status=TraceStatus.ERROR, error_message=str(exc))
+        except Exception as exc:
+            log.error("engine_run_unexpected", scenario=scenario.scenario_id, error=str(exc))
+            trace = recorder.finish(
+                status=TraceStatus.ERROR,
+                error_message=f"unexpected error: {exc}",
+            )
+        finally:
+            await proxy.stop()
+            await self._teardown_environments(environments)
+
+        return RunResult(trace=trace, environments=environments)
+
+    # ------------------------------------------------------------------
+    # Internal helpers
+    # ------------------------------------------------------------------
+
+    def _seed_trace(self, recorder: Recorder, scenario: ScenarioConfig) -> None:
+        """Persist approvals and replay metadata before the agent launches."""
+        runtime = scenario.agent_runtime
+        recorder.set_runtime_provenance(
+            mode=scenario.runtime_mode.value,
+            framework=runtime.framework if runtime else None,
+            framework_version=runtime.version if runtime else None,
+            adapter_name=runtime.adapter_name if runtime else None,
+            event_source=runtime.event_source if runtime else None,
+            metadata=dict(runtime.metadata) if runtime else None,
+        )
+        for approval in scenario.fixtures.approvals:
+            recorder.record_approval(
+                approval_id=approval.approval_id,
+                actor=approval.actor,
+                granted=approval.granted,
+            )
+        recorder.set_determinism_metadata(
+            fixture_payload=json.loads(scenario.fixtures.model_dump_json()),
+            replay_key=f"{scenario.scenario_id}:spec-v{scenario.spec_version}",
+        )
+
+    async def _setup_environments(
+        self, scenario: ScenarioConfig, resolver: Resolver
+    ) -> dict[str, Any]:
+        """Instantiate, seed, and register all scenario environments."""
+        from ase.environments.api import APIEnvironment
+        from ase.environments.database import DatabaseEnvironment
+        from ase.environments.email import EmailEnvironment
+        from ase.environments.filesystem import FilesystemEnvironment
+        from ase.environments.queue import QueueEnvironment
+
+        from ase.trace.model import ToolCallKind
+
+        environments: dict[str, Any] = {}
+        env_config = scenario.environment
+
+        if env_config.kind == EnvironmentKind.REAL:
+            # No simulation — pass through to real backends
+            return environments
+
+        if env_config.database is not None:
+            db = DatabaseEnvironment(seed=env_config.database)
+            await db.setup()
+            resolver.register(ToolCallKind.DATABASE, db)
+            environments["database"] = db
+
+        api_seed = _merged_api_seed(scenario)
+        if api_seed.recordings:
+            api = APIEnvironment(seed=api_seed)
+            await api.setup()
+            resolver.register(ToolCallKind.HTTP_API, api)
+            environments["api"] = api
+
+        # Email is always available in simulated mode
+        if env_config.kind == EnvironmentKind.SIMULATED:
+            email = EmailEnvironment()
+            await email.setup()
+            resolver.register(ToolCallKind.EMAIL, email)
+            environments["email"] = email
+            if scenario.fixtures.filesystem:
+                filesystem = FilesystemEnvironment(scenario.fixtures.filesystem)
+                await filesystem.setup()
+                environments["filesystem"] = filesystem
+            if scenario.fixtures.queue_messages or scenario.fixtures.webhook_events:
+                queue = QueueEnvironment(
+                    queue_messages=scenario.fixtures.queue_messages,
+                    webhook_events=scenario.fixtures.webhook_events,
+                )
+                await queue.setup()
+                environments["queue"] = queue
+
+        return environments
+
+    async def _execute_agent(
+        self,
+        scenario: ScenarioConfig,
+        recorder: Recorder,
+        resolver: Resolver,
+        proxy_address: str,
+        debug: bool = False,
+    ) -> Trace:
+        """Launch the agent subprocess and wait for it to complete.
+
+        proxy_address: the actual address of the proxy started for this run.
+        debug: when True, inherit terminal stdio so agent output streams live.
+        """
+        agent_cfg = scenario.agent
+        proxy_env = {
+            **os.environ,  # inherit PATH, API keys, etc. from the parent process
+            "HTTP_PROXY": proxy_address,
+            "HTTPS_PROXY": proxy_address,
+            "ASE_TRACE_ID": recorder.trace_id,
+            # LLM provider hosts must bypass the proxy — ASE intercepts tool calls,
+            # not the agent's model calls. Append to any existing NO_PROXY value.
+            "NO_PROXY": _build_no_proxy(os.environ.get("NO_PROXY", "")),
+            **agent_cfg.env,  # scenario-level overrides win
+        }
+
+        log.info(
+            "agent_launching",
+            scenario=scenario.scenario_id,
+            command=agent_cfg.command,
+        )
+
+        if debug:
+            return await self._execute_agent_debug(scenario, agent_cfg, proxy_env, recorder)
+
+        try:
+            process = await asyncio.create_subprocess_exec(
+                *agent_cfg.command,
+                env=proxy_env,
+                stdout=asyncio.subprocess.PIPE,
+                stderr=asyncio.subprocess.PIPE,
+            )
+            _, stderr = await asyncio.wait_for(
+                process.communicate(),
+                timeout=agent_cfg.timeout_seconds,
+            )
+        except TimeoutError:
+            log.warning("agent_timeout", scenario=scenario.scenario_id)
+            return recorder.finish(
+                status=TraceStatus.ERROR,
+                error_message=f"agent timed out after {agent_cfg.timeout_seconds}s",
+            )
+        except Exception as exc:
+            return recorder.finish(
+                status=TraceStatus.ERROR,
+                error_message=f"failed to launch agent: {exc}",
+            )
+
+        exit_code = process.returncode or 0
+        log.info("agent_exited", scenario=scenario.scenario_id, exit_code=exit_code)
+
+        stderr_text = stderr.decode("utf-8", errors="replace").strip()
+        status = TraceStatus.PASSED if exit_code == 0 else TraceStatus.FAILED
+        # Always capture stderr; only treat it as error_message on failure
+        return recorder.finish(
+            status=status,
+            error_message=stderr_text if exit_code != 0 else None,
+            stderr_output=stderr_text or None,
+        )
+
+    async def _execute_agent_debug(
+        self,
+        scenario: ScenarioConfig,
+        agent_cfg: object,
+        proxy_env: dict[str, str],
+        recorder: Recorder,
+    ) -> Trace:
+        """Debug variant — inherits terminal stdio so output streams live."""
+        from ase.scenario.model import AgentConfig
+        assert isinstance(agent_cfg, AgentConfig)
+        try:
+            process = await asyncio.create_subprocess_exec(
+                *agent_cfg.command,
+                env=proxy_env,
+                # no PIPE — output goes directly to the terminal
+            )
+            await asyncio.wait_for(process.wait(), timeout=agent_cfg.timeout_seconds)
+        except TimeoutError:
+            log.warning("agent_timeout", scenario=scenario.scenario_id)
+            return recorder.finish(
+                status=TraceStatus.ERROR,
+                error_message=f"agent timed out after {agent_cfg.timeout_seconds}s",
+            )
+        except Exception as exc:
+            return recorder.finish(
+                status=TraceStatus.ERROR,
+                error_message=f"failed to launch agent: {exc}",
+            )
+        exit_code = process.returncode or 0
+        log.info("agent_exited", scenario=scenario.scenario_id, exit_code=exit_code)
+        status = TraceStatus.PASSED if exit_code == 0 else TraceStatus.FAILED
+        return recorder.finish(status=status)
+
+    @staticmethod
+    async def _teardown_environments(environments: dict[str, Any]) -> None:
+        """Tear down all environments, logging but not re-raising errors."""
+        from ase.environments.base import EnvironmentProvider
+
+        for name, env in environments.items():
+            if isinstance(env, EnvironmentProvider):
+                try:
+                    await env.teardown()
+                except Exception as exc:
+                    log.warning("teardown_error", environment=name, error=str(exc))
+
+
+def _merged_api_seed(scenario: ScenarioConfig) -> APISeed:
+    """Merge API recordings declared in environment and fixtures."""
+    environment_recordings = list(
+        scenario.environment.api.recordings if scenario.environment.api else []
+    )
+    fixture_recordings = [
+        fixture.model_dump() for fixture in scenario.fixtures.http_recordings
+    ]
+    return APISeed(recordings=environment_recordings + fixture_recordings)

ase/errors.py

@@ -0,0 +1,59 @@
+"""Shared ASE exception hierarchy.
+
+Keeping one root error type makes CLI and engine layers propagate contextual
+failures consistently without depending on framework-specific exceptions.
+"""
+
+from __future__ import annotations
+
+
+class ASEError(Exception):
+    """Root of ASE's user-facing and internal error hierarchy."""
+
+
+class CLIError(ASEError):
+    """Raised when a CLI workflow cannot be completed as requested."""
+
+
+class ConfigError(ASEError):
+    """Raised when ASE configuration or environment loading fails."""
+
+
+class AdapterError(ASEError):
+    """Raised when adapter SDKs cannot emit or persist framework events."""
+
+
+class AdapterProtocolError(ASEError):
+    """Raised when adapter event streams are missing or malformed."""
+
+
+class RuntimeModeError(ASEError):
+    """Raised when direct runtime execution cannot produce a valid trace."""
+
+
+class TraceSerializationError(ASEError):
+    """Raised when native ASE traces cannot be parsed or written safely."""
+
+
+class TraceError(ASEError):
+    """Raised when trace construction or mutation violates ASE invariants."""
+
+
+class TraceSchemaMigrationError(ASEError):
+    """Raised when a stored trace cannot be interpreted by this schema version."""
+
+
+class ConformanceError(ASEError):
+    """Raised when certification inputs or outputs violate ASE contracts."""
+
+
+class EvaluatorNotFoundError(ASEError):
+    """Raised when a scenario references an unknown evaluator."""
+
+
+class CacheError(ASEError):
+    """Raised when the content-addressed response cache cannot be maintained."""
+
+
+class OTelImportError(ASEError):
+    """Raised when OTEL-like trace input cannot be converted into ASE format."""

ase/evaluation/__init__.py

@@ -0,0 +1,7 @@
+"""Source-backed evaluation package that composes with recovery overlays."""
+
+from __future__ import annotations
+
+from pkgutil import extend_path
+
+__path__ = extend_path(__path__, __name__)

ase/evaluation/base.py

@@ -0,0 +1,63 @@
+"""Shared evaluator protocol and result models."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from enum import StrEnum
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+
+class Pillar(StrEnum):
+    """Define ASE's top-level scoring buckets."""
+
+    CORRECTNESS = "correctness"
+    SAFETY = "safety"
+    EFFICIENCY = "efficiency"
+    CONSISTENCY = "consistency"
+    CUSTOM = "custom"
+
+
+class AssertionResult(BaseModel):
+    """Represent one evaluator outcome within a scenario run."""
+
+    evaluator: str
+    pillar: Pillar
+    passed: bool
+    score: float
+    message: str
+    details: dict[str, Any] = Field(default_factory=dict)
+
+
+class EvaluationSummary(BaseModel):
+    """Summarize all evaluator outcomes for one trace."""
+
+    trace_id: str
+    scenario_id: str
+    passed: bool
+    ase_score: float
+    total: int
+    passed_count: int
+    failed_count: int
+    results: list[AssertionResult] = Field(default_factory=list)
+    pillar_scores: dict[str, float] = Field(default_factory=dict)
+    failing_evaluators: list[str] = Field(default_factory=list)
+
+
+class Evaluator(ABC):
+    """Define the stable extension point for ASE assertions."""
+
+    @property
+    @abstractmethod
+    def name(self) -> str:
+        """Return the public evaluator name referenced by scenarios."""
+
+    @property
+    @abstractmethod
+    def pillar(self) -> Pillar:
+        """Return the default scoring pillar for this evaluator."""
+
+    @abstractmethod
+    def evaluate(self, trace: object, params: dict[str, Any], **context: Any) -> AssertionResult:
+        """Evaluate one trace and return a structured assertion result."""

ase/evaluation/consistency.py

@@ -0,0 +1,79 @@
+"""Consistency evaluators used for baseline-style comparisons."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from ase.evaluation.base import AssertionResult, Evaluator, Pillar
+
+
+class SameToolCallsEvaluator(Evaluator):
+    """Compare tool-call counts against a provided baseline trace."""
+
+    @property
+    def name(self) -> str:
+        return "same_tool_calls"
+
+    @property
+    def pillar(self) -> Pillar:
+        return Pillar.CONSISTENCY
+
+    def evaluate(self, trace: object, params: dict[str, Any], **context: Any) -> AssertionResult:
+        baseline = context.get("baseline_trace")
+        if baseline is None:
+            return _passing(self.name, self.pillar, "no baseline provided")
+        current = getattr(getattr(trace, "metrics", None), "total_tool_calls", 0)
+        previous = getattr(getattr(baseline, "metrics", None), "total_tool_calls", 0)
+        passed = current == previous
+        return AssertionResult(
+            evaluator=self.name,
+            pillar=self.pillar,
+            passed=passed,
+            score=1.0 if passed else 0.0,
+            message=f"tool calls {current} vs baseline {previous}",
+            details={"current": current, "baseline": previous},
+        )
+
+
+class SameMetricsEvaluator(Evaluator):
+    """Compare stable metrics against a provided baseline trace."""
+
+    @property
+    def name(self) -> str:
+        return "same_metrics"
+
+    @property
+    def pillar(self) -> Pillar:
+        return Pillar.CONSISTENCY
+
+    def evaluate(self, trace: object, params: dict[str, Any], **context: Any) -> AssertionResult:
+        baseline = context.get("baseline_trace")
+        if baseline is None:
+            return _passing(self.name, self.pillar, "no baseline provided")
+        current_metrics = getattr(trace, "metrics", None)
+        baseline_metrics = getattr(baseline, "metrics", None)
+        keys = ["total_tool_calls", "total_llm_calls", "total_tokens_used"]
+        deltas = {
+            key: getattr(current_metrics, key, 0) - getattr(baseline_metrics, key, 0)
+            for key in keys
+        }
+        passed = all(delta == 0 for delta in deltas.values())
+        return AssertionResult(
+            evaluator=self.name,
+            pillar=self.pillar,
+            passed=passed,
+            score=1.0 if passed else 0.0,
+            message="metrics match baseline" if passed else "metrics differ from baseline",
+            details=deltas,
+        )
+
+
+def _passing(evaluator: str, pillar: Pillar, message: str) -> AssertionResult:
+    """Return a neutral passing result when no baseline context exists."""
+    return AssertionResult(
+        evaluator=evaluator,
+        pillar=pillar,
+        passed=True,
+        score=1.0,
+        message=message,
+    )

ase/evaluation/correctness.py

@@ -0,0 +1,117 @@
+"""Correctness evaluators for observable agent behavior.
+
+These evaluators answer the core ASE question: did the agent take the
+expected actions against its environment, regardless of the exact text it
+produced for a user.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from ase.evaluation.base import AssertionResult, Evaluator, Pillar
+from ase.trace.model import ToolCallEvent, Trace, TraceEventKind
+
+
+class ToolCalledEvaluator(Evaluator):
+    """Verify that a scenario exercised the expected tool path."""
+
+    @property
+    def name(self) -> str:
+        return "tool_called"
+
+    @property
+    def pillar(self) -> Pillar:
+        return Pillar.CORRECTNESS
+
+    def evaluate(self, trace: Trace, params: dict[str, Any], **context: Any) -> AssertionResult:
+        del context
+        kind = str(params.get("kind", "")).strip().lower()
+        minimum = max(0, int(params.get("minimum", 1)))
+        method = _optional_upper(params.get("method"))
+        target_contains = _optional_lower(params.get("target_contains"))
+        matches = _matching_calls(trace, kind, method, target_contains)
+        passed = len(matches) >= minimum
+        label = kind or "tool"
+        return AssertionResult(
+            evaluator=self.name,
+            pillar=self.pillar,
+            passed=passed,
+            score=1.0 if passed else 0.0,
+            message=_tool_called_message(label, minimum, len(matches), passed),
+            details={
+                "kind": kind or None,
+                "minimum": minimum,
+                "actual": len(matches),
+                "method": method,
+                "target_contains": target_contains,
+            },
+        )
+
+
+class APICalledEvaluator(Evaluator):
+    """Keep API-call assertions as a neutral alias over the shared tool model."""
+
+    @property
+    def name(self) -> str:
+        return "api_called"
+
+    @property
+    def pillar(self) -> Pillar:
+        return Pillar.CORRECTNESS
+
+    def evaluate(self, trace: Trace, params: dict[str, Any], **context: Any) -> AssertionResult:
+        tool_params = dict(params)
+        tool_params["kind"] = "http_api"
+        return ToolCalledEvaluator().evaluate(trace, tool_params, **context).model_copy(
+            update={"evaluator": self.name}
+        )
+
+
+def _matching_calls(
+    trace: Trace,
+    kind: str,
+    method: str | None,
+    target_contains: str | None,
+) -> list[ToolCallEvent]:
+    """Filter tool calls using only generic trace-level properties."""
+    matches: list[ToolCallEvent] = []
+    for event in trace.events:
+        if event.kind != TraceEventKind.TOOL_CALL or event.tool_call is None:
+            continue
+        if kind and event.tool_call.kind.value != kind:
+            continue
+        if method and event.tool_call.method.upper() != method:
+            continue
+        if target_contains and target_contains not in event.tool_call.target.lower():
+            continue
+        matches.append(event.tool_call)
+    return matches
+
+
+def _optional_upper(value: object) -> str | None:
+    """Normalize optional HTTP or tool verbs for equality checks."""
+    if value is None:
+        return None
+    text = str(value).strip()
+    return text.upper() if text else None
+
+
+def _optional_lower(value: object) -> str | None:
+    """Normalize optional contains-style filters for case-insensitive matching."""
+    if value is None:
+        return None
+    text = str(value).strip()
+    return text.lower() if text else None
+
+
+def _tool_called_message(
+    kind: str,
+    minimum: int,
+    actual: int,
+    passed: bool,
+) -> str:
+    """Render stable operator-facing messages for correctness assertions."""
+    if passed:
+        return f"observed {actual} '{kind}' call(s)"
+    return f"expected >={minimum} '{kind}' call(s), got {actual}"