ase-python 0.1.0__py3-none-any.whl

ase/reporting/json_report.py

@@ -0,0 +1,45 @@
+"""JSON report helpers for ASE evaluation summaries and traces."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+from ase.errors import TraceSerializationError
+from ase.evaluation.base import EvaluationSummary
+from ase.trace.model import Trace
+
+
+def summary_dict(summary: EvaluationSummary) -> dict[str, Any]:
+    """Convert one evaluation summary into a stable JSON payload."""
+    return summary.model_dump(mode="json")
+
+
+def trace_dict(trace: Trace) -> dict[str, Any]:
+    """Convert one trace into a stable JSON payload."""
+    return trace.model_dump(mode="json")
+
+
+def to_string(
+    summary: EvaluationSummary | None = None,
+    trace: Trace | None = None,
+) -> str:
+    """Render a summary or trace as pretty JSON."""
+    if trace is not None:
+        return json.dumps(trace_dict(trace), indent=2)
+    if summary is None:
+        raise TraceSerializationError("json report requires a summary or trace")
+    return json.dumps(summary_dict(summary), indent=2)
+
+
+def write_to_file(
+    path: Path,
+    summary: EvaluationSummary | None = None,
+    trace: Trace | None = None,
+) -> None:
+    """Write one JSON report artifact to disk."""
+    try:
+        path.write_text(to_string(summary=summary, trace=trace) + "\n", encoding="utf-8")
+    except OSError as exc:
+        raise TraceSerializationError(f"failed to write JSON report {path}: {exc}") from exc

ase/reporting/junit.py

@@ -0,0 +1,38 @@
+"""JUnit XML output for ASE evaluation summaries."""
+
+from __future__ import annotations
+
+import xml.etree.ElementTree as ET
+from pathlib import Path
+
+from ase.errors import TraceSerializationError
+from ase.evaluation.base import EvaluationSummary
+
+
+def to_string(summary: EvaluationSummary) -> str:
+    """Render one evaluation summary as JUnit XML."""
+    suite = ET.Element(
+        "testsuite",
+        name=summary.scenario_id,
+        tests=str(summary.total),
+        failures=str(summary.failed_count),
+    )
+    for result in summary.results:
+        case = ET.SubElement(
+            suite,
+            "testcase",
+            classname=summary.scenario_id,
+            name=result.evaluator,
+        )
+        if not result.passed:
+            failure = ET.SubElement(case, "failure", message=result.message)
+            failure.text = result.message
+    return ET.tostring(suite, encoding="unicode")
+
+
+def write_to_file(summary: EvaluationSummary, path: Path) -> None:
+    """Write JUnit XML to disk."""
+    try:
+        path.write_text(to_string(summary) + "\n", encoding="utf-8")
+    except OSError as exc:
+        raise TraceSerializationError(f"failed to write JUnit report {path}: {exc}") from exc

ase/reporting/markdown.py

@@ -0,0 +1,32 @@
+"""Markdown report helpers for ASE summaries and traces."""
+
+from __future__ import annotations
+
+from ase.evaluation.base import EvaluationSummary
+from ase.trace.model import Trace
+
+
+def to_string(summary: EvaluationSummary | None = None, trace: Trace | None = None) -> str:
+    """Render a compact Markdown report."""
+    if trace is not None:
+        return "\n".join(
+            [
+                "# ASE Trace Report",
+                "",
+                f"- Trace ID: `{trace.trace_id}`",
+                f"- Scenario: `{trace.scenario_id}`",
+                f"- Status: `{trace.status.value}`",
+                f"- Tool calls: `{trace.metrics.total_tool_calls}`",
+            ]
+        )
+    assert summary is not None
+    return "\n".join(
+        [
+            "# ASE Evaluation Summary",
+            "",
+            f"- Scenario: `{summary.scenario_id}`",
+            f"- Passed: `{summary.passed}`",
+            f"- ASE score: `{summary.ase_score:.2f}`",
+            f"- Assertions: `{summary.passed_count}` passed / `{summary.failed_count}` failed",
+        ]
+    )

ase/reporting/terminal.py

@@ -0,0 +1,66 @@
+"""Rich terminal renderers for ASE summaries."""
+
+from __future__ import annotations
+
+from rich.console import Console
+from rich.panel import Panel
+from rich.table import Table
+
+from ase.evaluation.base import EvaluationSummary, Pillar
+from ase.trace.model import Trace
+
+
+def render(
+    summary: EvaluationSummary,
+    console: Console | None = None,
+    trace: Trace | None = None,
+) -> None:
+    """Render a full terminal report with optional trace context."""
+    target = console or Console()
+    target.print(_summary_panel(summary))
+    target.print(_results_table(summary))
+    if trace is not None and trace.stderr_output:
+        target.print(Panel(trace.stderr_output, title="Agent stderr"))
+
+
+def render_compact(summary: EvaluationSummary, console: Console | None = None) -> None:
+    """Render a compact one-panel summary for short workflows."""
+    target = console or Console()
+    target.print(_summary_panel(summary))
+
+
+def _summary_panel(summary: EvaluationSummary) -> Panel:
+    """Build the high-level summary panel."""
+    status = "PASS" if summary.passed else "FAIL"
+    body = "\n".join(
+        [
+            f"Scenario: {summary.scenario_id}",
+            f"Trace ID: {summary.trace_id}",
+            f"ASE Score: {summary.ase_score:.4f}",
+            "Assertions: "
+            f"{summary.passed_count} passed / "
+            f"{summary.failed_count} failed / "
+            f"{summary.total} total",
+        ]
+    )
+    return Panel(body, title=f"ASE Result — {status}")
+
+
+def _results_table(summary: EvaluationSummary) -> Table:
+    """Build the evaluator result table."""
+    table = Table(title="Assertions")
+    table.add_column("Evaluator")
+    table.add_column("Pillar")
+    table.add_column("Score")
+    table.add_column("Message")
+    for result in summary.results:
+        table.add_row(
+            result.evaluator,
+            result.pillar.value,
+            f"{result.score:.2f}",
+            result.message,
+        )
+    return table
+
+
+__all__ = ["EvaluationSummary", "Pillar", "Trace", "render", "render_compact"]

ase/scenario/__init__.py

@@ -0,0 +1,7 @@
+"""Source-backed scenario package that composes with recovery overlays."""
+
+from __future__ import annotations
+
+from pkgutil import extend_path
+
+__path__ = extend_path(__path__, __name__)

ase/scenario/model.py

@@ -0,0 +1,294 @@
+"""Scenario configuration models consumed by the ASE runtime."""
+
+from __future__ import annotations
+
+from enum import StrEnum
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+from ase.conformance.model import CertificationLevel
+
+SCENARIO_SPEC_VERSION = 1
+DEFAULT_BASELINE_METRICS = [
+    "total_tool_calls",
+    "total_llm_calls",
+    "total_tokens_used",
+]
+
+
+class EnvironmentKind(StrEnum):
+    """Which backend mode a scenario uses."""
+
+    REAL = "real"
+    MOCK = "mock"
+    SIMULATED = "simulated"
+
+
+class AgentRuntimeMode(StrEnum):
+    """Which ASE runtime path executes the scenario."""
+
+    PROXY = "proxy"
+    INSTRUMENTED = "instrumented"
+    ADAPTER = "adapter"
+
+
+class AdapterTransport(StrEnum):
+    """How adapter events are delivered into ASE."""
+
+    JSONL_STDIO = "jsonl-stdio"
+    HTTP = "http"
+
+
+class BaselineMode(StrEnum):
+    """How baseline comparison should behave."""
+
+    TOOL_CALLS = "tool_calls"
+    METRICS = "metrics"
+    COMBINED = "combined"
+
+
+class AgentConfig(BaseModel):
+    """How ASE launches the agent subprocess."""
+
+    command: list[str] = Field(description="Shell command to run the agent")
+    env: dict[str, str] = Field(default_factory=dict)
+    timeout_seconds: int = Field(default=60, ge=1)
+
+
+class AgentRuntimeConfig(BaseModel):
+    """Describe the runtime and methodology of the agent under test."""
+
+    mode: AgentRuntimeMode | None = None
+    framework: str | None = None
+    language: str | None = None
+    version: str | None = None
+    methodology: str | None = None
+    entrypoint: str | None = None
+    adapter_name: str | None = None
+    event_source: str | None = None
+    metadata: dict[str, Any] = Field(default_factory=dict)
+
+
+class AdapterConfig(BaseModel):
+    """Declare how a non-proxy runtime integrates with ASE."""
+
+    name: str
+    transport: AdapterTransport = AdapterTransport.JSONL_STDIO
+    command: list[str] = Field(default_factory=list)
+    url: str | None = None
+    timeout_seconds: int = Field(default=60, ge=1)
+    metadata: dict[str, Any] = Field(default_factory=dict)
+
+
+class ApprovalFixture(BaseModel):
+    """Declare one approval available during the run."""
+
+    approval_id: str
+    actor: str = "system"
+    granted: bool = True
+
+
+class HTTPRecordingFixture(BaseModel):
+    """Store one inline request/response replay fixture."""
+
+    request: dict[str, Any] = Field(default_factory=dict)
+    response: dict[str, Any] = Field(default_factory=dict)
+
+
+class FilesystemEntryFixture(BaseModel):
+    """Represent one deterministic filesystem fixture entry."""
+
+    path: str
+    content: str = ""
+    writable: bool = False
+
+
+class QueueMessageFixture(BaseModel):
+    """Represent one deterministic queue message."""
+
+    queue: str
+    body: dict[str, Any] = Field(default_factory=dict)
+    message_id: str | None = None
+
+
+class WebhookEventFixture(BaseModel):
+    """Represent one deterministic webhook event."""
+
+    endpoint: str
+    method: str = "POST"
+    payload: dict[str, Any] = Field(default_factory=dict)
+    event_id: str | None = None
+
+
+class FixturesConfig(BaseModel):
+    """Reusable fixtures that influence determinism and policy enforcement."""
+
+    approvals: list[ApprovalFixture] = Field(default_factory=list)
+    http_recordings: list[HTTPRecordingFixture] = Field(default_factory=list)
+    filesystem: list[FilesystemEntryFixture] = Field(default_factory=list)
+    queue_messages: list[QueueMessageFixture] = Field(default_factory=list)
+    webhook_events: list[WebhookEventFixture] = Field(default_factory=list)
+    metadata: dict[str, Any] = Field(default_factory=dict)
+
+
+class DatabaseSeed(BaseModel):
+    """SQL statements to run before a scenario starts."""
+
+    statements: list[str] = Field(default_factory=list)
+
+
+class APISeed(BaseModel):
+    """HTTP request/response pairs for record-replay."""
+
+    recordings: list[dict[str, Any]] = Field(default_factory=list)
+
+
+class BrowserSessionSeed(BaseModel):
+    """Placeholder schema for future browser/session replay."""
+
+    sessions: list[dict[str, Any]] = Field(default_factory=list)
+
+
+class EnvironmentConfig(BaseModel):
+    """Environment configuration for the scenario."""
+
+    kind: EnvironmentKind = EnvironmentKind.SIMULATED
+    database: DatabaseSeed | None = None
+    api: APISeed | None = None
+    browser_session: BrowserSessionSeed | None = None
+
+
+class SessionConfig(BaseModel):
+    """Capture session expectations for stateful agents."""
+
+    enabled: bool = False
+    session_id: str | None = None
+    stateful: bool = True
+    metadata: dict[str, Any] = Field(default_factory=dict)
+
+
+class HandoffConfig(BaseModel):
+    """Declare expected multi-agent handoff behavior."""
+
+    enabled: bool = False
+    expected_edges: list[dict[str, Any]] = Field(default_factory=list)
+    metadata: dict[str, Any] = Field(default_factory=dict)
+
+
+class HumanFeedbackConfig(BaseModel):
+    """Declare expected human feedback checkpoints."""
+
+    enabled: bool = False
+    required_actors: list[str] = Field(default_factory=list)
+    metadata: dict[str, Any] = Field(default_factory=dict)
+
+
+class StreamingConfig(BaseModel):
+    """Declare streaming-output expectations."""
+
+    enabled: bool = False
+    expected_chunk_count: int | None = Field(default=None, ge=0)
+    metadata: dict[str, Any] = Field(default_factory=dict)
+
+
+class RealtimeConfig(BaseModel):
+    """Declare realtime transport expectations."""
+
+    enabled: bool = False
+    transport: str | None = None
+    metadata: dict[str, Any] = Field(default_factory=dict)
+
+
+class MCPConfig(BaseModel):
+    """Declare MCP servers and resources involved in the scenario."""
+
+    enabled: bool = False
+    servers: list[str] = Field(default_factory=list)
+    resources: list[str] = Field(default_factory=list)
+    metadata: dict[str, Any] = Field(default_factory=dict)
+
+
+class InterAgentConfig(BaseModel):
+    """Declare non-local inter-agent protocol expectations."""
+
+    enabled: bool = False
+    protocol: str | None = None
+    topology: str | None = None
+    metadata: dict[str, Any] = Field(default_factory=dict)
+
+
+class AssertionConfig(BaseModel):
+    """One evaluator invocation attached to a scenario."""
+
+    evaluator: str
+    params: dict[str, Any] = Field(default_factory=dict)
+    pillar: str | None = None
+
+
+class PolicyConfig(BaseModel):
+    """One policy rule compiled into an assertion at evaluation time."""
+
+    evaluator: str
+    params: dict[str, Any] = Field(default_factory=dict)
+    policy_id: str | None = None
+    pillar: str | None = "safety"
+
+
+class BaselineConfig(BaseModel):
+    """Baseline comparison settings for regression detection."""
+
+    mode: BaselineMode = BaselineMode.COMBINED
+    trace_file: str
+    compare_tool_calls: bool = False
+    compare_metrics: bool = False
+    metrics: list[str] = Field(default_factory=lambda: list(DEFAULT_BASELINE_METRICS))
+    metrics_tolerance: float = Field(default=0.1, ge=0)
+
+
+class ScenarioConfig(BaseModel):
+    """Complete definition of a single ASE scenario."""
+
+    spec_version: int = SCENARIO_SPEC_VERSION
+    scenario_id: str
+    name: str
+    description: str = ""
+    agent: AgentConfig
+    agent_runtime: AgentRuntimeConfig | None = None
+    adapter: AdapterConfig | None = None
+    environment: EnvironmentConfig = Field(default_factory=EnvironmentConfig)
+    fixtures: FixturesConfig = Field(default_factory=FixturesConfig)
+    session: SessionConfig | None = None
+    handoffs: HandoffConfig | None = None
+    human_feedback: HumanFeedbackConfig | None = None
+    streaming: StreamingConfig | None = None
+    realtime: RealtimeConfig | None = None
+    mcp: MCPConfig | None = None
+    inter_agent: InterAgentConfig | None = None
+    assertions: list[AssertionConfig] = Field(default_factory=list)
+    policies: list[PolicyConfig] = Field(default_factory=list)
+    baselines: BaselineConfig | None = None
+    tags: dict[str, str] = Field(default_factory=dict)
+    run_metadata: dict[str, Any] = Field(default_factory=dict)
+
+    @property
+    def runtime_mode(self) -> AgentRuntimeMode:
+        """Return the effective execution mode for the scenario."""
+        if self.agent_runtime is None or self.agent_runtime.mode is None:
+            return AgentRuntimeMode.PROXY
+        return self.agent_runtime.mode
+
+    @property
+    def certification_level(self) -> CertificationLevel | None:
+        """Infer a coarse certification level from scenario capabilities."""
+        if self.mcp and self.mcp.enabled:
+            return CertificationLevel.MCP
+        if self.handoffs and self.handoffs.enabled:
+            return CertificationLevel.MULTI_AGENT
+        if self.session and self.session.enabled:
+            return CertificationLevel.STATEFUL
+        if self.streaming and self.streaming.enabled:
+            return CertificationLevel.REALTIME
+        if self.agent_runtime is not None:
+            return CertificationLevel.CORE
+        return None

ase/scenario/parser.py

@@ -0,0 +1,40 @@
+"""YAML parser for ASE scenarios."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import yaml
+
+from ase.errors import ConfigError
+from ase.scenario.model import ScenarioConfig
+
+_SCHEMA_PATH = Path(__file__).resolve().parents[3] / "schemas" / "ase_scenario.schema.json"
+
+
+def parse_file(path: Path) -> ScenarioConfig:
+    """Load one YAML scenario file and attach its source path metadata."""
+    if not path.exists():
+        raise ConfigError(f"failed to parse scenario: file not found {path}")
+    try:
+        raw = path.read_text(encoding="utf-8")
+    except OSError as exc:
+        raise ConfigError(f"failed to parse scenario: could not read {path}: {exc}") from exc
+    try:
+        data = yaml.safe_load(raw) or {}
+    except yaml.YAMLError as exc:
+        raise ConfigError(f"failed to parse scenario: invalid YAML in {path}: {exc}") from exc
+    if not isinstance(data, dict):
+        raise ConfigError(f"failed to parse scenario: root must be a mapping in {path}")
+    run_metadata = dict(data.get("run_metadata") or {})
+    run_metadata.setdefault("source", str(path.resolve()))
+    data["run_metadata"] = run_metadata
+    try:
+        return ScenarioConfig.model_validate(data)
+    except Exception as exc:
+        raise ConfigError(f"failed to parse scenario: invalid spec in {path}: {exc}") from exc
+
+
+def schema_path() -> Path:
+    """Return the public scenario schema location in the repo."""
+    return _SCHEMA_PATH

ase/storage/__init__.py

@@ -0,0 +1,7 @@
+"""Source-backed storage package that composes with recovery overlays."""
+
+from __future__ import annotations
+
+from pkgutil import extend_path
+
+__path__ = extend_path(__path__, __name__)

ase/storage/trace_store.py

@@ -0,0 +1,136 @@
+"""Persistent local storage for ASE traces."""
+
+from __future__ import annotations
+
+import json
+import sqlite3
+from pathlib import Path
+from typing import Any
+
+import structlog
+
+from ase.trace.model import Trace
+from ase.trace.serializer import deserialize, serialize
+
+log = structlog.get_logger(__name__)
+
+
+class TraceStore:
+    """Store and query traces in a local SQLite database."""
+
+    _DEFAULT_PATH = Path.home() / ".ase" / "traces.db"
+
+    def __init__(self, db_path: Path | None = None) -> None:
+        self._db_path = db_path or self._DEFAULT_PATH
+        self._conn: sqlite3.Connection | None = None
+
+    async def setup(self) -> None:
+        """Create the database and table if they do not already exist."""
+        self._db_path.parent.mkdir(parents=True, exist_ok=True)
+        self._conn = sqlite3.connect(self._db_path)
+        self._conn.row_factory = sqlite3.Row
+        self._conn.execute(
+            """
+            CREATE TABLE IF NOT EXISTS traces (
+                trace_id TEXT PRIMARY KEY,
+                scenario_id TEXT NOT NULL,
+                scenario_name TEXT NOT NULL,
+                status TEXT NOT NULL,
+                evaluation_status TEXT,
+                ase_score REAL,
+                runtime_mode TEXT,
+                certification_level TEXT,
+                started_at_ms REAL,
+                trace_json TEXT NOT NULL
+            )
+            """
+        )
+        self._conn.commit()
+        log.debug("trace_store_ready", path=str(self._db_path))
+
+    async def close(self) -> None:
+        """Close the SQLite connection if it is open."""
+        if self._conn is not None:
+            self._conn.close()
+            self._conn = None
+
+    async def save_trace(self, trace: Trace, ase_score: float | None = None) -> None:
+        """Persist one trace and its summary columns."""
+        conn = _require_conn(self._conn)
+        stored = sanitize_trace_for_storage(trace)
+        evaluation_status = None
+        if stored.evaluation is not None:
+            evaluation_status = "passed" if stored.evaluation.passed else "failed"
+        score_value = ase_score
+        if score_value is None and stored.evaluation is not None:
+            score_value = stored.evaluation.ase_score
+        conn.execute(
+            """
+            INSERT OR REPLACE INTO traces (
+                trace_id, scenario_id, scenario_name, status, evaluation_status,
+                ase_score, runtime_mode, certification_level, started_at_ms, trace_json
+            ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+            """,
+            (
+                stored.trace_id,
+                stored.scenario_id,
+                stored.scenario_name,
+                stored.status.value,
+                evaluation_status,
+                score_value,
+                stored.runtime_provenance.mode if stored.runtime_provenance else None,
+                stored.certification_level.value if stored.certification_level else None,
+                stored.started_at_ms,
+                serialize(stored),
+            ),
+        )
+        conn.commit()
+        log.debug("trace_saved", trace_id=stored.trace_id)
+
+    async def list_traces(
+        self,
+        scenario_id: str | None = None,
+        status: str | None = None,
+        limit: int = 50,
+    ) -> list[dict[str, Any]]:
+        """Return recent trace rows for history views."""
+        conn = _require_conn(self._conn)
+        query = "SELECT * FROM traces"
+        clauses: list[str] = []
+        params: list[Any] = []
+        if scenario_id:
+            clauses.append("scenario_id = ?")
+            params.append(scenario_id)
+        if status:
+            clauses.append("status = ?")
+            params.append(status)
+        if clauses:
+            query += " WHERE " + " AND ".join(clauses)
+        query += " ORDER BY started_at_ms DESC LIMIT ?"
+        params.append(limit)
+        rows = conn.execute(query, params).fetchall()
+        return [dict(row) for row in rows]
+
+    async def get_trace(self, trace_id: str) -> Trace | None:
+        """Return one persisted trace by id."""
+        conn = _require_conn(self._conn)
+        row = conn.execute(
+            "SELECT trace_json FROM traces WHERE trace_id = ?",
+            (trace_id,),
+        ).fetchone()
+        if row is None:
+            return None
+        return deserialize(row["trace_json"])
+
+
+def sanitize_trace_for_storage(trace: Trace) -> Trace:
+    """Copy a trace into a storage-safe form without mutating the caller."""
+    payload = json.loads(trace.model_dump_json())
+    return Trace.model_validate(payload)
+
+
+def _require_conn(conn: sqlite3.Connection | None) -> sqlite3.Connection:
+    """Ensure the trace store is initialized before use."""
+    if conn is None:
+        raise RuntimeError("trace store is not initialized")
+    return conn

ase/trace/__init__.py

@@ -0,0 +1,7 @@
+"""Source-backed trace package that composes with recovery overlays."""
+
+from __future__ import annotations
+
+from pkgutil import extend_path
+
+__path__ = extend_path(__path__, __name__)