ase-python 0.1.0__py3-none-any.whl

ase/cli/history_cmd.py

@@ -0,0 +1,126 @@
+"""ase history — list and inspect stored trace runs."""
+
+from __future__ import annotations
+
+import asyncio
+import datetime
+from typing import Annotated
+
+import typer
+from rich import box
+from rich.console import Console
+from rich.table import Table
+
+from ase.storage.trace_store import TraceStore
+from ase.trace.model import Trace
+
+_console = Console()
+
+
+def run(
+    scenario: Annotated[str | None, typer.Option("--scenario", "-s")] = None,
+    status: Annotated[str | None, typer.Option("--status")] = None,
+    limit: Annotated[int, typer.Option("--limit", "-n")] = 20,
+    trace_id: Annotated[str | None, typer.Option("--trace-id")] = None,
+) -> None:
+    """List or inspect stored trace runs from the persistent trace store."""
+    asyncio.run(_run(scenario, status, limit, trace_id))
+
+
+async def _run(
+    scenario: str | None,
+    status: str | None,
+    limit: int,
+    trace_id: str | None,
+) -> None:
+    store = TraceStore()
+    await store.setup()
+    if trace_id:
+        await _show_trace(store, trace_id)
+    else:
+        await _list_traces(store, scenario, status, limit)
+    await store.close()
+
+
+async def _show_trace(store: TraceStore, trace_id: str) -> None:
+    """Fetch and display one trace with runtime and evaluation metadata."""
+    trace = await store.get_trace(trace_id)
+    if trace is None:
+        _console.print(f"[red]Trace not found: {trace_id}[/red]")
+        raise typer.Exit(code=1)
+    _console.print(f"\n[bold]Trace:[/bold] {trace.trace_id}")
+    _console.print(f"Scenario: {trace.scenario_id} — {trace.scenario_name}")
+    _console.print(f"Execution:  [bold]{trace.status.value}[/bold]")
+    _console.print(f"Evaluation: [bold]{_evaluation_status(trace)}[/bold]")
+    _console.print(f"Runtime:    [bold]{_runtime_label(trace)}[/bold]")
+    if trace.certification_level is not None:
+        _console.print(f"Certified:  {trace.certification_level.value}")
+    if trace.evaluation is not None:
+        _console.print(f"ASE Score:  {trace.evaluation.ase_score:.2f}")
+    _console.print(f"Duration:   {trace.metrics.total_duration_ms:.0f}ms")
+    _console.print(f"Tool calls: {trace.metrics.total_tool_calls}")
+    if trace.error_message:
+        _console.print(f"\n[red]Error:[/red] {trace.error_message}")
+    if trace.stderr_output:
+        _console.print(f"\n[yellow]Stderr:[/yellow]\n{trace.stderr_output}")
+
+
+async def _list_traces(
+    store: TraceStore,
+    scenario: str | None,
+    status: str | None,
+    limit: int,
+) -> None:
+    """Display a summary table of recent runs."""
+    rows = await store.list_traces(scenario_id=scenario, status=status, limit=limit)
+    if not rows:
+        _console.print("[yellow]No traces found.[/yellow]")
+        return
+    table = Table(title="Trace History", box=box.SIMPLE_HEAD, expand=False)
+    table.add_column("Trace ID", style="dim", no_wrap=True)
+    table.add_column("Scenario", style="bold")
+    table.add_column("Runtime")
+    table.add_column("Evaluation")
+    table.add_column("Execution")
+    table.add_column("Certified")
+    table.add_column("Score", justify="right")
+    table.add_column("Started At")
+    for row in rows:
+        score = row["ase_score"]
+        table.add_row(
+            row["trace_id"][:26],
+            row["scenario_id"],
+            row.get("runtime_mode") or "unknown",
+            row["evaluation_status"] or "unknown",
+            row["status"],
+            row.get("certification_level") or "—",
+            f"{score:.2f}" if score is not None else "—",
+            _ms_to_str(row["started_at_ms"]),
+        )
+    _console.print()
+    _console.print(table)
+    _console.print(f"\n[dim]Showing {len(rows)} of last {limit} runs.[/dim]\n")
+
+
+def _ms_to_str(ms: float | None) -> str:
+    """Render millisecond timestamps as local human-readable strings."""
+    if ms is None:
+        return "—"
+    return datetime.datetime.fromtimestamp(ms / 1000).strftime("%Y-%m-%d %H:%M:%S")
+
+
+def _evaluation_status(trace: Trace) -> str:
+    """Return a human-readable evaluation outcome for a trace."""
+    if trace.evaluation is None:
+        return "unknown"
+    return "passed" if trace.evaluation.passed else "failed"
+
+
+def _runtime_label(trace: Trace) -> str:
+    """Return a compact runtime label for trace detail views."""
+    if trace.runtime_provenance is None:
+        return "unknown"
+    framework = trace.runtime_provenance.framework
+    if framework:
+        return f"{trace.runtime_provenance.mode} ({framework})"
+    return trace.runtime_provenance.mode

ase/cli/import_cmd.py

@@ -0,0 +1,34 @@
+"""ase import — convert external trace formats into native ASE traces."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Annotated
+
+import typer
+from rich.console import Console
+
+from ase.cli._trace_outputs import write_trace_artifacts
+from ase.config.model import OutputFormat
+from ase.trace.otel_import import read_otel_trace
+
+app = typer.Typer(help="Import external trace formats into native ASE traces.")
+_console = Console()
+
+
+@app.command("otel")
+def otel(
+    trace_file: Annotated[Path, typer.Argument(help="OTEL-like JSON trace file.")],
+    trace_out: Annotated[
+        Path | None,
+        typer.Option("--trace-out", help="Write native trace JSON here."),
+    ] = None,
+    output: Annotated[OutputFormat | None, typer.Option("--output", "-o")] = None,
+    out_file: Annotated[Path | None, typer.Option("--out-file", "-f")] = None,
+) -> None:
+    """Import one OTEL-like JSON trace into native ASE format."""
+    trace = read_otel_trace(trace_file)
+    write_trace_artifacts(trace, trace_out=trace_out, output=output, out_file=out_file)
+    _console.print(f"trace_id: {trace.trace_id}")
+    if trace_out is not None:
+        _console.print(f"trace_out: {trace_out}")

ase/cli/main.py

@@ -0,0 +1,134 @@
+"""ASE CLI entry point.
+
+Defines the top-level Typer app and registers all sub-commands.
+Run via:
+  ase watch | test | compare | report | init | history | doctor
+  ase spec | baseline | adapter | replay | import | certify
+"""
+
+from __future__ import annotations
+
+from importlib import import_module
+from pathlib import Path
+from typing import Annotated
+
+import typer
+
+from ase.config.env_loader import load_local_dotenv
+from ase.config.model import OutputFormat
+
+# Load .env from the project root (or any parent directory) before any command
+# runs. Keeping this in-repo avoids CLI boot failures caused by broken optional
+# dotenv/plugin installs in the user's environment.
+load_local_dotenv()
+
+app = typer.Typer(
+    name="ase",
+    help="Agent Simulation Engine — pytest for AI agent tool calls.",
+    no_args_is_help=True,
+    pretty_exceptions_enable=False,
+)
+
+
+def _run_test(
+    scenario: Annotated[
+        list[Path] | None,
+        typer.Argument(help="Scenario YAML file(s) or directories to run."),
+    ] = None,
+    config: Annotated[
+        Path | None,
+        typer.Option("--config", "-c", help="Path to ase.yaml config file."),
+    ] = None,
+    output: Annotated[
+        OutputFormat | None,
+        typer.Option("--output", "-o", help="Output format."),
+    ] = None,
+    out_file: Annotated[
+        Path | None,
+        typer.Option("--out-file", "-f", help="Write report to this file."),
+    ] = None,
+    fail_fast: Annotated[
+        bool,
+        typer.Option("--fail-fast", help="Stop after first failed scenario."),
+    ] = False,
+    workers: Annotated[
+        int,
+        typer.Option("--workers", "-w", help="Max concurrent scenarios."),
+    ] = 4,
+    debug: Annotated[
+        bool,
+        typer.Option("--debug", help="Stream agent stdout/stderr live to terminal."),
+    ] = False,
+) -> None:
+    """Lazy-load ase test so non-proxy commands do not import mitmproxy."""
+    from ase.cli.test_cmd import run
+    run(scenario, config, output, out_file, fail_fast, workers, debug)
+
+
+def _run_watch(
+    ctx: typer.Context,
+    command: Annotated[
+        list[str] | None,
+        typer.Argument(help="Agent command to run, e.g. python agent.py"),
+    ] = None,
+    port: Annotated[
+        int,
+        typer.Option("--port", "-p", help="Proxy port to listen on."),
+    ] = 0,
+    timeout: Annotated[
+        int,
+        typer.Option("--timeout", "-t", help="Max seconds to wait for agent."),
+    ] = 120,
+) -> None:
+    """Lazy-load ase watch so the proxy stack is optional at import time."""
+    from ase.cli.watch import run
+    run(ctx, command, port, timeout)
+
+
+def _run_doctor() -> None:
+    """Lazy-load ase doctor so core CLI boot is resilient to optional modules."""
+    from ase.cli.doctor_cmd import run
+    run()
+
+
+def _register_command(name: str, module_path: str, attr: str = "run") -> None:
+    """Register one command only when its module is importable in this env."""
+    try:
+        module = import_module(module_path)
+    except ImportError:
+        return
+    app.command(name)(getattr(module, attr))
+
+
+def _register_typer(name: str, module_path: str, attr: str = "app") -> None:
+    """Register one Typer sub-app only when its module is importable."""
+    try:
+        module = import_module(module_path)
+    except ImportError:
+        return
+    app.add_typer(getattr(module, attr), name=name)
+
+
+app.command("watch")(_run_watch)
+app.command("test")(_run_test)
+app.command("doctor")(_run_doctor)
+_register_command("compare", "ase.cli.compare")
+_register_command("report", "ase.cli.report")
+_register_command("certify", "ase.cli.certify_cmd")
+_register_command("replay", "ase.cli.replay_cmd")
+_register_command("init", "ase.cli.init_cmd")
+_register_command("history", "ase.cli.history_cmd")
+_register_typer("spec", "ase.cli.spec_cmd")
+_register_typer("baseline", "ase.cli.baseline_cmd")
+_register_typer("adapter", "ase.cli.adapter_cmd")
+_register_typer("examples", "ase.cli.examples_cmd")
+_register_typer("import", "ase.cli.import_cmd")
+
+
+def main() -> None:
+    """Entry point called by the `ase` console script."""
+    app()
+
+
+if __name__ == "__main__":
+    main()

ase/cli/replay_cmd.py

@@ -0,0 +1,48 @@
+"""ase replay — convert adapter events into native ASE traces."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Annotated
+
+import typer
+from rich.console import Console
+
+from ase.adapters.protocol import read_and_verify
+from ase.adapters.replay import trace_from_adapter_events
+from ase.cli._trace_outputs import write_trace_artifacts
+from ase.config.model import OutputFormat
+from ase.errors import AdapterProtocolError
+
+_console = Console()
+
+
+def run(
+    event_file: Annotated[Path, typer.Argument(help="JSONL adapter event file.")],
+    scenario_id: Annotated[
+        str,
+        typer.Option("--scenario-id", help="Scenario id for the replayed trace."),
+    ],
+    scenario_name: Annotated[
+        str,
+        typer.Option(
+            "--scenario-name",
+            help="Scenario name for the replayed trace.",
+        ),
+    ],
+    trace_out: Annotated[
+        Path | None,
+        typer.Option("--trace-out", help="Write native trace JSON here."),
+    ] = None,
+    output: Annotated[OutputFormat | None, typer.Option("--output", "-o")] = None,
+    out_file: Annotated[Path | None, typer.Option("--out-file", "-f")] = None,
+) -> None:
+    """Replay one validated adapter event stream into a native ASE trace."""
+    events, result = read_and_verify(event_file)
+    if not result.passed:
+        raise AdapterProtocolError("adapter event stream failed verification")
+    trace = trace_from_adapter_events(events, scenario_id, scenario_name)
+    write_trace_artifacts(trace, trace_out=trace_out, output=output, out_file=out_file)
+    _console.print(f"trace_id: {trace.trace_id}")
+    if trace_out is not None:
+        _console.print(f"trace_out: {trace_out}")

ase/cli/report.py

@@ -0,0 +1,115 @@
+"""ase report — render one saved trace for operators and CI."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Annotated
+
+import typer
+from rich.console import Console
+
+from ase.config.model import OutputFormat
+from ase.errors import CLIError, TraceSerializationError
+from ase.trace.model import Trace
+
+_console = Console()
+
+
+def run(
+    trace_file: Annotated[Path, typer.Argument(help="Native ASE trace JSON file.")],
+    output: Annotated[
+        OutputFormat,
+        typer.Option("--output", "-o", help="Output format."),
+    ] = OutputFormat.TERMINAL,
+    out_file: Annotated[
+        Path | None,
+        typer.Option("--out-file", "-f", help="Write the rendered report to this file."),
+    ] = None,
+) -> None:
+    """Render a trace in a compact operator-facing or machine-readable format."""
+    trace = _load_trace(trace_file)
+    rendered = _render_trace(trace, output)
+    if out_file is not None:
+        suffix = "\n" if not rendered.endswith("\n") else ""
+        out_file.write_text(rendered + suffix, encoding="utf-8")
+        return
+    _console.print(rendered)
+
+
+def _load_trace(path: Path) -> Trace:
+    """Load one native ASE trace with contextual parse errors."""
+    try:
+        return Trace.model_validate_json(path.read_text(encoding="utf-8"))
+    except OSError as exc:
+        raise TraceSerializationError(f"failed to read trace file {path}: {exc}") from exc
+    except Exception as exc:  # noqa: BLE001
+        raise TraceSerializationError(f"failed to parse trace file {path}: {exc}") from exc
+
+
+def _render_trace(trace: Trace, output: OutputFormat) -> str:
+    """Choose the stable renderer for the requested output format."""
+    if output == OutputFormat.JSON:
+        return json.dumps(trace.model_dump(mode="json"), indent=2)
+    if output == OutputFormat.MARKDOWN:
+        return _to_markdown(trace)
+    if output == OutputFormat.OTEL_JSON:
+        return _to_otel_json(trace)
+    if output == OutputFormat.TERMINAL:
+        return _to_terminal_text(trace)
+    raise CLIError(f"unsupported report output format: {output}")
+
+
+def _to_terminal_text(trace: Trace) -> str:
+    """Summarize the key execution, runtime, and evaluation facts for operators."""
+    evaluation = trace.evaluation
+    runtime = trace.runtime_provenance
+    lines = [
+        f"trace_id: {trace.trace_id}",
+        f"scenario: {trace.scenario_id}",
+        f"status: {trace.status}",
+        f"runtime_mode: {runtime.mode if runtime else 'unknown'}",
+        f"framework: {runtime.framework if runtime and runtime.framework else 'unknown'}",
+        f"tool_calls: {trace.metrics.total_tool_calls}",
+        f"llm_calls: {trace.metrics.total_llm_calls}",
+    ]
+    if evaluation is not None:
+        assertions = (
+            f"{evaluation.passed_count} passed / "
+            f"{evaluation.failed_count} failed / "
+            f"{evaluation.total} total"
+        )
+        lines.extend(
+            [
+                f"evaluation: {'passed' if evaluation.passed else 'failed'}",
+                f"ase_score: {evaluation.ase_score:.2f}",
+                f"assertions: {assertions}",
+            ]
+        )
+    return "\n".join(lines)
+
+
+def _to_markdown(trace: Trace) -> str:
+    """Render a short Markdown report suitable for CI summaries and PR comments."""
+    evaluation = trace.evaluation
+    runtime = trace.runtime_provenance
+    lines = [
+        "# ASE Trace Report",
+        "",
+        f"- Trace ID: `{trace.trace_id}`",
+        f"- Scenario: `{trace.scenario_id}`",
+        f"- Status: `{trace.status}`",
+        f"- Runtime: `{runtime.mode if runtime else 'unknown'}`",
+        f"- Framework: `{runtime.framework if runtime and runtime.framework else 'unknown'}`",
+        f"- Tool calls: `{trace.metrics.total_tool_calls}`",
+    ]
+    if evaluation is not None:
+        lines.append(f"- ASE score: `{evaluation.ase_score:.2f}`")
+    return "\n".join(lines)
+
+
+def _to_otel_json(trace: Trace) -> str:
+    """Delegate OTEL-like export to the trace interoperability layer."""
+    from ase.trace.otel_export import to_otel_dict
+
+    return json.dumps(to_otel_dict(trace), indent=2)

ase/cli/spec_cmd.py

@@ -0,0 +1,53 @@
+"""ase spec — validate scenarios and expose their JSON schema."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Annotated
+
+import typer
+from rich.console import Console
+
+from ase.scenario.parser import parse_file, schema_path
+
+app = typer.Typer(help="Validate ASE scenarios and inspect the public schema.")
+_console = Console()
+
+
+@app.command("validate")
+def validate(
+    scenario: Annotated[Path, typer.Argument(help="Scenario YAML file to validate.")],
+) -> None:
+    """Validate one scenario file against ASE's current scenario model."""
+    config = parse_file(scenario)
+    _console.print(f"valid: {config.scenario_id}")
+
+
+@app.command("print-schema")
+def print_schema(
+    kind: Annotated[
+        str,
+        typer.Option("--kind", help="scenario | conformance-manifest | conformance-result"),
+    ] = "scenario",
+    path: Annotated[bool, typer.Option("--path", help="Print the schema file path only.")] = False,
+) -> None:
+    """Print the public scenario schema or its repo path."""
+    target = _schema_path(kind)
+    if path:
+        _console.print(str(target))
+        return
+    payload = json.loads(target.read_text(encoding="utf-8"))
+    _console.print(json.dumps(payload, indent=2))
+
+
+def _schema_path(kind: str) -> Path:
+    """Resolve public schema aliases onto repo paths."""
+    if kind == "scenario":
+        return schema_path()
+    root = schema_path().parent
+    if kind == "conformance-manifest":
+        return root / "ase_conformance_manifest.schema.json"
+    if kind == "conformance-result":
+        return root / "ase_conformance_result.schema.json"
+    raise typer.BadParameter(f"unknown schema kind: {kind}")

ase/cli/test_cmd.py

@@ -0,0 +1,121 @@
+"""ase test — run adapter-backed scenarios through ASE's evaluation flow."""
+
+from __future__ import annotations
+
+import subprocess
+from pathlib import Path
+from typing import Annotated
+
+import typer
+from rich.console import Console
+
+from ase.adapters.protocol import read_and_verify
+from ase.adapters.replay import trace_from_adapter_events
+from ase.config.model import OutputFormat
+from ase.errors import CLIError
+from ase.evaluation.engine import EvaluationEngine
+from ase.evaluation.trace_summary import attach_summary
+from ase.scenario.model import AgentRuntimeMode, ScenarioConfig
+from ase.scenario.parser import parse_file
+from ase.storage.trace_store import TraceStore
+from ase.trace.model import TraceStatus
+
+_console = Console()
+
+
+def run(
+    scenario: Annotated[list[Path] | None, typer.Argument()] = None,
+    config: Annotated[Path | None, typer.Option("--config", "-c")] = None,
+    output: Annotated[OutputFormat | None, typer.Option("--output", "-o")] = None,
+    out_file: Annotated[Path | None, typer.Option("--out-file", "-f")] = None,
+    fail_fast: Annotated[bool, typer.Option("--fail-fast")] = False,
+    workers: Annotated[int, typer.Option("--workers", "-w")] = 4,
+    debug: Annotated[bool, typer.Option("--debug")] = False,
+) -> None:
+    """Run adapter-backed scenarios through replay, evaluation, and persistence."""
+    del config, output, out_file, workers, debug
+    scenario_paths = scenario or []
+    if not scenario_paths:
+        raise typer.Exit(code=1)
+    store = TraceStore()
+    _run_all(scenario_paths, store, fail_fast=fail_fast)
+
+
+def _run_all(paths: list[Path], store: TraceStore, *, fail_fast: bool) -> None:
+    """Execute all requested scenarios and stop early only when requested."""
+    import asyncio
+
+    asyncio.run(store.setup())
+    failures = 0
+    for path in paths:
+        try:
+            _run_one(path, store)
+        except CLIError as exc:
+            failures += 1
+            _console.print(f"[red]{exc}[/red]")
+            if fail_fast:
+                break
+    asyncio.run(store.close())
+    if failures:
+        raise typer.Exit(code=1)
+
+
+def _run_one(path: Path, store: TraceStore) -> None:
+    """Execute one adapter scenario end to end and persist its trace."""
+    scenario = parse_file(path)
+    if scenario.runtime_mode != AgentRuntimeMode.ADAPTER:
+        raise CLIError(f"recovered ase test currently supports adapter mode only: {path}")
+    event_path = _event_path(path, scenario)
+    result = _run_agent(scenario, event_path)
+    events, verification = read_and_verify(event_path)
+    if not verification.passed:
+        details = ", ".join(verification.errors)
+        raise CLIError(f"adapter event stream failed verification: {details}")
+    trace = trace_from_adapter_events(events, scenario.scenario_id, scenario.name)
+    trace.stderr_output = result.stderr.strip() or None
+    if result.returncode != 0:
+        trace.status = TraceStatus.FAILED
+        trace.error_message = trace.stderr_output or f"agent exited with code {result.returncode}"
+    evaluators = list(scenario.assertions) + list(scenario.policies)
+    summary = EvaluationEngine().evaluate(trace, evaluators, {})
+    attach_summary(trace, summary)
+    import asyncio
+
+    asyncio.run(store.save_trace(trace, ase_score=summary.ase_score))
+    _render_summary(trace, summary)
+    if not summary.passed or result.returncode != 0:
+        raise CLIError(f"scenario failed: {scenario.scenario_id}")
+
+
+def _run_agent(scenario: ScenarioConfig, event_path: Path) -> subprocess.CompletedProcess[str]:
+    """Run the scenario agent with the event sink path exported to the process."""
+    event_path.unlink(missing_ok=True)
+    env = dict(scenario.agent.env)
+    env.update({"ASE_ADAPTER_EVENT_SOURCE": str(event_path)})
+    return subprocess.run(
+        scenario.agent.command,
+        cwd=Path.cwd(),
+        env={**dict(__import__("os").environ), **env},
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+
+
+def _event_path(path: Path, scenario: ScenarioConfig) -> Path:
+    """Resolve an adapter event file relative to the scenario file location."""
+    runtime = scenario.agent_runtime
+    if runtime is None or not runtime.event_source:
+        raise CLIError(f"adapter runtime missing event_source: {path}")
+    source = Path(runtime.event_source)
+    return source if source.is_absolute() else path.resolve().parent / source
+
+
+def _render_summary(trace: object, summary: object) -> None:
+    """Print a compact operator-facing outcome for the recovered test path."""
+    trace_id = getattr(trace, "trace_id", "unknown")
+    scenario_id = getattr(trace, "scenario_id", "unknown")
+    passed = getattr(summary, "passed", False)
+    score = getattr(summary, "ase_score", 0.0)
+    status = "[green]PASS[/green]" if passed else "[red]FAIL[/red]"
+    _console.print(f"{status} {scenario_id} trace={trace_id} ase_score={score:.2f}")