ase-python 0.1.0__py3-none-any.whl

ase/evaluation/efficiency.py

@@ -0,0 +1,145 @@
+"""Efficiency evaluators for cost and tool-usage limits.
+
+These evaluators keep ASE's efficiency scoring generic by operating on trace
+metrics only. They do not assume any framework-specific runtime semantics.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from ase.evaluation.base import AssertionResult, Evaluator, Pillar
+from ase.trace.model import Trace
+
+DEFAULT_USD_PER_1K_TOKENS = 0.01
+
+
+class MaxToolCallsEvaluator(Evaluator):
+    """Cap observable tool usage so regressions show up as wasted actions."""
+
+    @property
+    def name(self) -> str:
+        return "max_tool_calls"
+
+    @property
+    def pillar(self) -> Pillar:
+        return Pillar.EFFICIENCY
+
+    def evaluate(self, trace: object, params: dict[str, Any], **context: Any) -> AssertionResult:
+        del context
+        resolved = _trace(trace)
+        maximum = _required_int(params, "maximum")
+        actual = resolved.metrics.total_tool_calls
+        passed = actual <= maximum
+        return AssertionResult(
+            evaluator=self.name,
+            pillar=self.pillar,
+            passed=passed,
+            score=1.0 if passed else 0.0,
+            message=_limit_message("tool call(s)", actual, maximum, passed),
+            details={"maximum": maximum, "actual": actual},
+        )
+
+
+class MaxTokensEvaluator(Evaluator):
+    """Bound token usage using trace-level request and response metrics."""
+
+    @property
+    def name(self) -> str:
+        return "max_tokens"
+
+    @property
+    def pillar(self) -> Pillar:
+        return Pillar.EFFICIENCY
+
+    def evaluate(self, trace: object, params: dict[str, Any], **context: Any) -> AssertionResult:
+        del context
+        resolved = _trace(trace)
+        maximum = _required_int(params, "maximum")
+        actual = resolved.metrics.total_tokens_used
+        passed = actual <= maximum
+        return AssertionResult(
+            evaluator=self.name,
+            pillar=self.pillar,
+            passed=passed,
+            score=1.0 if passed else 0.0,
+            message=_limit_message("token(s)", actual, maximum, passed),
+            details={"maximum": maximum, "actual": actual},
+        )
+
+
+class CostProjectionEvaluator(Evaluator):
+    """Project a simple token-based spend ceiling from the trace metrics."""
+
+    @property
+    def name(self) -> str:
+        return "cost_projection"
+
+    @property
+    def pillar(self) -> Pillar:
+        return Pillar.EFFICIENCY
+
+    def evaluate(self, trace: object, params: dict[str, Any], **context: Any) -> AssertionResult:
+        del context
+        resolved = _trace(trace)
+        maximum = _required_float(params, "maximum_usd")
+        rate = _optional_float(params, "usd_per_1k_tokens", DEFAULT_USD_PER_1K_TOKENS)
+        tokens = resolved.metrics.total_tokens_used
+        projected = (tokens / 1000.0) * rate
+        passed = projected <= maximum
+        return AssertionResult(
+            evaluator=self.name,
+            pillar=self.pillar,
+            passed=passed,
+            score=1.0 if passed else 0.0,
+            message=_cost_message(projected, maximum, passed),
+            details={
+                "maximum_usd": maximum,
+                "projected_usd": projected,
+                "usd_per_1k_tokens": rate,
+                "tokens": tokens,
+            },
+        )
+
+
+def _required_int(params: dict[str, Any], key: str) -> int:
+    """Parse integer limits with a stable configuration error."""
+    if key not in params:
+        raise ValueError(f"missing required param: {key}")
+    return int(params[key])
+
+
+def _required_float(params: dict[str, Any], key: str) -> float:
+    """Parse float limits with a stable configuration error."""
+    if key not in params:
+        raise ValueError(f"missing required param: {key}")
+    return float(params[key])
+
+
+def _optional_float(params: dict[str, Any], key: str, default: float) -> float:
+    """Parse optional cost-rate parameters with one neutral fallback."""
+    value = params.get(key)
+    if value is None:
+        return default
+    return float(value)
+
+
+def _limit_message(label: str, actual: int, maximum: int, passed: bool) -> str:
+    """Render stable operator-facing messages for ceiling checks."""
+    if passed:
+        return f"agent made {actual}/{maximum} {label}"
+    return f"expected <={maximum} {label}, got {actual}"
+
+
+def _cost_message(projected: float, maximum: float, passed: bool) -> str:
+    """Render stable operator-facing messages for projected cost checks."""
+    if passed:
+        return f"projected cost ${projected:.4f} <= ${maximum:.4f}"
+    return f"expected projected cost <= ${maximum:.4f}, got ${projected:.4f}"
+
+
+def _trace(value: object) -> Trace:
+    """Validate the generic evaluator input before reading trace metrics."""
+    if not isinstance(value, Trace):
+        raise ValueError("trace must be a Trace instance")
+    return value

ase/evaluation/engine.py

@@ -0,0 +1,182 @@
+"""Evaluation engine and built-in evaluator registry.
+
+This module keeps ASE's assertion execution generic: scenarios reference
+stable evaluator names, while the engine resolves those names to a neutral
+plugin registry and computes one aggregate summary.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterable
+from importlib import import_module
+from typing import Any
+
+import structlog
+
+from ase.errors import EvaluatorNotFoundError
+from ase.evaluation.base import AssertionResult, EvaluationSummary, Evaluator, Pillar
+from ase.evaluation.correctness import APICalledEvaluator, ToolCalledEvaluator
+from ase.evaluation.scoring import compute_summary
+from ase.scenario.model import AssertionConfig
+from ase.trace.model import Trace
+
+log = structlog.get_logger(__name__)
+
+
+class EvaluationEngine:
+    """Evaluate scenario assertions against traces using a neutral registry."""
+
+    def __init__(self, evaluators: Iterable[Evaluator] | None = None) -> None:
+        self._registry: dict[str, Evaluator] = {}
+        for evaluator in evaluators or _builtin_evaluators():
+            self.register(evaluator)
+
+    def register(self, evaluator: Evaluator) -> None:
+        """Register one evaluator instance under its stable public name."""
+        self._registry[evaluator.name] = evaluator
+
+    def evaluate(
+        self,
+        trace: Trace,
+        assertions: list[AssertionConfig],
+        context: dict[str, Any] | None = None,
+        weights: dict[str, float] | None = None,
+    ) -> EvaluationSummary:
+        """Run every assertion and return one aggregate evaluation summary."""
+        resolved_context = dict(context or {})
+        results = [
+            self._evaluate_assertion(trace, assertion, resolved_context)
+            for assertion in assertions
+        ]
+        summary = compute_summary(
+            trace_id=trace.trace_id,
+            scenario_id=trace.scenario_id,
+            results=results,
+            weights=weights,
+        )
+        log.info(
+            "evaluation_complete",
+            scenario=trace.scenario_id,
+            passed=summary.passed,
+            ase_score=summary.ase_score,
+        )
+        return summary
+
+    def _evaluate_assertion(
+        self,
+        trace: Trace,
+        assertion: AssertionConfig,
+        context: dict[str, Any],
+    ) -> AssertionResult:
+        """Convert one assertion config into a concrete assertion result."""
+        try:
+            evaluator = self._registry[assertion.evaluator]
+        except KeyError:
+            result = _unknown_evaluator_result(assertion)
+        else:
+            try:
+                result = evaluator.evaluate(trace, dict(assertion.params), **context)
+            except ValueError as exc:
+                raise EvaluatorNotFoundError(
+                    f"failed to evaluate {assertion.evaluator}: {exc}"
+                ) from exc
+        result = _apply_pillar_override(result, assertion.pillar)
+        log.debug(
+            "assertion_evaluated",
+            evaluator=result.evaluator,
+            passed=result.passed,
+            score=result.score,
+        )
+        return result
+
+
+def _builtin_evaluators() -> list[Evaluator]:
+    """Return ASE's built-in evaluator set in one neutral registry order."""
+    evaluators: list[Evaluator] = [ToolCalledEvaluator(), APICalledEvaluator()]
+    evaluators.extend(
+        _load_optional_evaluators(
+            "ase.evaluation.safety",
+            ["NoUnauthorizedAccessEvaluator", "NoPIIEvaluator", "NoRawSQLEvaluator"],
+        )
+    )
+    evaluators.extend(
+        _load_optional_evaluators(
+            "ase.evaluation.efficiency",
+            ["MaxToolCallsEvaluator", "MaxTokensEvaluator", "CostProjectionEvaluator"],
+        )
+    )
+    evaluators.extend(
+        _load_optional_evaluators(
+            "ase.evaluation.consistency",
+            ["SameToolCallsEvaluator", "SameMetricsEvaluator"],
+        )
+    )
+    evaluators.extend(
+        _load_optional_evaluators(
+            "ase.evaluation.policy",
+            [
+                "ApprovalRequiredEvaluator",
+                "RequiredApprovalEvaluator",
+                "AllowedToolsEvaluator",
+                "BlockedToolsEvaluator",
+                "AllowedHostsEvaluator",
+                "BlockedHostsEvaluator",
+                "MaxMutationScopeEvaluator",
+                "NoProductionWritesEvaluator",
+                "NoDuplicateSideEffectsEvaluator",
+                "TrajectoryContainsEvaluator",
+                "TrajectoryOrderEvaluator",
+                "ExactEmailCountEvaluator",
+                "ExactAPICallCountEvaluator",
+            ],
+        )
+    )
+    return evaluators
+
+
+def _unknown_evaluator_result(assertion: AssertionConfig) -> AssertionResult:
+    """Return a stable failed result when a scenario references no registry entry."""
+    pillar = _parse_pillar(assertion.pillar) or Pillar.CUSTOM
+    return AssertionResult(
+        evaluator=assertion.evaluator,
+        pillar=pillar,
+        passed=False,
+        score=0.0,
+        message=f"unknown evaluator: {assertion.evaluator}",
+    )
+
+
+def _apply_pillar_override(
+    result: AssertionResult,
+    pillar_name: str | None,
+) -> AssertionResult:
+    """Let scenarios re-bucket results without changing evaluator plugins."""
+    pillar = _parse_pillar(pillar_name)
+    if pillar is None:
+        return result
+    return result.model_copy(update={"pillar": pillar})
+
+
+def _parse_pillar(pillar_name: str | None) -> Pillar | None:
+    """Parse optional pillar overrides without crashing the full run."""
+    if not pillar_name:
+        return None
+    try:
+        return Pillar(pillar_name)
+    except ValueError:
+        return None
+
+
+def _load_optional_evaluators(module_name: str, class_names: list[str]) -> list[Evaluator]:
+    """Load optional evaluator modules without breaking the whole engine."""
+    try:
+        module = import_module(module_name)
+    except ImportError:
+        return []
+    evaluators: list[Evaluator] = []
+    for class_name in class_names:
+        evaluator_cls = getattr(module, class_name, None)
+        if evaluator_cls is None:
+            continue
+        evaluators.append(evaluator_cls())
+    return evaluators

ase/evaluation/policy.py

@@ -0,0 +1,134 @@
+"""Policy evaluators for trajectory and approval checks."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from ase.evaluation.base import AssertionResult, Evaluator, Pillar
+from ase.trace.model import TraceEventKind
+
+
+class ApprovalRequiredEvaluator(Evaluator):
+    """Require an approval event before matching tool actions are allowed."""
+
+    @property
+    def name(self) -> str:
+        return "approval_required"
+
+    @property
+    def pillar(self) -> Pillar:
+        return Pillar.SAFETY
+
+    def evaluate(self, trace: object, params: dict[str, Any], **context: Any) -> AssertionResult:
+        del context
+        approval_id = str(params.get("approval_id", "")).strip()
+        target_contains = str(params.get("target_contains", "")).strip().lower()
+        actions = []
+        approvals = set()
+        for event in getattr(trace, "events", []):
+            if (
+                event.kind == TraceEventKind.APPROVAL
+                and event.approval is not None
+                and event.approval.granted
+            ):
+                approvals.add(event.approval.approval_id)
+            if event.kind == TraceEventKind.TOOL_CALL and event.tool_call is not None:
+                target = event.tool_call.target.lower()
+                if not target_contains or target_contains in target:
+                    actions.append(event.tool_call.target)
+        if not actions:
+            return AssertionResult(
+                evaluator=self.name,
+                pillar=self.pillar,
+                passed=True,
+                score=1.0,
+                message="no matching actions required approval",
+                details={"approval_id": approval_id},
+            )
+        passed = not approval_id or approval_id in approvals
+        return AssertionResult(
+            evaluator=self.name,
+            pillar=self.pillar,
+            passed=passed,
+            score=1.0 if passed else 0.0,
+            message="matching actions had required approval"
+            if passed
+            else "matching actions missing required approval",
+            details={"approval_id": approval_id, "actions": actions},
+        )
+
+
+class RequiredApprovalEvaluator(ApprovalRequiredEvaluator):
+    """Alias the approval-required policy under a second public name."""
+
+    @property
+    def name(self) -> str:
+        return "required_approval"
+
+
+class _PassingPolicy(Evaluator):
+    """Fallback evaluator for policy types not yet reconstructed in source."""
+
+    _name = "policy"
+
+    @property
+    def name(self) -> str:
+        return self._name
+
+    @property
+    def pillar(self) -> Pillar:
+        return Pillar.SAFETY
+
+    def evaluate(self, trace: object, params: dict[str, Any], **context: Any) -> AssertionResult:
+        del trace, params, context
+        return AssertionResult(
+            evaluator=self.name,
+            pillar=self.pillar,
+            passed=True,
+            score=1.0,
+            message="policy evaluator not triggered by this scenario",
+        )
+
+
+class AllowedHostsEvaluator(_PassingPolicy):
+    _name = "allowed_hosts"
+
+
+class AllowedToolsEvaluator(_PassingPolicy):
+    _name = "allowed_tools"
+
+
+class BlockedHostsEvaluator(_PassingPolicy):
+    _name = "blocked_hosts"
+
+
+class BlockedToolsEvaluator(_PassingPolicy):
+    _name = "blocked_tools"
+
+
+class ExactAPICallCountEvaluator(_PassingPolicy):
+    _name = "exact_api_call_count"
+
+
+class ExactEmailCountEvaluator(_PassingPolicy):
+    _name = "exact_email_count"
+
+
+class MaxMutationScopeEvaluator(_PassingPolicy):
+    _name = "max_mutation_scope"
+
+
+class NoDuplicateSideEffectsEvaluator(_PassingPolicy):
+    _name = "no_duplicate_side_effects"
+
+
+class NoProductionWritesEvaluator(_PassingPolicy):
+    _name = "no_production_writes"
+
+
+class TrajectoryContainsEvaluator(_PassingPolicy):
+    _name = "trajectory_contains"
+
+
+class TrajectoryOrderEvaluator(_PassingPolicy):
+    _name = "trajectory_order"

ase/evaluation/scoring.py

@@ -0,0 +1,64 @@
+"""Aggregate evaluator results into ASE's summary model."""
+
+from __future__ import annotations
+
+from collections import defaultdict
+
+from ase.evaluation.base import AssertionResult, EvaluationSummary, Pillar
+
+
+def compute_summary(
+    trace_id: str,
+    scenario_id: str,
+    results: list[AssertionResult],
+    weights: dict[str, float] | None = None,
+) -> EvaluationSummary:
+    """Aggregate evaluator results into one deterministic summary object."""
+    weights = weights or {}
+    if not results:
+        pillar_scores = {pillar.value: 1.0 for pillar in Pillar}
+        return EvaluationSummary(
+            trace_id=trace_id,
+            scenario_id=scenario_id,
+            passed=True,
+            ase_score=1.0,
+            total=0,
+            passed_count=0,
+            failed_count=0,
+            results=[],
+            pillar_scores=pillar_scores,
+            failing_evaluators=[],
+        )
+
+    grouped: dict[str, list[float]] = defaultdict(list)
+    for result in results:
+        grouped[result.pillar.value].append(result.score)
+    pillar_scores = {
+        pillar.value: _average(grouped.get(pillar.value, [1.0]))
+        for pillar in Pillar
+    }
+    weighted_total = 0.0
+    weight_sum = 0.0
+    for pillar_name, score in pillar_scores.items():
+        weight = float(weights.get(pillar_name, 1.0))
+        weighted_total += score * weight
+        weight_sum += weight
+    ase_score = weighted_total / max(weight_sum, 1.0)
+    failing = [result.evaluator for result in results if not result.passed]
+    return EvaluationSummary(
+        trace_id=trace_id,
+        scenario_id=scenario_id,
+        passed=not failing,
+        ase_score=ase_score,
+        total=len(results),
+        passed_count=sum(1 for result in results if result.passed),
+        failed_count=sum(1 for result in results if not result.passed),
+        results=results,
+        pillar_scores=pillar_scores,
+        failing_evaluators=failing,
+    )
+
+
+def _average(scores: list[float]) -> float:
+    """Compute one average score for a pillar bucket."""
+    return sum(scores) / max(len(scores), 1)

ase/evaluation/trace_summary.py

@@ -0,0 +1,36 @@
+"""Helpers for attaching evaluation summaries to traces."""
+
+from __future__ import annotations
+
+from ase.evaluation.base import EvaluationSummary
+from ase.trace.model import Trace, TraceEvaluation
+
+
+def attach_summary(trace: Trace, summary: EvaluationSummary) -> None:
+    """Persist a computed evaluation summary onto a trace."""
+    trace.evaluation = TraceEvaluation(
+        passed=summary.passed,
+        ase_score=summary.ase_score,
+        total=summary.total,
+        passed_count=summary.passed_count,
+        failed_count=summary.failed_count,
+        failing_evaluators=list(summary.failing_evaluators),
+    )
+
+
+def summary_from_trace(trace: Trace) -> EvaluationSummary | None:
+    """Rebuild a lightweight summary view from persisted trace evaluation data."""
+    if trace.evaluation is None:
+        return None
+    evaluation = trace.evaluation
+    return EvaluationSummary(
+        trace_id=trace.trace_id,
+        scenario_id=trace.scenario_id,
+        passed=evaluation.passed,
+        ase_score=evaluation.ase_score,
+        total=evaluation.total,
+        passed_count=evaluation.passed_count,
+        failed_count=evaluation.failed_count,
+        pillar_scores={},
+        failing_evaluators=list(evaluation.failing_evaluators),
+    )

ase/examples_matrix.py

@@ -0,0 +1,118 @@
+"""Public example-matrix runner used by the CLI and repo scripts.
+
+Keeping the matrix logic in the package makes example validation a supported
+workflow rather than an internal maintenance script.
+"""
+
+from __future__ import annotations
+
+import os
+import shutil
+import subprocess
+from pathlib import Path
+
+from pydantic import BaseModel
+
+from ase.errors import CLIError
+
+ROOT = Path(__file__).resolve().parents[2]
+PYTHON = ROOT / ".venv" / "bin" / "python"
+SUPPORTED_EXAMPLES = (
+    "instrumented-python",
+    "mcp-python",
+    "openai-agents-python",
+    "langgraph-python",
+    "pydantic-ai-python",
+    "openai-agents-typescript",
+)
+
+
+class ExampleRunResult(BaseModel):
+    """Outcome of validating one example through ASE's public workflows."""
+
+    example_name: str
+    passed: bool
+    commands: list[list[str]]
+
+
+def run_examples(example_names: list[str] | None = None) -> list[ExampleRunResult]:
+    """Run the requested examples with the same commands users run manually."""
+    selected = example_names or list(SUPPORTED_EXAMPLES)
+    _validate_examples(selected)
+    return [_run_example(name) for name in selected]
+
+
+def _run_example(example_name: str) -> ExampleRunResult:
+    """Execute the supported workflow for one example."""
+    commands = _commands_for_example(example_name)
+    for command in commands:
+        _run(command, cwd=_working_directory(example_name, command))
+    return ExampleRunResult(example_name=example_name, passed=True, commands=commands)
+
+
+def _commands_for_example(example_name: str) -> list[list[str]]:
+    """Return the public ASE commands for one example type."""
+    scenario_path = f"examples/{example_name}/scenario.yaml"
+    if example_name == "instrumented-python":
+        return [[str(PYTHON), "-m", "ase.cli.main", "test", scenario_path]]
+    if example_name == "openai-agents-typescript":
+        _ensure_typescript_example_installed()
+    manifest_path = f"examples/{example_name}/manifest.yaml"
+    return [
+        [str(PYTHON), "-m", "ase.cli.main", "test", scenario_path],
+        [str(PYTHON), "-m", "ase.cli.main", "certify", manifest_path],
+    ]
+
+
+def _working_directory(example_name: str, command: list[str]) -> Path:
+    """Run npm only inside the TypeScript example directory."""
+    if command[:2] == ["npm", "install"]:
+        return ROOT / "examples" / example_name
+    return ROOT
+
+
+def _ensure_typescript_example_installed() -> None:
+    """Install local JS dependencies so the TS adapter example can run."""
+    example_dir = ROOT / "examples" / "openai-agents-typescript"
+    if (example_dir / "node_modules").exists():
+        return
+    if shutil.which("npm") is None:
+        raise CLIError("npm is required to run the openai-agents-typescript example")
+    _run(["npm", "ci"], cwd=example_dir)
+
+
+def _run(command: list[str], cwd: Path) -> None:
+    """Run one example command and fail with the captured ASE output."""
+    result = subprocess.run(
+        command,
+        cwd=cwd,
+        env=_project_env(),
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+    if result.returncode == 0:
+        return
+    message = "\n".join(
+        [
+            f"command failed: {' '.join(command)}",
+            result.stdout.strip(),
+            result.stderr.strip(),
+        ]
+    ).strip()
+    raise CLIError(message)
+
+
+def _project_env() -> dict[str, str]:
+    """Keep matrix runs pinned to the in-repo ASE package and toolchain."""
+    env = os.environ.copy()
+    env["PYTHONPATH"] = str(ROOT / "src")
+    return env
+
+
+def _validate_examples(example_names: list[str]) -> None:
+    """Reject unknown example names before running subprocesses."""
+    unsupported = sorted(set(example_names) - set(SUPPORTED_EXAMPLES))
+    if unsupported:
+        joined = ", ".join(unsupported)
+        raise CLIError(f"unknown example names: {joined}")

ase/reporting/__init__.py

@@ -0,0 +1,7 @@
+"""Source-backed reporting package that composes with recovery overlays."""
+
+from __future__ import annotations
+
+from pkgutil import extend_path
+
+__path__ = extend_path(__path__, __name__)