agentops-accelerator 0.3.0__py3-none-any.whl

agentops/core/evaluators.py

@@ -0,0 +1,480 @@
+"""Evaluator catalog and auto-selection for AgentOps 1.0.
+
+This module replaces the layered ``bundle.yaml`` system. There is no
+user-facing ``scenario`` concept. Evaluators are picked from two inputs:
+
+1. The resolved target kind (agent vs model). Model targets only get the
+   baseline quality evaluators - agent-specific evaluators are skipped even
+   if the dataset contains those fields.
+2. The shape of the dataset rows:
+
+   * Always: baseline quality evaluators (Coherence, Fluency).
+   * Model-direct targets add exact/reference-answer checks (Similarity,
+     F1Score).
+   * Agent targets add answer-quality checks that tolerate free-form responses
+     (Similarity, ResponseCompleteness).
+   * If rows include ``context``: add RAG evaluators (Groundedness,
+     Retrieval, Relevance, ResponseCompleteness).
+   * If rows include ``tool_calls`` or ``tool_definitions``: add agent
+     evaluators (ToolCallAccuracy, IntentResolution, TaskAdherence).
+
+The :func:`select_evaluators` function returns a list of resolved
+:class:`EvaluatorPreset` objects. Each preset carries its class name, the
+input mapping it requires, the score key it produces, and a default
+threshold. The runner uses these presets to instantiate
+``azure-ai-evaluation`` evaluator classes against each dataset row.
+
+Power users can override the auto-selection by listing evaluator names in
+``agentops.yaml`` under ``evaluators:``. When set, the override list is the
+final word - no auto-detection runs.
+"""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Dict, FrozenSet, Iterable, List, Optional, Tuple
+
+from agentops.core.agentops_config import TargetKind, TargetResolution, Threshold
+
+
+# ---------------------------------------------------------------------------
+# Catalog
+# ---------------------------------------------------------------------------
+
+
+@dataclass(frozen=True)
+class EvaluatorPreset:
+    """Metadata for a single evaluator known to AgentOps.
+
+    ``input_mapping`` keys are the parameter names the evaluator class
+    expects; values use the placeholder syntax ``$prompt``, ``$prediction``,
+    ``$context``, ``$expected``, ``$tool_calls``, ``$tool_definitions``
+    which the runner resolves per row.
+    """
+
+    name: str
+    class_name: str
+    score_key: str
+    input_mapping: Dict[str, str]
+    default_threshold: Optional[Threshold] = None
+    #: Categories that this evaluator belongs to. Used by the inference rules.
+    categories: FrozenSet[str] = field(default_factory=frozenset)
+    #: Set when this evaluator is not safe to run for raw model deployments.
+    agent_only: bool = False
+    #: When True and the row carries ``tool_calls``, the runner upgrades the
+    #: ``query`` and ``response`` kwargs from plain strings to conversation
+    #: message lists that include the agent's tool_call + tool_result trace.
+    #: This is required for evaluators that judge agent reasoning (e.g.
+    #: TaskAdherence, IntentResolution) - without the trace they only see a
+    #: short final answer and consistently score it as 1/5.
+    needs_conversation: bool = False
+
+
+def _t(metric: str, criteria: str, value: float) -> Threshold:
+    return Threshold(metric=metric, criteria=criteria, value=value)  # type: ignore[arg-type]
+
+
+_QUALITY_BASELINE: Tuple[EvaluatorPreset, ...] = (
+    EvaluatorPreset(
+        name="CoherenceEvaluator",
+        class_name="CoherenceEvaluator",
+        score_key="coherence",
+        input_mapping={"query": "$prompt", "response": "$prediction"},
+        default_threshold=_t("coherence", ">=", 3.0),
+        categories=frozenset({"quality"}),
+    ),
+    EvaluatorPreset(
+        name="FluencyEvaluator",
+        class_name="FluencyEvaluator",
+        score_key="fluency",
+        input_mapping={"response": "$prediction"},
+        default_threshold=_t("fluency", ">=", 3.0),
+        categories=frozenset({"quality"}),
+    ),
+)
+
+_SIMILARITY = EvaluatorPreset(
+    name="SimilarityEvaluator",
+    class_name="SimilarityEvaluator",
+    score_key="similarity",
+    input_mapping={
+        "query": "$prompt",
+        "response": "$prediction",
+        "ground_truth": "$expected",
+    },
+    default_threshold=_t("similarity", ">=", 3.0),
+    categories=frozenset({"quality"}),
+)
+
+_F1_SCORE = EvaluatorPreset(
+    name="F1ScoreEvaluator",
+    class_name="F1ScoreEvaluator",
+    score_key="f1_score",
+    input_mapping={
+        "response": "$prediction",
+        "ground_truth": "$expected",
+    },
+    default_threshold=_t("f1_score", ">=", 0.5),
+    categories=frozenset({"quality", "exact_reference"}),
+)
+
+_RESPONSE_COMPLETENESS = EvaluatorPreset(
+    name="ResponseCompletenessEvaluator",
+    class_name="ResponseCompletenessEvaluator",
+    score_key="response_completeness",
+    input_mapping={
+        "query": "$prompt",
+        "response": "$prediction",
+        "ground_truth": "$expected",
+    },
+    default_threshold=_t("response_completeness", ">=", 3.0),
+    categories=frozenset({"quality", "rag"}),
+    agent_only=True,
+)
+
+_MODEL_REFERENCE_EVALUATORS: Tuple[EvaluatorPreset, ...] = (
+    _SIMILARITY,
+    _F1_SCORE,
+)
+
+_AGENT_ANSWER_EVALUATORS: Tuple[EvaluatorPreset, ...] = (
+    _SIMILARITY,
+    _RESPONSE_COMPLETENESS,
+)
+
+
+_RAG_EVALUATORS: Tuple[EvaluatorPreset, ...] = (
+    EvaluatorPreset(
+        name="GroundednessEvaluator",
+        class_name="GroundednessEvaluator",
+        score_key="groundedness",
+        input_mapping={
+            "query": "$prompt",
+            "response": "$prediction",
+            "context": "$context",
+        },
+        default_threshold=_t("groundedness", ">=", 3.0),
+        categories=frozenset({"rag"}),
+        agent_only=True,
+    ),
+    EvaluatorPreset(
+        name="RelevanceEvaluator",
+        class_name="RelevanceEvaluator",
+        score_key="relevance",
+        input_mapping={
+            "query": "$prompt",
+            "response": "$prediction",
+            "context": "$context",
+        },
+        default_threshold=_t("relevance", ">=", 3.0),
+        categories=frozenset({"rag"}),
+        agent_only=True,
+    ),
+    EvaluatorPreset(
+        name="RetrievalEvaluator",
+        class_name="RetrievalEvaluator",
+        score_key="retrieval",
+        input_mapping={"query": "$prompt", "context": "$context"},
+        default_threshold=_t("retrieval", ">=", 3.0),
+        categories=frozenset({"rag"}),
+        agent_only=True,
+    ),
+    _RESPONSE_COMPLETENESS,
+)
+
+
+_TOOL_USE_EVALUATORS: Tuple[EvaluatorPreset, ...] = (
+    EvaluatorPreset(
+        name="ToolCallAccuracyEvaluator",
+        class_name="ToolCallAccuracyEvaluator",
+        score_key="tool_call_accuracy",
+        input_mapping={
+            "query": "$prompt",
+            "tool_calls": "$tool_calls",
+            "tool_definitions": "$tool_definitions",
+        },
+        default_threshold=_t("tool_call_accuracy", ">=", 0.7),
+        categories=frozenset({"agent"}),
+        agent_only=True,
+    ),
+    EvaluatorPreset(
+        name="IntentResolutionEvaluator",
+        class_name="IntentResolutionEvaluator",
+        score_key="intent_resolution",
+        input_mapping={
+            "query": "$prompt",
+            "response": "$prediction",
+            "tool_definitions": "$tool_definitions",
+        },
+        default_threshold=_t("intent_resolution", ">=", 3.0),
+        categories=frozenset({"agent"}),
+        agent_only=True,
+        needs_conversation=True,
+    ),
+    EvaluatorPreset(
+        name="TaskAdherenceEvaluator",
+        class_name="TaskAdherenceEvaluator",
+        score_key="task_adherence",
+        input_mapping={
+            "query": "$prompt",
+            "response": "$prediction",
+            "tool_definitions": "$tool_definitions",
+        },
+        # azure-ai-evaluation's TaskAdherenceEvaluator returns a binary
+        # 0/1 score (0 = flagged, 1 = adheres) - *not* a 1–5 Likert scale
+        # like IntentResolutionEvaluator. We default to >=0.5 so a score
+        # of 1.0 passes and 0.0 fails.
+        default_threshold=_t("task_adherence", ">=", 0.5),
+        categories=frozenset({"agent"}),
+        agent_only=True,
+        needs_conversation=True,
+    ),
+)
+
+
+_LATENCY = EvaluatorPreset(
+    name="avg_latency_seconds",
+    class_name="_latency",
+    score_key="avg_latency_seconds",
+    input_mapping={},
+    # 30s default accommodates Foundry cold-start spikes on the first
+    # invocation after an agent is created or scaled to zero. Production
+    # users should tighten this in agentops.yaml once warmup is observed.
+    default_threshold=_t("avg_latency_seconds", "<=", 30.0),
+    categories=frozenset({"runtime"}),
+)
+
+
+CATALOG: Dict[str, EvaluatorPreset] = {
+    preset.name: preset
+    for preset in (
+        *_QUALITY_BASELINE,
+        *_MODEL_REFERENCE_EVALUATORS,
+        *_AGENT_ANSWER_EVALUATORS,
+        *_RAG_EVALUATORS,
+        *_TOOL_USE_EVALUATORS,
+        _LATENCY,
+    )
+}
+
+
+# ---------------------------------------------------------------------------
+# Dataset shape detection
+# ---------------------------------------------------------------------------
+
+
+@dataclass(frozen=True)
+class DatasetShape:
+    """Boolean flags summarising the columns present in a dataset."""
+
+    has_context: bool
+    has_tool_calls: bool
+    has_tool_definitions: bool
+    row_count: int
+
+    @property
+    def looks_rag(self) -> bool:
+        return self.has_context
+
+    @property
+    def looks_tool_use(self) -> bool:
+        return self.has_tool_calls or self.has_tool_definitions
+
+
+def detect_dataset_shape(dataset_path: Path, *, sample: int = 50) -> DatasetShape:
+    """Inspect up to ``sample`` rows of ``dataset_path`` and report the shape.
+
+    Truthy values are required - empty strings, empty lists, and ``None`` do
+    not count as the field being present.
+    """
+    if not dataset_path.exists():
+        raise FileNotFoundError(f"dataset file not found: {dataset_path}")
+
+    has_context = False
+    has_tool_calls = False
+    has_tool_definitions = False
+    count = 0
+
+    with dataset_path.open("r", encoding="utf-8") as handle:
+        for line in handle:
+            stripped = line.strip()
+            if not stripped:
+                continue
+            count += 1
+            try:
+                row = json.loads(stripped)
+            except json.JSONDecodeError as exc:
+                raise ValueError(
+                    f"{dataset_path}: invalid JSON on line {count}: {exc}"
+                ) from exc
+            if not isinstance(row, dict):
+                raise ValueError(
+                    f"{dataset_path}: line {count} is not a JSON object"
+                )
+
+            if not has_context and row.get("context"):
+                has_context = True
+            if not has_tool_calls and row.get("tool_calls"):
+                has_tool_calls = True
+            if not has_tool_definitions and row.get("tool_definitions"):
+                has_tool_definitions = True
+
+            if count >= sample and (
+                has_context and (has_tool_calls or has_tool_definitions)
+            ):
+                # Already saw both signals; no need to keep reading.
+                break
+
+    if count == 0:
+        raise ValueError(f"{dataset_path}: dataset is empty")
+
+    return DatasetShape(
+        has_context=has_context,
+        has_tool_calls=has_tool_calls,
+        has_tool_definitions=has_tool_definitions,
+        row_count=count,
+    )
+
+
+# ---------------------------------------------------------------------------
+# Selection
+# ---------------------------------------------------------------------------
+
+
+def select_evaluators(
+    target: TargetResolution,
+    shape: DatasetShape,
+    *,
+    overrides: Optional[List[str]] = None,
+    threshold_metrics: Optional[Iterable[str]] = None,
+) -> List[EvaluatorPreset]:
+    """Return the ordered list of evaluators to run.
+
+    When ``overrides`` is provided it wins outright - the inference rules are
+    bypassed. Each name must exist in :data:`CATALOG` or a ``ValueError`` is
+    raised.
+
+    Otherwise the rules are:
+
+    * Always include the baseline quality evaluators.
+    * If the target is a raw model, add exact/reference-answer evaluators and
+      stop here. Agent-specific evaluators are not meaningful (no tool calls,
+      no retrieved context).
+    * If the target is an agent without RAG/tool-use rows, add answer-quality
+      evaluators that tolerate free-form responses.
+    * If the dataset has ``context`` rows, add the RAG evaluators.
+    * If the dataset has ``tool_calls`` or ``tool_definitions``, add the agent
+      evaluators.
+    * Always append the runtime ``avg_latency_seconds`` evaluator.
+    """
+    if overrides:
+        resolved: List[EvaluatorPreset] = []
+        for name in overrides:
+            preset = CATALOG.get(name)
+            if preset is None:
+                known = ", ".join(sorted(CATALOG.keys()))
+                raise ValueError(
+                    f"unknown evaluator override {name!r}. "
+                    f"Known evaluators: {known}"
+                )
+            resolved.append(preset)
+        return resolved
+
+    selected: List[EvaluatorPreset] = list(_QUALITY_BASELINE)
+
+    if target.kind == "model_direct":
+        selected.extend(_MODEL_REFERENCE_EVALUATORS)
+    elif _is_agent_target(target.kind):
+        if not shape.looks_rag and not shape.looks_tool_use:
+            selected.extend(_AGENT_ANSWER_EVALUATORS)
+        if shape.looks_rag:
+            selected.extend(_RAG_EVALUATORS)
+        if shape.looks_tool_use:
+            selected.extend(_TOOL_USE_EVALUATORS)
+            # F1ScoreEvaluator and SimilarityEvaluator compare the
+            # assistant's natural-language reply against ``expected``. In
+            # tool-using datasets ``expected`` is conventionally a behavior
+            # description (e.g. "Calls lookup_order with order_id='ORD-12345'")
+            # rather than the literal reply, so token overlap and semantic
+            # similarity are meaningless and gate well-behaved agents on a
+            # metric that does not apply. Drop both from the selection.
+            _drop = {"F1ScoreEvaluator", "SimilarityEvaluator"}
+            selected = [p for p in selected if p.name not in _drop]
+
+    if threshold_metrics:
+        selected = _include_thresholded_quality_presets(
+            selected,
+            target.kind,
+            threshold_metrics,
+        )
+
+    selected.append(_LATENCY)
+    return selected
+
+
+def _is_agent_target(kind: TargetKind) -> bool:
+    return kind in {"foundry_prompt", "foundry_hosted", "http_json"}
+
+
+def _include_thresholded_quality_presets(
+    selected: List[EvaluatorPreset],
+    target_kind: TargetKind,
+    threshold_metrics: Iterable[str],
+) -> List[EvaluatorPreset]:
+    """Honor explicit thresholds for optional quality/reference metrics.
+
+    A user who already set ``thresholds: {f1_score: ...}`` should not also need
+    to learn the evaluator override escape hatch. This keeps exact-reference F1
+    available as an opt-in gate without making it a default for free-form agent
+    responses.
+    """
+    by_score_key = {preset.score_key: preset for preset in CATALOG.values()}
+    selected_names = {preset.name for preset in selected}
+    result = list(selected)
+    for metric in threshold_metrics:
+        preset = by_score_key.get(metric)
+        if preset is None or preset.name in selected_names:
+            continue
+        if "runtime" in preset.categories:
+            continue
+        if "quality" not in preset.categories and "exact_reference" not in preset.categories:
+            continue
+        if preset.agent_only and not _is_agent_target(target_kind):
+            continue
+        result.append(preset)
+        selected_names.add(preset.name)
+    return result
+
+
+def merge_thresholds(
+    presets: List[EvaluatorPreset],
+    user_thresholds: List[Threshold],
+) -> List[Threshold]:
+    """Combine evaluator default thresholds with user overrides.
+
+    User entries override the preset default for the same metric. Metrics
+    listed by the user that don't correspond to any selected preset are kept
+    as-is - the threshold engine will report them as unmet rather than
+    silently drop them.
+    """
+    by_metric: Dict[str, Threshold] = {}
+    for preset in presets:
+        if preset.default_threshold is not None:
+            by_metric[preset.default_threshold.metric] = preset.default_threshold
+    for override in user_thresholds:
+        by_metric[override.metric] = override
+    # Preserve preset order, then append user-only metrics in original order.
+    ordered: List[Threshold] = []
+    seen: set[str] = set()
+    for preset in presets:
+        if preset.default_threshold is not None:
+            metric = preset.default_threshold.metric
+            ordered.append(by_metric[metric])
+            seen.add(metric)
+    for override in user_thresholds:
+        if override.metric not in seen:
+            ordered.append(override)
+            seen.add(override.metric)
+    return ordered

agentops/core/release_evidence.py

@@ -0,0 +1,56 @@
+"""Versioned schema for AgentOps release evidence artifacts."""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List, Literal, Optional
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+ReadinessStatus = Literal["ready", "ready_with_warnings", "blocked"]
+CheckStatus = Literal["ready", "warning", "blocked", "unknown"]
+
+
+class ReleaseEvidenceLink(BaseModel):
+    """A navigable reference related to the release decision."""
+
+    label: str
+    url: str
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ReleaseEvidenceCheck(BaseModel):
+    """One production-readiness check in the evidence pack."""
+
+    name: str
+    status: CheckStatus
+    summary: str
+    evidence: Dict[str, Any] = Field(default_factory=dict)
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ReleaseEvidence(BaseModel):
+    """Stable machine-readable production-readiness evidence contract."""
+
+    version: int = 1
+    generated_at: str
+    workspace: str
+    status: ReadinessStatus
+    target: Optional[str] = None
+    blockers: List[str] = Field(default_factory=list)
+    warnings: List[str] = Field(default_factory=list)
+    ready: List[str] = Field(default_factory=list)
+    checks: List[ReleaseEvidenceCheck] = Field(default_factory=list)
+    links: List[ReleaseEvidenceLink] = Field(default_factory=list)
+    latest_eval: Dict[str, Any] = Field(default_factory=dict)
+    official_eval: Dict[str, Any] = Field(default_factory=dict)
+    doctor: Dict[str, Any] = Field(default_factory=dict)
+    workflows: Dict[str, Any] = Field(default_factory=dict)
+    foundry: Dict[str, Any] = Field(default_factory=dict)
+    monitoring: Dict[str, Any] = Field(default_factory=dict)
+    trace_dataset: Dict[str, Any] = Field(default_factory=dict)
+    ailz: Dict[str, Any] = Field(default_factory=dict)
+
+    model_config = ConfigDict(extra="forbid")

agentops/core/results.py

@@ -0,0 +1,117 @@
+"""Result dataclasses for the AgentOps 1.0 pipeline.
+
+These shapes are written to ``results.json`` after every ``agentops eval`` run
+and consumed by the reporter and comparison logic.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class RowMetric(BaseModel):
+    """A single evaluator score for one dataset row."""
+
+    name: str
+    value: Optional[float] = None
+    error: Optional[str] = None
+    reason: Optional[str] = None
+
+
+class RowResult(BaseModel):
+    """One evaluated dataset row."""
+
+    row_index: int
+    input: str
+    expected: Optional[str] = None
+    response: str = ""
+    context: Optional[str] = None
+    latency_seconds: Optional[float] = None
+    tool_calls: Optional[List[Any]] = None
+    metrics: List[RowMetric] = Field(default_factory=list)
+    error: Optional[str] = None
+
+
+class ThresholdEvaluation(BaseModel):
+    """A pass/fail check for a single metric on the run aggregate."""
+
+    metric: str
+    criteria: str
+    expected: str
+    actual: str
+    passed: bool
+
+
+class RunSummary(BaseModel):
+    """Top-level pass/fail summary of an evaluation run."""
+
+    items_total: int
+    items_passed_all: int
+    items_pass_rate: float
+    thresholds_total: int
+    thresholds_passed: int
+    threshold_pass_rate: float
+    overall_passed: bool
+
+
+class TargetInfo(BaseModel):
+    """Resolved target information (echoed into results.json)."""
+
+    kind: str
+    raw: str
+    protocol: Optional[str] = None
+    name: Optional[str] = None
+    version: Optional[str] = None
+    url: Optional[str] = None
+    deployment: Optional[str] = None
+
+
+class ComparisonMetric(BaseModel):
+    """Per-metric delta between the current run and a baseline."""
+
+    metric: str
+    current: Optional[float] = None
+    baseline: Optional[float] = None
+    delta: Optional[float] = None
+    direction: str  # "improved" | "regressed" | "unchanged"
+
+
+class ComparisonRow(BaseModel):
+    """Per-row regression / improvement against a baseline."""
+
+    row_index: int
+    current_passed: bool
+    baseline_passed: Optional[bool] = None
+    direction: str  # "improved" | "regressed" | "unchanged" | "new"
+
+
+class ComparisonInfo(BaseModel):
+    """Comparison block included when ``--baseline`` was provided."""
+
+    baseline_path: str
+    baseline_started_at: Optional[str] = None
+    baseline_overall_passed: Optional[bool] = None
+    metrics: List[ComparisonMetric] = Field(default_factory=list)
+    rows: List[ComparisonRow] = Field(default_factory=list)
+
+
+class RunResult(BaseModel):
+    """Full ``results.json`` payload."""
+
+    version: int = 1
+    started_at: str
+    finished_at: str
+    duration_seconds: float
+    target: TargetInfo
+    dataset_path: str
+    evaluators: List[str] = Field(default_factory=list)
+    rows: List[RowResult] = Field(default_factory=list)
+    aggregate_metrics: Dict[str, float] = Field(default_factory=dict)
+    thresholds: List[ThresholdEvaluation] = Field(default_factory=list)
+    summary: RunSummary
+    comparison: Optional[ComparisonInfo] = None
+    config: Dict[str, Any] = Field(default_factory=dict)
+
+    model_config = ConfigDict(extra="forbid")

agentops/mcp/__init__.py

@@ -0,0 +1,10 @@
+"""MCP (Model Context Protocol) server for AgentOps.
+
+Exposes the most-used AgentOps capabilities as MCP tools over stdio so that
+MCP-aware coding agents (Claude Code, Copilot Chat, etc.) can drive AgentOps
+without shelling out to the CLI.
+
+The server is opt-in: it requires the optional ``mcp`` extra
+(``pip install agentops-accelerator[mcp]``) and is started via
+``agentops mcp serve``.
+"""