agentops-accelerator 0.3.0__py3-none-any.whl

agentops/pipeline/comparison.py

@@ -0,0 +1,108 @@
+"""``--baseline`` comparison helpers."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Dict, List, Optional
+
+from agentops.core.results import (
+    ComparisonInfo,
+    ComparisonMetric,
+    ComparisonRow,
+    RunResult,
+)
+
+
+def load_baseline(path: Path) -> RunResult:
+    """Load a previous ``results.json`` for comparison."""
+    if not path.exists():
+        raise FileNotFoundError(f"baseline file not found: {path}")
+    with path.open("r", encoding="utf-8") as handle:
+        payload = json.load(handle)
+    return RunResult.model_validate(payload)
+
+
+def _direction(current: Optional[float], baseline: Optional[float]) -> str:
+    if current is None or baseline is None:
+        return "unchanged"
+    if current > baseline:
+        return "improved"
+    if current < baseline:
+        return "regressed"
+    return "unchanged"
+
+
+def _row_passed(row_metrics: List[Dict[str, float | None]]) -> bool:
+    """Best-effort proxy: a row is "passing" when no metric reports an error."""
+    return all("error" not in metric or not metric["error"] for metric in row_metrics)
+
+
+def build_comparison(
+    *,
+    current: RunResult,
+    baseline: RunResult,
+    baseline_path: Path,
+) -> ComparisonInfo:
+    metrics: List[ComparisonMetric] = []
+    metric_names = sorted(set(current.aggregate_metrics) | set(baseline.aggregate_metrics))
+    for name in metric_names:
+        current_value = current.aggregate_metrics.get(name)
+        baseline_value = baseline.aggregate_metrics.get(name)
+        delta = (
+            current_value - baseline_value
+            if current_value is not None and baseline_value is not None
+            else None
+        )
+        metrics.append(
+            ComparisonMetric(
+                metric=name,
+                current=current_value,
+                baseline=baseline_value,
+                delta=delta,
+                direction=_direction(current_value, baseline_value),
+            )
+        )
+
+    rows: List[ComparisonRow] = []
+    baseline_by_index = {row.row_index: row for row in baseline.rows}
+    for row in current.rows:
+        baseline_row = baseline_by_index.get(row.row_index)
+        current_pass = row.error is None and all(
+            m.value is not None or m.error is None for m in row.metrics
+        )
+        if baseline_row is None:
+            rows.append(
+                ComparisonRow(
+                    row_index=row.row_index,
+                    current_passed=current_pass,
+                    baseline_passed=None,
+                    direction="new",
+                )
+            )
+            continue
+        baseline_pass = baseline_row.error is None and all(
+            m.value is not None or m.error is None for m in baseline_row.metrics
+        )
+        if current_pass and not baseline_pass:
+            direction = "improved"
+        elif baseline_pass and not current_pass:
+            direction = "regressed"
+        else:
+            direction = "unchanged"
+        rows.append(
+            ComparisonRow(
+                row_index=row.row_index,
+                current_passed=current_pass,
+                baseline_passed=baseline_pass,
+                direction=direction,
+            )
+        )
+
+    return ComparisonInfo(
+        baseline_path=str(baseline_path),
+        baseline_started_at=baseline.started_at,
+        baseline_overall_passed=baseline.summary.overall_passed,
+        metrics=metrics,
+        rows=rows,
+    )

agentops/pipeline/diagnostics.py

@@ -0,0 +1,51 @@
+"""Shared runtime diagnostics for pipeline errors."""
+
+from __future__ import annotations
+
+import re
+
+
+_TENANT_MISMATCH_MARKERS = (
+    "Tenant provided in token does not match resource token",
+    "Tenant provided in token does not match resource tenant",
+    "does not match resource tenant",
+)
+
+_TENANT_MISMATCH_GUIDANCE = (
+    " Check that `az login` is using the same tenant as the Foundry project, "
+    "or run `az login --tenant <tenant-id>`."
+)
+
+
+def with_tenant_mismatch_guidance(message: str) -> str:
+    """Append actionable Azure tenant guidance to matching error messages."""
+    if "az login --tenant" in message:
+        return message
+    if any(marker in message for marker in _TENANT_MISMATCH_MARKERS):
+        return f"{message}{_TENANT_MISMATCH_GUIDANCE}"
+    return message
+
+
+def summarize_exception(exc: BaseException) -> str:
+    """Return a concise, user-facing summary for known noisy SDK errors."""
+    message = with_tenant_mismatch_guidance(str(exc))
+    validation_messages = _extract_foundry_validation_messages(message)
+    if validation_messages:
+        joined = "; ".join(validation_messages)
+        return f"Foundry cloud validation failed: {joined}"
+    return message
+
+
+def _extract_foundry_validation_messages(message: str) -> list[str]:
+    if "Evaluation failed validation" not in message:
+        return []
+    matches = re.findall(r"Message:\s*(.+?)(?=\\n\}|\n\}|$)", message)
+    cleaned: list[str] = []
+    seen: set[str] = set()
+    for raw in matches:
+        text = raw.strip().strip("'\"")
+        if not text or text in seen:
+            continue
+        seen.add(text)
+        cleaned.append(text)
+    return cleaned

agentops/pipeline/invocations.py

@@ -0,0 +1,535 @@
+"""Target invocation backends for AgentOps 1.0.
+
+Each backend is a single function with the signature::
+
+    invoke(
+        target: TargetResolution,
+        config: AgentOpsConfig,
+        row: dict[str, Any],
+        *,
+        timeout: float,
+    ) -> InvocationResult
+
+The orchestrator dispatches based on :attr:`TargetResolution.kind`. All Azure
+SDK imports are lazy so the package imports without optional dependencies.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import time
+import urllib.error
+import urllib.request
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional
+
+from agentops.core.agentops_config import AgentOpsConfig, TargetResolution
+from agentops.pipeline.diagnostics import with_tenant_mismatch_guidance
+
+
+@dataclass
+class InvocationResult:
+    """Outcome of invoking the target on one dataset row."""
+
+    response: str
+    latency_seconds: float
+    tool_calls: Optional[List[Any]] = None
+    metadata: Dict[str, Any] = field(default_factory=dict)
+
+
+# Maximum number of follow-up calls when running the tool-execution loop
+# against a Foundry hosted/prompt agent. Most agents resolve in 1–2 hops;
+# the cap exists to bound retries against pathological multi-step plans.
+_MAX_TOOL_ITERATIONS = 4
+
+# Generic stub returned to the agent for every function call during the
+# tool-execution loop. The toolkit cannot run project-specific tool
+# implementations, so a uniform "ok" stub keeps the loop fully generic
+# while letting the agent produce its final natural-language reply.
+_TOOL_STUB_OUTPUT = '{"status": "ok"}'
+
+
+def _summarise_tool_calls(calls: List[Any]) -> str:
+    """Build a short, human-readable summary of executed tool calls.
+
+    Used as a last-resort fallback when the agent never produces
+    assistant text - quality evaluators still need a non-empty
+    ``response`` string to score.
+    """
+    parts: List[str] = []
+    for call in calls:
+        if not isinstance(call, dict):
+            continue
+        name = call.get("name") or "tool"
+        args = call.get("arguments")
+        parts.append(f"[Called {name}({args})]" if args else f"[Called {name}]")
+    return " ".join(parts) if parts else "[tool_call]"
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+
+def _credential() -> Any:
+    """Return a cached ``DefaultAzureCredential`` singleton.
+
+    Caching matters: each row invocation needs a token, and constructing a
+    fresh ``DefaultAzureCredential`` per call walks the full credential
+    chain (Azure CLI / PowerShell subprocesses included), which is both
+    slow and prone to transient subprocess failures on the first try.
+    Caching the credential lets the SDK reuse its internal token cache
+    across rows.
+    """
+    global _CREDENTIAL_SINGLETON
+    if _CREDENTIAL_SINGLETON is None:
+        from azure.identity import DefaultAzureCredential  # noqa: WPS433
+
+        _CREDENTIAL_SINGLETON = DefaultAzureCredential(exclude_developer_cli_credential=True, process_timeout=30)
+    return _CREDENTIAL_SINGLETON
+
+
+_CREDENTIAL_SINGLETON: Any = None
+
+
+def _get_token(scope: str) -> str:
+    """Acquire a token for ``scope``, retrying once on transient failures.
+
+    The first credential-chain walk on Windows occasionally fails because
+    the Azure CLI / PowerShell subprocess is slow to spawn. Retrying once
+    after the credential is warmed up almost always succeeds.
+    """
+    try:
+        return _credential().get_token(scope).token
+    except Exception:  # noqa: BLE001 - retry once on any transient failure
+        global _CREDENTIAL_SINGLETON
+        _CREDENTIAL_SINGLETON = None  # force a fresh chain walk
+        return _credential().get_token(scope).token
+
+
+def _project_endpoint(config: AgentOpsConfig) -> str:
+    endpoint = config.project_endpoint or os.getenv("AZURE_AI_FOUNDRY_PROJECT_ENDPOINT")
+    if not endpoint:
+        raise RuntimeError(
+            "Foundry targets require a project endpoint URL. Set "
+            "'project_endpoint' in agentops.yaml or set the "
+            "AZURE_AI_FOUNDRY_PROJECT_ENDPOINT environment variable."
+        )
+    return endpoint.rstrip("/")
+
+
+def _row_input(row: Dict[str, Any]) -> str:
+    value = row.get("input")
+    if value is None:
+        raise ValueError("dataset row is missing required 'input' field")
+    return str(value)
+
+
+def _http_request_json(
+    *,
+    method: str,
+    url: str,
+    headers: Dict[str, str],
+    body: Optional[Dict[str, Any]] = None,
+    timeout: float,
+) -> Dict[str, Any]:
+    encoded = json.dumps(body or {}).encode("utf-8") if method != "GET" else None
+    request = urllib.request.Request(
+        url=url, data=encoded, method=method, headers=headers
+    )
+    last_exc: Optional[BaseException] = None
+    for attempt in range(1, 4):
+        try:
+            with urllib.request.urlopen(request, timeout=timeout) as response:  # noqa: S310
+                payload = response.read().decode("utf-8")
+            break
+        except urllib.error.HTTPError as exc:
+            detail = exc.read().decode("utf-8", errors="replace") if exc.fp else ""
+            transient = exc.code >= 500 or exc.code == 429
+            if transient and attempt < 3:
+                time.sleep(2 ** attempt)
+                last_exc = exc
+                continue
+            message = f"HTTP {exc.code} from {url}: {detail or exc.reason}"
+            raise RuntimeError(with_tenant_mismatch_guidance(message)) from exc
+        except urllib.error.URLError as exc:
+            if attempt < 3:
+                time.sleep(2 ** attempt)
+                last_exc = exc
+                continue
+            raise
+    else:  # pragma: no cover - loop exits via break/raise
+        raise RuntimeError(f"HTTP request to {url} failed: {last_exc!r}")
+    if not payload:
+        return {}
+    return json.loads(payload)
+
+
+def _dot_path(payload: Any, path: str) -> Any:
+    """Resolve ``a.b.c`` or ``a.0.b`` against a JSON-like object."""
+    current = payload
+    for token in path.split("."):
+        if current is None:
+            return None
+        if isinstance(current, list):
+            try:
+                current = current[int(token)]
+            except (ValueError, IndexError):
+                return None
+            continue
+        if isinstance(current, dict):
+            current = current.get(token)
+            continue
+        return None
+    return current
+
+
+def _extract_responses_text(payload: Dict[str, Any]) -> str:
+    """Pull assistant text from a Foundry/Responses-API payload.
+
+    Returns an empty string when the response only contains tool/function
+    calls (the caller must submit ``function_call_output`` items via a
+    follow-up call to obtain the final natural-language reply).
+    """
+    direct = payload.get("output_text")
+    if isinstance(direct, str) and direct.strip():
+        return direct.strip()
+
+    output = payload.get("output")
+    if isinstance(output, list):
+        parts: List[str] = []
+        for item in output:
+            if not isinstance(item, dict):
+                continue
+            if (
+                item.get("type") in {"message", "assistant_message"}
+                or item.get("role") == "assistant"
+            ):
+                content = item.get("content")
+                if isinstance(content, str):
+                    parts.append(content)
+                elif isinstance(content, list):
+                    for chunk in content:
+                        if isinstance(chunk, dict):
+                            text = chunk.get("text") or chunk.get("output_text")
+                            if isinstance(text, str):
+                                parts.append(text)
+                        elif isinstance(chunk, str):
+                            parts.append(chunk)
+        if parts:
+            return "\n".join(parts).strip()
+        return ""
+
+    return ""
+
+
+def _extract_responses_tool_calls(payload: Dict[str, Any]) -> Optional[List[Any]]:
+    output = payload.get("output")
+    if not isinstance(output, list):
+        return None
+    calls: List[Any] = []
+    for item in output:
+        if isinstance(item, dict) and item.get("type") in {
+            "tool_call",
+            "function_call",
+        }:
+            calls.append(item)
+    return calls or None
+
+
+# ---------------------------------------------------------------------------
+# Backends
+# ---------------------------------------------------------------------------
+
+
+def _invoke_model_direct(
+    target: TargetResolution,
+    config: AgentOpsConfig,
+    row: Dict[str, Any],
+    *,
+    timeout: float,  # noqa: ARG001
+) -> InvocationResult:
+    from azure.ai.projects import AIProjectClient  # noqa: WPS433
+
+    project_endpoint = _project_endpoint(config)
+    client = AIProjectClient(endpoint=project_endpoint, credential=_credential())
+    openai_client = client.get_openai_client()
+
+    assert target.deployment is not None
+    started = time.perf_counter()
+    last_exc: Optional[BaseException] = None
+    response = None
+    for attempt in range(1, 4):
+        try:
+            response = openai_client.chat.completions.create(
+                model=target.deployment,
+                messages=[{"role": "user", "content": _row_input(row)}],
+            )
+            break
+        except Exception as exc:  # noqa: BLE001
+            status = getattr(exc, "status_code", None)
+            transient = status is None or status >= 500 or status == 429
+            if transient and attempt < 3:
+                time.sleep(2 ** attempt)
+                last_exc = exc
+                continue
+            raise
+    if response is None:
+        raise RuntimeError(f"model_direct invocation failed after retries: {last_exc!r}")
+    elapsed = time.perf_counter() - started
+
+    text = ""
+    if response.choices:
+        message = response.choices[0].message
+        if message and message.content:
+            text = message.content.strip()
+    if not text:
+        raise RuntimeError("model_direct invocation returned empty content")
+
+    return InvocationResult(response=text, latency_seconds=elapsed)
+
+
+def _run_responses_tool_loop(
+    *,
+    url: str,
+    headers: Dict[str, str],
+    initial_body: Dict[str, Any],
+    timeout: float,
+    follow_up_extras: Optional[Dict[str, Any]] = None,
+) -> tuple[str, List[Any], float]:
+    """Drive a Foundry/Responses-API tool-execution loop.
+
+    Sends ``initial_body`` to ``url``, then repeatedly submits stub
+    ``function_call_output`` items back via ``previous_response_id`` until
+    the agent emits assistant text or the iteration cap is reached.
+
+    ``follow_up_extras`` is merged into every follow-up request body
+    (e.g. ``agent_reference`` for prompt agents).
+
+    Returns ``(text, aggregated_tool_calls, elapsed_seconds)``.
+    """
+    started = time.perf_counter()
+    aggregated_tool_calls: List[Any] = []
+    text = ""
+    body = initial_body
+
+    for _iteration in range(_MAX_TOOL_ITERATIONS):
+        payload = _http_request_json(
+            method="POST",
+            url=url,
+            headers=headers,
+            body=body,
+            timeout=timeout,
+        )
+
+        iteration_calls = _extract_responses_tool_calls(payload) or []
+        aggregated_tool_calls.extend(iteration_calls)
+
+        text = _extract_responses_text(payload)
+        if text or not iteration_calls:
+            break
+
+        previous_response_id = payload.get("id")
+        follow_up_input: List[Dict[str, Any]] = []
+        for call in iteration_calls:
+            call_id = call.get("call_id") or call.get("id")
+            if not call_id:
+                continue
+            follow_up_input.append(
+                {
+                    "type": "function_call_output",
+                    "call_id": call_id,
+                    "output": _TOOL_STUB_OUTPUT,
+                }
+            )
+
+        if not follow_up_input or not previous_response_id:
+            break
+
+        body = {
+            "input": follow_up_input,
+            "previous_response_id": previous_response_id,
+        }
+        if follow_up_extras:
+            body.update(follow_up_extras)
+
+    elapsed = time.perf_counter() - started
+    return text, aggregated_tool_calls, elapsed
+
+
+def _invoke_foundry_prompt(
+    target: TargetResolution,
+    config: AgentOpsConfig,
+    row: Dict[str, Any],
+    *,
+    timeout: float,
+) -> InvocationResult:
+    project_endpoint = _project_endpoint(config)
+    token = _get_token("https://ai.azure.com/.default")
+    headers = {
+        "Content-Type": "application/json",
+        "Authorization": f"Bearer {token}",
+    }
+
+    assert target.name is not None and target.version is not None
+    url = f"{project_endpoint}/openai/v1/responses"
+    agent_reference = {
+        "type": "agent_reference",
+        "name": target.name,
+        "version": target.version,
+    }
+    initial_body: Dict[str, Any] = {
+        "input": [{"role": "user", "content": _row_input(row)}],
+        "agent_reference": agent_reference,
+    }
+
+    text, aggregated_tool_calls, elapsed = _run_responses_tool_loop(
+        url=url,
+        headers=headers,
+        initial_body=initial_body,
+        timeout=timeout,
+        follow_up_extras={"agent_reference": agent_reference},
+    )
+
+    if not text:
+        if aggregated_tool_calls:
+            text = _summarise_tool_calls(aggregated_tool_calls)
+        else:
+            raise ValueError(
+                "Foundry response did not include assistant output text"
+            )
+
+    return InvocationResult(
+        response=text,
+        latency_seconds=elapsed,
+        tool_calls=aggregated_tool_calls or None,
+    )
+
+
+def _invoke_foundry_hosted(
+    target: TargetResolution,
+    config: AgentOpsConfig,
+    row: Dict[str, Any],
+    *,
+    timeout: float,
+) -> InvocationResult:
+    assert target.url is not None
+    token = _get_token("https://ai.azure.com/.default")
+    headers = {
+        "Content-Type": "application/json",
+        "Authorization": f"Bearer {token}",
+        **config.headers,
+    }
+
+    if target.protocol == "responses":
+        url = target.url.rstrip("/")
+        if not url.endswith("/responses"):
+            url = f"{url}/responses"
+        initial_body = {"input": [{"role": "user", "content": _row_input(row)}]}
+
+        text, aggregated_tool_calls, elapsed = _run_responses_tool_loop(
+            url=url,
+            headers=headers,
+            initial_body=initial_body,
+            timeout=timeout,
+        )
+
+        if not text:
+            if aggregated_tool_calls:
+                text = _summarise_tool_calls(aggregated_tool_calls)
+            else:
+                raise ValueError(
+                    "Foundry response did not include assistant output text"
+                )
+
+        return InvocationResult(
+            response=text,
+            latency_seconds=elapsed,
+            tool_calls=aggregated_tool_calls or None,
+        )
+
+    return _invoke_http_json(target, config, row, timeout=timeout)
+
+
+def _invoke_http_json(
+    target: TargetResolution,
+    config: AgentOpsConfig,
+    row: Dict[str, Any],
+    *,
+    timeout: float,
+) -> InvocationResult:
+    assert target.url is not None
+    headers: Dict[str, str] = {"Content-Type": "application/json", **config.headers}
+    if config.auth_header_env:
+        token = os.getenv(config.auth_header_env)
+        if not token:
+            raise RuntimeError(
+                f"auth_header_env {config.auth_header_env!r} is set in config but "
+                "the environment variable is empty"
+            )
+        headers["Authorization"] = f"Bearer {token}"
+
+    request_field = config.request_field or "message"
+    body: Dict[str, Any] = {request_field: _row_input(row)}
+
+    started = time.perf_counter()
+    payload = _http_request_json(
+        method="POST",
+        url=target.url,
+        headers=headers,
+        body=body,
+        timeout=timeout,
+    )
+    elapsed = time.perf_counter() - started
+
+    response_path = config.response_field or "text"
+    response_text = _dot_path(payload, response_path)
+    if response_text is None:
+        for fallback in ("response", "output", "content", "message", "text"):
+            response_text = payload.get(fallback)
+            if response_text:
+                break
+    if response_text is None:
+        raise ValueError(
+            f"HTTP/JSON response did not contain field {response_path!r}; "
+            f"got top-level keys: {sorted(payload.keys())}"
+        )
+    if not isinstance(response_text, str):
+        response_text = json.dumps(response_text, ensure_ascii=False)
+
+    tool_calls: Optional[List[Any]] = None
+    if config.tool_calls_field:
+        extracted = _dot_path(payload, config.tool_calls_field)
+        if isinstance(extracted, list):
+            tool_calls = extracted
+
+    return InvocationResult(
+        response=response_text.strip(),
+        latency_seconds=elapsed,
+        tool_calls=tool_calls,
+    )
+
+
+# ---------------------------------------------------------------------------
+# Dispatch
+# ---------------------------------------------------------------------------
+
+
+def invoke(
+    target: TargetResolution,
+    config: AgentOpsConfig,
+    row: Dict[str, Any],
+    *,
+    timeout: float,
+) -> InvocationResult:
+    """Dispatch to the right backend based on the resolved target kind."""
+    if target.kind == "model_direct":
+        return _invoke_model_direct(target, config, row, timeout=timeout)
+    if target.kind == "foundry_prompt":
+        return _invoke_foundry_prompt(target, config, row, timeout=timeout)
+    if target.kind == "foundry_hosted":
+        return _invoke_foundry_hosted(target, config, row, timeout=timeout)
+    if target.kind == "http_json":
+        return _invoke_http_json(target, config, row, timeout=timeout)
+    raise ValueError(f"unknown target kind: {target.kind}")