agentops-accelerator 0.3.0__py3-none-any.whl

agentops/agent/checks/errors.py

@@ -0,0 +1,279 @@
+"""Errors / failure rate check."""
+
+from __future__ import annotations
+
+from typing import List, Optional
+
+from agentops.agent.config import ErrorsCheckConfig
+from agentops.agent.findings import Category, Finding, Severity
+from agentops.agent.sources.azure_monitor import AzureMonitorPayload
+from agentops.agent.sources.foundry_control import FoundryControlPayload
+
+
+def run_errors_check(
+    monitor: Optional[AzureMonitorPayload],
+    foundry: Optional[FoundryControlPayload],
+    config: ErrorsCheckConfig,
+) -> List[Finding]:
+    findings: List[Finding] = []
+
+    if (
+        monitor
+        and monitor.error_rate is not None
+        and monitor.error_rate > config.rate_threshold
+    ):
+        severity = (
+            Severity.CRITICAL
+            if monitor.error_rate >= config.rate_threshold * 2
+            else Severity.WARNING
+        )
+        findings.append(
+            Finding(
+                id="errors.production_rate",
+                severity=severity,
+                category=Category.RELIABILITY,
+                title="Production error rate above threshold",
+                summary=(
+                    f"App Insights reports {monitor.error_count} failed "
+                    f"requests over {monitor.request_count} total "
+                    f"({monitor.error_rate * 100:.2f}%), above the "
+                    f"{config.rate_threshold * 100:.2f}% threshold."
+                ),
+                recommendation=(
+                    "Open the App Insights resource, group failures by "
+                    "operation, and inspect the most common exception "
+                    "type."
+                ),
+                source="azure_monitor",
+                evidence={
+                    "error_count": monitor.error_count,
+                    "request_count": monitor.request_count,
+                    "error_rate": monitor.error_rate,
+                    "threshold": config.rate_threshold,
+                },
+            )
+        )
+
+    if (
+        foundry
+        and foundry.failure_rate is not None
+        and foundry.failure_rate > config.rate_threshold
+    ):
+        findings.append(
+            Finding(
+                id="errors.foundry_runs",
+                severity=Severity.WARNING,
+                category=Category.RELIABILITY,
+                title="Foundry agent run failure rate elevated",
+                summary=(
+                    f"Foundry control plane reports "
+                    f"{foundry.failed_runs}/{foundry.total_runs} failed "
+                    f"runs ({foundry.failure_rate * 100:.2f}%)."
+                ),
+                recommendation=(
+                    "Review recent Foundry runs, paying attention to "
+                    "tool-call errors and rate limits."
+                ),
+                source="foundry_control",
+                evidence={
+                    "failed_runs": foundry.failed_runs,
+                    "total_runs": foundry.total_runs,
+                    "failure_rate": foundry.failure_rate,
+                },
+            )
+        )
+
+    findings.extend(_check_no_runtime_telemetry(monitor))
+    findings.extend(_check_rate_limit_pressure(monitor, config))
+    findings.extend(_check_no_token_telemetry(monitor))
+
+    return findings
+
+
+def _check_rate_limit_pressure(
+    monitor: Optional[AzureMonitorPayload],
+    config: ErrorsCheckConfig,
+) -> List[Finding]:
+    """AI.154 — surface HTTP 429 spikes from Azure OpenAI / AI Services.
+
+    Rate-limit responses indicate the workload is exhausting its TPM /
+    RPM quota or PTU capacity. Even when the overall error rate is
+    healthy, 429s tell the team to raise quotas or add a backoff /
+    gateway layer **before** users see degraded behaviour.
+    """
+    if monitor is None or not monitor.rate_limit_429_count:
+        return []
+    # Treat the same rate threshold as the error-rate check: if 429s
+    # exceed ``rate_threshold`` of total requests, escalate. With no
+    # request_count info, fall back to a hard floor of 10 hits.
+    total = monitor.request_count
+    threshold_hits = max(10, int(total * config.rate_threshold)) if total else 10
+    if monitor.rate_limit_429_count < threshold_hits:
+        return []
+    severity = (
+        Severity.CRITICAL
+        if monitor.rate_limit_429_count >= threshold_hits * 2
+        else Severity.WARNING
+    )
+    return [
+        Finding(
+            id="errors.rate_limit_pressure",
+            severity=severity,
+            category=Category.RELIABILITY,
+            title="Azure OpenAI rate-limit responses (HTTP 429) above threshold",
+            summary=(
+                f"App Insights reports {monitor.rate_limit_429_count} HTTP "
+                f"429 responses from Azure OpenAI / AI Services over the "
+                "lookback window. The workload is hitting its TPM / RPM "
+                "ceiling and clients are being throttled."
+            ),
+            recommendation=(
+                "Raise the deployment's TPM / RPM quota, switch high-volume "
+                "flows to a Provisioned-Throughput Unit (PTU) deployment, "
+                "or add an APIM gateway with retry + backoff so clients "
+                "do not see the 429s directly."
+            ),
+            source="azure_monitor",
+            evidence={
+                "rate_limit_429_count": monitor.rate_limit_429_count,
+                "request_count": monitor.request_count,
+                "threshold_hits": threshold_hits,
+            },
+        )
+    ]
+
+
+def _check_no_token_telemetry(
+    monitor: Optional[AzureMonitorPayload],
+) -> List[Finding]:
+    """AI.132 — warn when the runtime emits requests but no token telemetry.
+
+    The OpenTelemetry GenAI semantic conventions
+    (``gen_ai.usage.input_tokens`` / ``gen_ai.usage.output_tokens``)
+    are the canonical signal for token-cost monitoring. When the agent
+    runtime emits dependency spans but no token attributes, the team
+    flies blind on cost and on AI.132's "Monitor token usage" guidance.
+    """
+    if monitor is None:
+        return []
+    if (monitor.diagnostics or {}).get("token_status") == "error":
+        return []
+    if monitor.request_count <= 0:
+        return []  # absence of telemetry is covered by errors.no_runtime_telemetry
+    in_t = monitor.input_token_count or 0
+    out_t = monitor.output_token_count or 0
+    if in_t > 0 or out_t > 0:
+        return []
+    return [
+        Finding(
+            id="opex.no_token_telemetry",
+            severity=Severity.WARNING,
+            category=Category.OPERATIONAL_EXCELLENCE,
+            title="Runtime emits requests but no token-usage telemetry",
+            summary=(
+                f"App Insights recorded {monitor.request_count} agent "
+                "requests but reports zero input / output tokens. The "
+                "OpenTelemetry GenAI conventions "
+                "(`gen_ai.usage.input_tokens` / "
+                "`gen_ai.usage.output_tokens`) are not being emitted, so "
+                "token-cost monitoring and the Tokens card on the "
+                "cockpit stay grey."
+            ),
+            recommendation=(
+                "Wire the OpenAI instrumentor on the agent runtime "
+                "(`opentelemetry-instrumentation-openai-v2` or the "
+                "Azure SDK's built-in tracing). The instrumentor sets "
+                "the token-usage attributes from the model response "
+                "automatically."
+            ),
+            source="azure_monitor",
+            evidence={
+                "request_count": monitor.request_count,
+                "input_token_count": in_t,
+                "output_token_count": out_t,
+            },
+        )
+    ]
+
+
+def _check_no_runtime_telemetry(
+    monitor: Optional[AzureMonitorPayload],
+) -> List[Finding]:
+    """Warn when Azure Monitor is not wired, or wired but silent.
+
+    Two failure modes count, both blockers for production
+    observability:
+
+    * **Not configured.** The ``azure_monitor`` source is enabled but
+      has no ``app_insights_resource_id`` / ``log_analytics_workspace_id``,
+      so it reports ``status: skipped``. Doctor has no production
+      observability at all.
+    * **Configured but empty.** The source reports ``status: ok`` but
+      ``request_count == 0`` over the lookback, so the App Insights
+      workspace exists but the agent runtime is not emitting
+      telemetry to it.
+
+    The two cases share one finding because the user-facing
+    remediation is identical: wire the OpenTelemetry exporter on the
+    agent runtime side, and configure the resource id on the
+    ``azure_monitor`` source in ``agent.yaml``. If the source is
+    explicitly ``enabled: false`` we treat that as an opt-out and
+    stay quiet.
+    """
+    if monitor is None:
+        return []
+    diag = monitor.diagnostics or {}
+    status = diag.get("status")
+
+    if status == "disabled":
+        return []
+
+    if status == "ok" and monitor.request_count <= 0:
+        summary = (
+            "Application Insights / Log Analytics is reachable but "
+            "reports 0 requests over the lookback window. The "
+            "agent runtime is not emitting telemetry, so the "
+            "cockpit, latency, errors, and runtime-safety "
+            "checks have nothing to grade."
+        )
+        evidence = {
+            "request_count": monitor.request_count,
+            "monitor_status": status,
+            "mode": "configured_but_empty",
+        }
+    elif status == "skipped":
+        summary = (
+            "The `azure_monitor` source is not configured "
+            f"({diag.get('reason') or 'unknown reason'}). Without "
+            "App Insights wired up, Doctor has no production "
+            "observability, so latency, errors, runtime safety, and "
+            "telemetry-based reliability checks all stay grey."
+        )
+        evidence = {
+            "monitor_status": status,
+            "reason": diag.get("reason"),
+            "mode": "not_configured",
+        }
+    else:
+        return []
+
+    return [
+        Finding(
+            id="errors.no_runtime_telemetry",
+            severity=Severity.WARNING,
+            category=Category.RELIABILITY,
+            title="Production telemetry is not wired to the agent",
+            summary=summary,
+            recommendation=(
+                "Configure `sources.azure_monitor.app_insights_resource_id` "
+                "or set `APPLICATIONINSIGHTS_CONNECTION_STRING` with an "
+                "`ApplicationId`, install the `[agent]` extra, and connect "
+                "Azure Monitor OpenTelemetry on the agent runtime "
+                "(call `configure_azure_monitor()` on startup). "
+                "See `docs/tutorial-end-to-end.md` -> "
+                "'Wire observability'."
+            ),
+            source="azure_monitor",
+            evidence=evidence,
+        )
+    ]

agentops/agent/checks/foundry_config.py

@@ -0,0 +1,75 @@
+"""Foundry control-plane configuration audit (Operational Excellence category).
+
+Mirrors the ``errors.no_runtime_telemetry`` pattern for the Foundry
+control plane. The Doctor warns when Foundry was expected but the
+control-plane source is unconfigured or unreachable.
+
+If the user explicitly opted out (``foundry_control.enabled: false``)
+we stay silent - that is the documented way to say "we are not on
+Foundry".
+"""
+
+from __future__ import annotations
+
+from typing import List, Optional
+
+from agentops.agent.findings import Category, Finding, Severity
+from agentops.agent.sources.foundry_control import FoundryControlPayload
+
+SOURCE_NAME = "foundry_control"
+
+
+def run_foundry_config_check(
+    foundry: Optional[FoundryControlPayload],
+) -> List[Finding]:
+    """Audit the Foundry control plane configuration."""
+    if foundry is None:
+        return []
+
+    diag = foundry.diagnostics or {}
+    status = diag.get("status")
+
+    if status == "disabled":
+        return []
+
+    findings: List[Finding] = []
+    if status != "ok":
+        findings.append(_no_foundry_control_finding(diag))
+        return findings
+
+    return findings
+
+
+def _no_foundry_control_finding(diag: dict) -> Finding:
+    status = diag.get("status") or "unknown"
+    reason = diag.get("reason") or (
+        "the source is enabled but did not return a healthy status"
+    )
+    return Finding(
+        id="opex.no_foundry_control_configured",
+        severity=Severity.WARNING,
+        category=Category.OPERATIONAL_EXCELLENCE,
+        title="Foundry control plane is not configured",
+        summary=(
+            "The `foundry_control` source is enabled but reports "
+            f"`status: {status}` ({reason}). Without it, Doctor "
+            "cannot see Foundry-side agents, evaluation rules, or "
+            "run failures, so safety-config and Foundry-run checks "
+            "stay grey."
+        ),
+        recommendation=(
+            "Set `sources.foundry_control.project_endpoint` (or the "
+            "`AZURE_AI_FOUNDRY_PROJECT_ENDPOINT` env var) in "
+            "`.agentops/agent.yaml`, install the `[foundry]` extra, "
+            "and grant the running identity at least `Reader` on the "
+            "Foundry project. If this project does not use Foundry, "
+            "set `sources.foundry_control.enabled: false` to opt out "
+            "explicitly."
+        ),
+        source=SOURCE_NAME,
+        evidence={
+            "monitor_status": status,
+            "reason": reason,
+            "mode": "not_configured",
+        },
+    )

agentops/agent/checks/latency.py

@@ -0,0 +1,84 @@
+"""Latency check based on Azure Monitor and AgentOps results history."""
+
+from __future__ import annotations
+
+from typing import List, Optional
+
+from agentops.agent.config import LatencyCheckConfig
+from agentops.agent.findings import Category, Finding, Severity
+from agentops.agent.sources.azure_monitor import AzureMonitorPayload
+from agentops.agent.sources.results_history import ResultsHistory
+
+
+def run_latency_check(
+    history: ResultsHistory,
+    monitor: Optional[AzureMonitorPayload],
+    config: LatencyCheckConfig,
+) -> List[Finding]:
+    findings: List[Finding] = []
+    threshold = config.p95_threshold_seconds
+
+    if monitor and monitor.p95_duration_seconds is not None:
+        p95 = monitor.p95_duration_seconds
+        if p95 > threshold:
+            severity = (
+                Severity.CRITICAL if p95 >= threshold * 2 else Severity.WARNING
+            )
+            findings.append(
+                Finding(
+                    id="latency.p95_production",
+                    severity=severity,
+                    category=Category.PERFORMANCE,
+                    title="Production p95 latency exceeds threshold",
+                    summary=(
+                        f"Application Insights reports p95 latency of "
+                        f"{p95:.2f}s, above the configured threshold of "
+                        f"{threshold:.2f}s."
+                    ),
+                    recommendation=(
+                        "Review recent deployments for tool-call loops or "
+                        "long-running RAG retrievals, and consider scaling "
+                        "out the agent runtime."
+                    ),
+                    source="azure_monitor",
+                    evidence={
+                        "p95_seconds": p95,
+                        "threshold_seconds": threshold,
+                        "request_count": monitor.request_count,
+                    },
+                )
+            )
+
+    if history.runs:
+        latest = history.runs[-1]
+        avg_latency = latest.metrics.get("avg_latency_seconds")
+        if avg_latency is not None and avg_latency > threshold:
+            severity = (
+                Severity.CRITICAL
+                if avg_latency >= threshold * 2
+                else Severity.WARNING
+            )
+            findings.append(
+                Finding(
+                    id="latency.eval_avg",
+                    severity=severity,
+                    category=Category.PERFORMANCE,
+                    title="Evaluation average latency above threshold",
+                    summary=(
+                        f"Run `{latest.run_id}` averaged "
+                        f"{avg_latency:.2f}s per item, above the "
+                        f"{threshold:.2f}s threshold."
+                    ),
+                    recommendation=(
+                        "Profile the slowest dataset rows and inspect tool "
+                        "calls; re-run evals after addressing the regression."
+                    ),
+                    source="results_history",
+                    evidence={
+                        "run_id": latest.run_id,
+                        "avg_latency_seconds": avg_latency,
+                        "threshold_seconds": threshold,
+                    },
+                )
+            )
+    return findings

agentops/agent/checks/opex.py

@@ -0,0 +1,157 @@
+"""Operational excellence check.
+
+Pipeline-hygiene findings that are time-based or stability-based rather
+than file-based (which live in :mod:`agentops.agent.checks.mlops`).
+
+Findings emitted:
+
+* ``opex.stale_evaluation`` - Doctor warns when no fresh eval run has
+  landed in the configured window.
+* ``opex.flaky_metric`` - a metric's coefficient of variation across
+  recent runs is high enough to suggest a flaky judge / non-deterministic
+  prompt rather than real change.
+"""
+
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from statistics import mean, pstdev
+from typing import List
+
+from agentops.agent.config import OpexCheckConfig
+from agentops.agent.findings import Category, Finding, Severity
+from agentops.agent.sources.results_history import ResultsHistory
+
+SOURCE_NAME = "results_history"
+
+
+def run_opex_check(
+    history: ResultsHistory, config: OpexCheckConfig
+) -> List[Finding]:
+    """Detect operational-excellence regressions (stale runs + flaky metrics)."""
+    if not config.enabled:
+        return []
+
+    findings: List[Finding] = []
+    findings.extend(_check_stale_evaluation(history, config))
+    findings.extend(_check_flaky_metric(history, config))
+    return findings
+
+
+def _check_stale_evaluation(
+    history: ResultsHistory, config: OpexCheckConfig
+) -> List[Finding]:
+    if not history.runs:
+        return []
+    latest = history.runs[-1]
+    if latest.timestamp is None:
+        return []
+    now = datetime.now(timezone.utc)
+    age_days = (now - latest.timestamp).total_seconds() / 86400.0
+    if age_days <= config.stale_after_days:
+        return []
+
+    severity = (
+        Severity.CRITICAL
+        if age_days >= config.stale_after_days * 2
+        else Severity.WARNING
+    )
+    return [
+        Finding(
+            id="opex.stale_evaluation",
+            severity=severity,
+            category=Category.OPERATIONAL_EXCELLENCE,
+            title="No fresh evaluation run in the configured window",
+            summary=(
+                f"The most recent eval run (`{latest.run_id}`) is "
+                f"{age_days:.1f} day(s) old, above the configured "
+                f"threshold of {config.stale_after_days} day(s). The "
+                "agent's measured quality is drifting away from its "
+                "last validated baseline."
+            ),
+            recommendation=(
+                "Run `agentops eval run` (locally or via CI) to "
+                "produce a fresh local `results.json` or Foundry cloud "
+                "evaluation, then re-run `agentops doctor`."
+            ),
+            source=SOURCE_NAME,
+            evidence={
+                "latest_run_id": latest.run_id,
+                "latest_timestamp": latest.timestamp.isoformat(),
+                "age_days": round(age_days, 2),
+                "threshold_days": config.stale_after_days,
+            },
+        )
+    ]
+
+
+def _check_flaky_metric(
+    history: ResultsHistory, config: OpexCheckConfig
+) -> List[Finding]:
+    """Flag metrics whose coefficient of variation is suspiciously high.
+
+    A high CV (stddev / mean) across many runs without a corresponding
+    agent change is the fingerprint of a non-deterministic judge or a
+    prompt that's overly sensitive to phrasing. Real regressions show
+    up as monotonic drops (caught by the ``regression`` check); flaky
+    metrics oscillate.
+
+    We only consider metrics with at least ``min_runs_for_flaky`` data
+    points and a mean that's safely above zero to avoid amplifying noise
+    on near-zero scores.
+    """
+    runs = history.runs
+    if len(runs) < config.min_runs_for_flaky:
+        return []
+
+    # Collect each metric's series across the recent window.
+    series: dict[str, List[float]] = {}
+    for run in runs[-config.min_runs_for_flaky :]:
+        for name, value in run.metrics.items():
+            series.setdefault(name, []).append(value)
+
+    findings: List[Finding] = []
+    for metric, values in series.items():
+        if len(values) < config.min_runs_for_flaky:
+            continue
+        avg = mean(values)
+        if avg <= 0.05:
+            # Near-zero metrics make CV explode without signal.
+            continue
+        cv = pstdev(values) / avg
+        if cv < config.flaky_cv_threshold:
+            continue
+        findings.append(
+            Finding(
+                id=f"opex.flaky_metric.{metric}",
+                severity=Severity.WARNING,
+                category=Category.OPERATIONAL_EXCELLENCE,
+                title=f"`{metric}` is unstable across recent runs",
+                summary=(
+                    f"`{metric}` shows a coefficient of variation of "
+                    f"{cv * 100:.1f}% across the last {len(values)} "
+                    "runs (threshold: "
+                    f"{config.flaky_cv_threshold * 100:.0f}%). That "
+                    "kind of oscillation usually points at a "
+                    "non-deterministic judge model or a prompt that's "
+                    "overly sensitive to phrasing - not at real "
+                    "agent change."
+                ),
+                recommendation=(
+                    "Pin the judge model's `temperature` / `seed` "
+                    "(or switch to a deterministic evaluator), and "
+                    "review the metric's prompt for ambiguity. If "
+                    "the metric is intrinsically noisy, raise "
+                    "`min_runs` on the regression check so signals "
+                    "average out."
+                ),
+                source=SOURCE_NAME,
+                evidence={
+                    "metric": metric,
+                    "cv": round(cv, 4),
+                    "mean": round(avg, 4),
+                    "samples": len(values),
+                },
+            )
+        )
+    return findings