agentops-accelerator 0.3.0__py3-none-any.whl

agentops/agent/checks/safety.py

@@ -0,0 +1,247 @@
+"""Safety check: scans local evaluations, production telemetry, and
+Foundry control-plane configuration for Responsible-AI gaps.
+
+The check emits findings under :class:`Category.RESPONSIBLE_AI` from
+three independent layers:
+
+* **Eval** - content-safety metric hits in the latest eval run.
+  Backwards-compatible id ``safety.<metric>``; evidence carries
+  ``layer=eval``.
+* **Runtime** - content-filter triggers observed in App Insights /
+  Log Analytics within the lookback window. Id ``safety.runtime.content_filter``.
+* **Config** - Foundry continuous-evaluation rules missing or disabled.
+  Id ``safety.config.continuous_eval_missing`` /
+  ``safety.config.continuous_eval_disabled``.
+
+Each layer fails open: if its source did not produce a payload, that
+layer simply emits nothing.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional
+
+from agentops.agent.config import SafetyCheckConfig
+from agentops.agent.findings import Category, Finding, Severity
+from agentops.agent.sources.azure_monitor import AzureMonitorPayload
+from agentops.agent.sources.foundry_control import FoundryControlPayload
+from agentops.agent.sources.results_history import ResultsHistory
+
+_SEVERITY_RANK = {"low": 0, "medium": 1, "high": 2}
+
+_SAFETY_METRICS = (
+    "violence",
+    "self_harm",
+    "sexual",
+    "hate_unfairness",
+    "protected_material",
+)
+
+
+def _severity_value(raw: Any) -> int:
+    if isinstance(raw, str):
+        return _SEVERITY_RANK.get(raw.strip().lower(), -1)
+    if isinstance(raw, (int, float)):
+        return int(raw)
+    return -1
+
+
+def run_safety_check(
+    history: ResultsHistory,
+    config: SafetyCheckConfig,
+    monitor: Optional[AzureMonitorPayload] = None,
+    foundry: Optional[FoundryControlPayload] = None,
+) -> List[Finding]:
+    """Run all three safety layers and return the merged findings."""
+    findings: List[Finding] = []
+    findings.extend(_find_eval_safety(history, config))
+    if monitor is not None:
+        findings.extend(_find_runtime_safety(monitor, config))
+    if foundry is not None:
+        findings.extend(_find_config_safety(foundry))
+    return findings
+
+
+def _find_eval_safety(
+    history: ResultsHistory, config: SafetyCheckConfig
+) -> List[Finding]:
+    if not history.runs:
+        return []
+
+    floor = _SEVERITY_RANK.get(config.severity_floor.strip().lower(), 1)
+    latest = history.runs[-1]
+
+    hits: Dict[str, Dict[str, Any]] = {}
+    for item in latest.item_evaluations:
+        if not isinstance(item, dict):
+            continue
+        metrics = item.get("metrics") or item.get("scores") or {}
+        if not isinstance(metrics, dict):
+            continue
+        for key, value in metrics.items():
+            metric = str(key).lower()
+            if not any(m in metric for m in _SAFETY_METRICS):
+                continue
+            level = _severity_value(value)
+            if level < floor:
+                continue
+            existing = hits.get(metric)
+            if existing is None or level > existing.get("level", -1):
+                hits[metric] = {
+                    "level": level,
+                    "value": value,
+                    "row": item.get("input")
+                    or item.get("question")
+                    or item.get("id"),
+                }
+
+    findings: List[Finding] = []
+    for metric, info in hits.items():
+        severity = (
+            Severity.CRITICAL if info["level"] >= 2 else Severity.WARNING
+        )
+        findings.append(
+            Finding(
+                id=f"safety.{metric}",
+                severity=severity,
+                category=Category.RESPONSIBLE_AI,
+                title=f"Content-safety hit on `{metric}`",
+                summary=(
+                    f"Run `{latest.run_id}` produced a `{metric}` rating "
+                    f"of `{info['value']}` on at least one row."
+                ),
+                recommendation=(
+                    "Inspect the offending dataset row and the model "
+                    "response, tighten the system prompt or add a safety "
+                    "filter, and re-evaluate."
+                ),
+                source="results_history",
+                evidence={
+                    "layer": "eval",
+                    "metric": metric,
+                    "value": info["value"],
+                    "row": info.get("row"),
+                    "run_id": latest.run_id,
+                },
+            )
+        )
+    return findings
+
+
+def _find_runtime_safety(
+    monitor: AzureMonitorPayload, config: SafetyCheckConfig
+) -> List[Finding]:
+    findings: List[Finding] = []
+    for violation in monitor.safety_violations:
+        if not isinstance(violation, dict):
+            continue
+        hits = int(violation.get("hits", 0) or 0)
+        if hits < config.min_runtime_hits:
+            continue
+        signal = str(violation.get("signal") or "content_filter")
+        severity = (
+            Severity.CRITICAL
+            if hits >= config.runtime_critical_hits
+            else Severity.WARNING
+        )
+        findings.append(
+            Finding(
+                id=f"safety.runtime.{signal}",
+                severity=severity,
+                category=Category.RESPONSIBLE_AI,
+                title=f"Content-filter triggers detected in production (`{signal}`)",
+                summary=(
+                    f"App Insights observed {hits} `{signal}` event(s) "
+                    "over the lookback window. Each one is a response "
+                    "the model refused to complete or a request blocked "
+                    "by Azure AI Content Safety."
+                ),
+                recommendation=(
+                    "Inspect the underlying traces in Application "
+                    "Insights, identify whether the spike originates "
+                    "from a single client, a regression in the system "
+                    "prompt, or actual adversarial input, and adjust "
+                    "guardrails accordingly."
+                ),
+                source="azure_monitor",
+                evidence={
+                    "layer": "runtime",
+                    "signal": signal,
+                    "hits": hits,
+                },
+            )
+        )
+    return findings
+
+
+def _find_config_safety(foundry: FoundryControlPayload) -> List[Finding]:
+    if not foundry.agents:
+        return []
+
+    rules = foundry.evaluation_rules
+    diag = foundry.diagnostics or {}
+
+    # We only emit config findings if we were actually able to *probe*
+    # for rules (avoid false positives when the SDK lacks the surface).
+    if (
+        "evaluation_rules_count" not in diag
+        and "evaluation_rules_warning" not in diag
+    ):
+        return []
+
+    findings: List[Finding] = []
+
+    if not rules:
+        findings.append(
+            Finding(
+                id="safety.config.continuous_eval_missing",
+                severity=Severity.WARNING,
+                category=Category.RESPONSIBLE_AI,
+                title="No continuous evaluation rules configured",
+                summary=(
+                    f"Foundry project lists {len(foundry.agents)} agent(s) "
+                    "but no continuous-evaluation rules. Production "
+                    "responses are not being scored on quality / safety "
+                    "after deployment."
+                ),
+                recommendation=(
+                    "Attach continuous evaluation rules to your agents "
+                    "in Foundry (Operate -> Evaluations) so deployed "
+                    "responses are scored against safety and quality "
+                    "metrics in production."
+                ),
+                source="foundry_control",
+                evidence={
+                    "layer": "config",
+                    "agents": [a.agent_id for a in foundry.agents],
+                },
+            )
+        )
+        return findings
+
+    disabled = [r for r in rules if r.enabled is False]
+    if disabled:
+        findings.append(
+            Finding(
+                id="safety.config.continuous_eval_disabled",
+                severity=Severity.WARNING,
+                category=Category.RESPONSIBLE_AI,
+                title="One or more continuous evaluation rules are disabled",
+                summary=(
+                    f"{len(disabled)} of {len(rules)} continuous "
+                    "evaluation rule(s) are disabled. Production safety "
+                    "scoring is partially or fully turned off."
+                ),
+                recommendation=(
+                    "Re-enable the disabled rules in Foundry "
+                    "(Operate -> Evaluations) or remove them if they "
+                    "are intentionally retired."
+                ),
+                source="foundry_control",
+                evidence={
+                    "layer": "config",
+                    "disabled_rules": [r.rule_id for r in disabled],
+                },
+            )
+        )
+    return findings

agentops/agent/checks/spec_conformance.py

@@ -0,0 +1,375 @@
+"""Spec-conformance check (Operational Excellence pillar).
+
+Compares the project's spec-driven-development artifacts
+(``.specify/spec.md`` + ``plan.md`` + ``tasks.md``, ``AGENTS.md``,
+``.github/copilot-instructions.md``) against the AgentOps workspace
+(``run.yaml``, ``.agentops/bundles/``, ``.agentops/datasets/``)
+and flags drift between the two.
+
+All findings live under :class:`Category.OPERATIONAL_EXCELLENCE` with
+the ``opex.spec_conformance.*`` id prefix. Deterministic rules emit
+``info``/``warning`` only — never ``critical`` — because spec
+conformance is a soft signal.
+
+The companion opt-in LLM rule
+(``opex.spec_conformance.llm.implementation_gap``) lives in
+:mod:`agentops.agent.llm_assist._spec_rules`.
+"""
+
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Iterable, List, Optional
+
+import yaml
+
+from agentops.agent.config import SpecConformanceCheckConfig
+from agentops.agent.findings import Category, Finding, Severity
+from agentops.agent.sources.spec_detectors import (
+    DETECTORS,
+    Detector,
+    SpecDocument,
+)
+
+SOURCE_NAME = "spec_workspace"
+
+def run_spec_conformance_check(
+    workspace: Path,
+    config: SpecConformanceCheckConfig,
+) -> List[Finding]:
+    """Run all deterministic spec-conformance rules and return findings."""
+    if not config.enabled:
+        return []
+
+    selected = _select_detectors(config.detectors)
+    documents: List[SpecDocument] = []
+    hint_only: List[Path] = []
+    for det in selected:
+        doc = det.detect(workspace)
+        if doc is not None:
+            documents.append(doc)
+        else:
+            hint_only.extend(det.hint_paths(workspace))
+
+    findings: List[Finding] = []
+
+    if not documents:
+        if hint_only:
+            findings.append(
+                Finding(
+                    id="opex.spec_conformance.spec_missing",
+                    severity=Severity.WARNING,
+                    category=Category.OPERATIONAL_EXCELLENCE,
+                    title=(
+                        "Spec setup detected, but no usable specification was found"
+                    ),
+                    summary=(
+                        "Doctor found signs that this repo uses "
+                        "spec-driven development (for example "
+                        "`.specify/`, `AGENTS.md`, or a "
+                        "`copilot-instructions.md` shell), but could "
+                        "not load a real spec body. Without that "
+                        "reference, Doctor cannot check whether "
+                        "bundles, datasets, tasks, and "
+                        "implementation still match the intended agent "
+                        "behavior."
+                    ),
+                    recommendation=(
+                        "Add a readable spec such as `.specify/spec.md` "
+                        "(spec-kit) or `AGENTS.md` that describes the "
+                        "agent's intended behavior, capabilities, "
+                        "evaluators, and datasets, then re-run "
+                        "`agentops doctor`."
+                    ),
+                    source=SOURCE_NAME,
+                    evidence={"hint_paths": [str(p) for p in hint_only]},
+                )
+            )
+        return _filter_skipped(findings, config.skip)
+
+    for doc in documents:
+        findings.extend(_check_tasks(doc, config.stale_after_days))
+        findings.extend(_check_evaluator_drift(workspace, doc))
+        findings.extend(_check_dataset_drift(workspace, doc))
+        findings.extend(_check_agent_drift(workspace, doc))
+
+    deduped: List[Finding] = []
+    seen: set[tuple[str, str]] = set()
+    for f in findings:
+        key = (f.id, _evidence_key(f))
+        if key in seen:
+            continue
+        seen.add(key)
+        deduped.append(f)
+
+    return _filter_skipped(deduped, config.skip)
+
+
+def _select_detectors(names: Iterable[str]) -> List[Detector]:
+    requested = {n.strip().lower() for n in names if n and n.strip()}
+    if not requested:
+        return list(DETECTORS)
+    return [d for d in DETECTORS if d.name.lower() in requested]
+
+
+def _filter_skipped(findings: List[Finding], skip: Iterable[str]) -> List[Finding]:
+    skip_set = {s.strip() for s in skip if s and s.strip()}
+    if not skip_set:
+        return findings
+    return [f for f in findings if f.id not in skip_set]
+
+
+def _evidence_key(f: Finding) -> str:
+    """Stable key from a finding's evidence for de-duplication across detectors."""
+    if not isinstance(f.evidence, dict):
+        return ""
+    parts = []
+    for k in sorted(f.evidence):
+        v = f.evidence[k]
+        parts.append(f"{k}={v!r}")
+    return "|".join(parts)
+
+
+def _check_tasks(doc: SpecDocument, stale_after_days: int) -> List[Finding]:
+    findings: List[Finding] = []
+    if not doc.tasks:
+        return findings
+
+    now = datetime.now(timezone.utc)
+    last_modified = doc.last_modified
+    age_days = (
+        (now - last_modified).total_seconds() / 86400.0
+        if last_modified is not None
+        else None
+    )
+
+    unchecked = [t for t in doc.tasks if not t.checked]
+    if unchecked and age_days is not None and age_days > stale_after_days:
+        findings.append(
+            Finding(
+                id="opex.spec_conformance.tasks_stale",
+                severity=Severity.WARNING,
+                category=Category.OPERATIONAL_EXCELLENCE,
+                title="Spec tasks have been left open past the freshness window",
+                summary=(
+                    f"Doctor found {len(unchecked)} unchecked task(s) "
+                    "in the spec (for example `tasks.md` in a spec-kit "
+                    "workspace), and the spec has not been updated for "
+                    f"{age_days:.1f} day(s). The configured freshness "
+                    f"window is {stale_after_days} day(s). This usually "
+                    "means the implementation plan is no longer "
+                    "trustworthy: either the work is done but the tasks "
+                    "were not checked off, the tasks are no longer "
+                    "relevant, or the agent behavior changed without the "
+                    "spec being refreshed."
+                ),
+                recommendation=(
+                    "Review the open tasks. Check off completed work, "
+                    "remove tasks that no longer apply, or update the "
+                    "spec so the task list reflects the current agent "
+                    "behavior and evaluation plan."
+                ),
+                source=SOURCE_NAME,
+                evidence={
+                    "format": doc.format,
+                    "open_tasks": len(unchecked),
+                    "age_days": round(age_days, 2),
+                    "threshold_days": stale_after_days,
+                },
+            )
+        )
+
+    orphans: List[str] = []
+    for task in doc.tasks:
+        if not task.checked:
+            continue
+        for rel in task.mentioned_paths:
+            candidate = doc.root / rel
+            if not candidate.exists():
+                # Try resolving from the workspace root instead of the
+                # spec root (e.g. spec-kit lives under .specify/ but
+                # paths are workspace-relative).
+                if not (doc.root.parent / rel).exists():
+                    orphans.append(rel)
+
+    if orphans:
+        findings.append(
+            Finding(
+                id="opex.spec_conformance.tasks_orphaned",
+                severity=Severity.WARNING,
+                category=Category.OPERATIONAL_EXCELLENCE,
+                title="Completed tasks reference paths that don't exist",
+                summary=(
+                    "One or more checked task items in the spec point "
+                    "at files that aren't in the workspace. Either "
+                    "the implementation was removed or the spec is "
+                    "out of date."
+                ),
+                recommendation=(
+                    "Update the spec to reflect the current code "
+                    "layout, or restore the missing files."
+                ),
+                source=SOURCE_NAME,
+                evidence={
+                    "format": doc.format,
+                    "missing_paths": orphans[:10],
+                },
+            )
+        )
+
+    return findings
+
+
+def _check_evaluator_drift(workspace: Path, doc: SpecDocument) -> List[Finding]:
+    mentioned = doc.references.get("evaluators") or []
+    if not mentioned:
+        return []
+    declared = _collect_evaluator_names(workspace)
+    if not declared:
+        return []
+    missing = [e for e in mentioned if e not in declared]
+    if not missing:
+        return []
+    return [
+        Finding(
+            id="opex.spec_conformance.evaluator_drift",
+            severity=Severity.WARNING,
+            category=Category.OPERATIONAL_EXCELLENCE,
+            title="Spec names evaluators that no bundle declares",
+            summary=(
+                "The spec mentions evaluator classes that are absent "
+                "from every `.agentops/bundles/*.yaml`. The "
+                "implementation isn't measuring what the spec "
+                "promises."
+            ),
+            recommendation=(
+                "Either add the missing evaluator(s) to a bundle or "
+                "update the spec to reflect what the project actually "
+                "evaluates."
+            ),
+            source=SOURCE_NAME,
+            evidence={"missing_evaluators": missing[:10]},
+        )
+    ]
+
+
+def _check_dataset_drift(workspace: Path, doc: SpecDocument) -> List[Finding]:
+    mentioned = doc.references.get("datasets") or []
+    if not mentioned:
+        return []
+    available = {p.name for p in (workspace / ".agentops" / "datasets").glob("*.y*ml")}
+    available |= {p.name for p in (workspace / ".agentops" / "data").glob("*.jsonl")}
+    if not available:
+        return []
+    missing = [d for d in mentioned if Path(d).name not in available]
+    if not missing:
+        return []
+    return [
+        Finding(
+            id="opex.spec_conformance.dataset_drift",
+            severity=Severity.WARNING,
+            category=Category.OPERATIONAL_EXCELLENCE,
+            title="Spec references datasets that aren't in the workspace",
+            summary=(
+                "Dataset filenames mentioned in the spec do not "
+                "exist under `.agentops/datasets/` or "
+                "`.agentops/data/`."
+            ),
+            recommendation=(
+                "Add the missing dataset file(s) under "
+                "`.agentops/datasets/` (and the matching JSONL under "
+                "`.agentops/data/`), or update the spec."
+            ),
+            source=SOURCE_NAME,
+            evidence={"missing_datasets": missing[:10]},
+        )
+    ]
+
+
+def _check_agent_drift(workspace: Path, doc: SpecDocument) -> List[Finding]:
+    mentioned = doc.references.get("agent_ids") or []
+    if not mentioned:
+        return []
+    run_yaml = workspace / ".agentops" / "run.yaml"
+    if not run_yaml.exists():
+        run_yaml = workspace / "run.yaml"
+    if not run_yaml.exists():
+        return []
+    try:
+        raw = yaml.safe_load(run_yaml.read_text(encoding="utf-8"))
+    except (OSError, yaml.YAMLError):
+        return []
+    if not isinstance(raw, dict):
+        return []
+    target = raw.get("target") or {}
+    endpoint = target.get("endpoint") or {}
+    declared_agent = str(endpoint.get("agent_id") or "")
+    if not declared_agent:
+        return []
+    if declared_agent in mentioned:
+        return []
+    return [
+        Finding(
+            id="opex.spec_conformance.agent_drift",
+            severity=Severity.WARNING,
+            category=Category.OPERATIONAL_EXCELLENCE,
+            title="Spec's agent identifier doesn't match `run.yaml`",
+            summary=(
+                f"`run.yaml` targets agent `{declared_agent}` but the "
+                "spec mentions a different agent identifier. The "
+                "evaluation is running against a different agent "
+                "than the spec describes."
+            ),
+            recommendation=(
+                "Pin `run.yaml`'s `target.endpoint.agent_id` to the "
+                "agent named in the spec, or update the spec to "
+                "match."
+            ),
+            source=SOURCE_NAME,
+            evidence={
+                "spec_agent_ids": mentioned[:5],
+                "run_yaml_agent_id": declared_agent,
+            },
+        )
+    ]
+
+def _collect_evaluator_names(workspace: Path) -> set[str]:
+    """Read every bundle YAML and return the set of evaluator class names."""
+    out: set[str] = set()
+    bundles_dir = workspace / ".agentops" / "bundles"
+    if not bundles_dir.is_dir():
+        return out
+    for p in bundles_dir.glob("*.y*ml"):
+        try:
+            raw = yaml.safe_load(p.read_text(encoding="utf-8"))
+        except (OSError, yaml.YAMLError):
+            continue
+        if not isinstance(raw, dict):
+            continue
+        for ev in raw.get("evaluators") or []:
+            if isinstance(ev, dict):
+                name = ev.get("class") or ev.get("name")
+                if isinstance(name, str):
+                    out.add(name.strip())
+            elif isinstance(ev, str):
+                out.add(ev.strip())
+    return out
+
+
+def detect_documents(
+    workspace: Path,
+    config: Optional[SpecConformanceCheckConfig] = None,
+) -> List[SpecDocument]:
+    """Public helper: return all spec documents discovered in ``workspace``.
+
+    Used by the LLM rule to share detection with the deterministic
+    check without re-implementing the registry walk.
+    """
+    cfg = config or SpecConformanceCheckConfig()
+    out: List[SpecDocument] = []
+    for det in _select_detectors(cfg.detectors):
+        doc = det.detect(workspace)
+        if doc is not None:
+            out.append(doc)
+    return out