agentops-accelerator 0.3.0__py3-none-any.whl

agentops/agent/knowledge/waf-checklist.csv

@@ -0,0 +1,39 @@
+pillar,area,item_id,title,detection_source,detection_signal,doctor_check_id,status,reference_url
+Security,Identity,waf.security.local_auth_disabled,Disable local (key-based) auth on the AI account,azure_resources,account.disable_local_auth == true,waf.security.local_auth_disabled,implemented,https://learn.microsoft.com/azure/well-architected/ai/security
+Security,Identity,waf.security.managed_identity,Use managed identity for the AI account,azure_resources,"account.identity.type in {SystemAssigned, UserAssigned}",waf.security.managed_identity,implemented,https://learn.microsoft.com/azure/well-architected/ai/security
+Security,Telemetry,waf.security.diagnostic_settings,Diagnostic settings forward logs to a workspace,azure_resources,account has at least one diagnostic setting with a workspace_id,waf.security.diagnostic_settings,implemented,https://learn.microsoft.com/azure/well-architected/ai/security
+ResponsibleAI,ContentSafety,waf.rai.safety_metric_hit,Content-safety evaluator flagged a row in the latest eval,results_history,"row metric >= severity_floor on Violence/SelfHarm/Sexual/HateUnfairness",safety,implemented,https://learn.microsoft.com/azure/ai-foundry/concepts/evaluation-metrics-built-in
+ResponsibleAI,ContentSafety,waf.rai.runtime_content_filter,Content-filter triggers detected in production,azure_monitor,KQL hits on gen_ai.response.finish_reasons contains content_filter,safety.runtime.content_filter,implemented,https://learn.microsoft.com/azure/ai-foundry/concepts/content-filtering
+ResponsibleAI,ContinuousEval,waf.rai.continuous_eval_missing,Continuous evaluation rules attached to agents,foundry_control,foundry.evaluation_rules is empty while agents exist,safety.config.continuous_eval_missing,implemented,https://learn.microsoft.com/azure/ai-foundry/how-to/online-evaluation
+ResponsibleAI,ContinuousEval,waf.rai.continuous_eval_disabled,Continuous evaluation rules enabled,foundry_control,any evaluation_rule.enabled == false,safety.config.continuous_eval_disabled,implemented,https://learn.microsoft.com/azure/ai-foundry/how-to/online-evaluation
+OperationalExcellence,CI-CD,waf.opex.unpinned_agent,Agent target is pinned to a version,workspace_files,"agentops.yaml agent: lacks :version (or :latest)",opex.unpinned_agent,implemented,https://learn.microsoft.com/azure/well-architected/ai/operations
+OperationalExcellence,CI-CD,waf.opex.thresholds_defined,agentops.yaml declares explicit thresholds,workspace_files,agentops.yaml has no thresholds: block,opex.no_thresholds,implemented,https://learn.microsoft.com/azure/well-architected/ai/operations
+OperationalExcellence,CI-CD,waf.opex.pr_gate,Repository has an AgentOps PR gate,workspace_files,.github/workflows/agentops-pr.yml exists,opex.no_pr_gate,implemented,https://learn.microsoft.com/azure/well-architected/ai/operations
+OperationalExcellence,CI-CD,waf.opex.deploy_gate,Repository has AgentOps deploy workflows,workspace_files,at least one .github/workflows/agentops-deploy-*.yml exists,opex.no_deploy_workflow,implemented,https://learn.microsoft.com/azure/well-architected/ai/operations
+OperationalExcellence,Governance,waf.opex.results_gitignored,Eval results are not committed to git,workspace_files,.agentops/results/ entry in any reachable .gitignore,opex.results_not_gitignored,implemented,https://learn.microsoft.com/azure/well-architected/ai/operations
+OperationalExcellence,Governance,waf.opex.dataset_versioned,Dataset YAML files declare a version,workspace_files,.agentops/datasets/*.yaml has a top-level version: field,opex.unversioned_dataset,implemented,https://learn.microsoft.com/azure/well-architected/ai/operations
+OperationalExcellence,Telemetry,waf.opex.stale_evaluation,Evaluations are run regularly,results_history,latest run timestamp older than stale_after_days,opex.stale_evaluation,implemented,https://learn.microsoft.com/azure/well-architected/ai/operations
+Quality,Regression,waf.quality.metric_drop,No regression in evaluation metrics,results_history,latest_metric - baseline_metric > threshold_drop,regression,implemented,https://learn.microsoft.com/azure/ai-foundry/concepts/evaluation-approach-gen-ai
+Performance,Latency,waf.perf.latency_threshold,p95 and eval-average latency under threshold,azure_monitor + results_history,"p95_seconds > p95_threshold_seconds or avg_latency > threshold",latency,implemented,https://learn.microsoft.com/azure/well-architected/ai/performance-efficiency
+Reliability,Telemetry,waf.reliability.production_error_rate,Production error rate under threshold,azure_monitor,error_rate > rate_threshold,errors.production_rate,implemented,https://learn.microsoft.com/azure/well-architected/ai/reliability
+Reliability,Telemetry,waf.reliability.foundry_run_failures,Foundry agent runs not failing,foundry_control,foundry.failure_rate > rate_threshold,errors.foundry_runs,implemented,https://learn.microsoft.com/azure/well-architected/ai/reliability
+Reliability,Telemetry,waf.reliability.no_runtime_telemetry,Production telemetry is wired to the agent,azure_monitor,monitor.status == ok and request_count == 0 over lookback,errors.no_runtime_telemetry,implemented,https://learn.microsoft.com/azure/well-architected/ai/reliability
+OperationalExcellence,Governance,waf.opex.bundle_versioned,Bundle YAML files declare a version,workspace_files,.agentops/bundles/*.yaml has a top-level version: field,opex.unversioned_bundle,implemented,https://learn.microsoft.com/azure/well-architected/ai/operations
+OperationalExcellence,Retention,waf.opex.results_dir_bloat,Eval results directory has an archival policy,workspace_files,.agentops/results/ holds <= 50 run folders,opex.results_dir_bloat,implemented,https://learn.microsoft.com/azure/well-architected/ai/operations
+OperationalExcellence,Stability,waf.opex.flaky_metric,Eval metrics are stable across runs,results_history,coefficient of variation across last N runs < flaky_cv_threshold,opex.flaky_metric,implemented,https://learn.microsoft.com/azure/well-architected/ai/operations
+OperationalExcellence,Cost,waf.opex.workflow_concurrency,AgentOps workflows declare a concurrency block,workspace_files,agentops-pr.yml / agentops-deploy-*.yml has a top-level concurrency: block,opex.workflow_concurrency_lock,implemented,https://learn.microsoft.com/azure/well-architected/ai/operations
+OperationalExcellence,Reproducibility,waf.opex.workflow_action_pinning,AgentOps workflows pin actions by commit SHA,workspace_files,every uses: pins to a 40-char SHA,opex.workflow_action_sha_pinning,implemented,https://learn.microsoft.com/azure/well-architected/ai/operations
+OperationalExcellence,Foundry,waf.opex.foundry_control_configured,Foundry control plane source is wired,foundry_control,foundry_control.diagnostics.status == ok,opex.no_foundry_control_configured,implemented,https://learn.microsoft.com/azure/well-architected/ai/operations
+ResponsibleAI,Transparency,waf.rai.prompt_transparency,Agent system prompt declares AI nature and source citation,foundry_control,LLM judges agent instructions for AI-disclosure / source-citation / role-scope,responsible_ai.llm.prompt_transparency,implemented,https://learn.microsoft.com/azure/well-architected/ai/responsible-ai
+ResponsibleAI,SafetyGuardrails,waf.rai.prompt_safety_guardrails,Agent system prompt has explicit refusal patterns,foundry_control,LLM judges instructions for refusal guidance across the four harm categories,responsible_ai.llm.prompt_safety_guardrails,implemented,https://learn.microsoft.com/azure/well-architected/ai/responsible-ai
+ResponsibleAI,SafetyGuardrails,waf.rai.prompt_jailbreak_surface,Agent system prompt resists jailbreak / injection trapdoors,foundry_control,LLM scans instructions for override-phrasing / embedded secrets / unbounded role-play,responsible_ai.llm.prompt_jailbreak_surface,implemented,https://learn.microsoft.com/azure/well-architected/ai/responsible-ai
+ResponsibleAI,Privacy,waf.rai.dataset_pii_risk,Evaluation dataset is free of PII,workspace_files,LLM scans .agentops/data/*.jsonl sample for personal information,responsible_ai.llm.dataset_pii_risk,implemented,https://learn.microsoft.com/azure/well-architected/ai/responsible-ai
+ResponsibleAI,Fairness,waf.rai.dataset_bias_signals,Evaluation dataset covers diverse cohorts,workspace_files,LLM judges dataset sample for demographic / role / domain / tone skew,responsible_ai.llm.dataset_bias_signals,implemented,https://learn.microsoft.com/azure/well-architected/ai/responsible-ai
+OperationalExcellence,EvaluatorCoverage,waf.opex.llm_bundle_coverage,Bundle covers the evaluators the agent needs,workspace_files,LLM compares bundle YAML against agent description and recommends missing built-ins,opex.llm.bundle_coverage,implemented,https://learn.microsoft.com/azure/well-architected/ai/operations
+OperationalExcellence,Documentation,waf.opex.spec_present,Spec-driven scaffolding has spec content,workspace_files,detector hint paths present but no spec document found,opex.spec_conformance.spec_missing,implemented,https://learn.microsoft.com/azure/well-architected/ai/operations
+OperationalExcellence,Documentation,waf.opex.tasks_fresh,Spec tasks are kept fresh,workspace_files,tasks.md unchecked items older than stale_after_days,opex.spec_conformance.tasks_stale,implemented,https://learn.microsoft.com/azure/well-architected/ai/operations
+OperationalExcellence,Documentation,waf.opex.tasks_grounded,Completed spec tasks reference paths that exist,workspace_files,checked task references a path missing from the workspace,opex.spec_conformance.tasks_orphaned,implemented,https://learn.microsoft.com/azure/well-architected/ai/operations
+OperationalExcellence,Documentation,waf.opex.evaluator_alignment,Spec evaluators match bundle evaluators,workspace_files,evaluator mentioned in spec absent from every bundle YAML,opex.spec_conformance.evaluator_drift,implemented,https://learn.microsoft.com/azure/well-architected/ai/operations
+OperationalExcellence,Documentation,waf.opex.dataset_alignment,Spec datasets exist in workspace,workspace_files,dataset mentioned in spec absent from .agentops/datasets or .agentops/data,opex.spec_conformance.dataset_drift,implemented,https://learn.microsoft.com/azure/well-architected/ai/operations
+OperationalExcellence,Documentation,waf.opex.agent_alignment,Spec agent id matches run.yaml,workspace_files,spec mentions an agent_id that doesn't match run.yaml target.endpoint.agent_id,opex.spec_conformance.agent_drift,implemented,https://learn.microsoft.com/azure/well-architected/ai/operations
+OperationalExcellence,Documentation,waf.opex.llm_spec_implementation_gap,LLM judge cross-checks spec vs implementation,workspace_files,LLM judge compares spec capabilities to workspace fingerprint,opex.spec_conformance.llm.implementation_gap,implemented,https://learn.microsoft.com/azure/well-architected/ai/operations

agentops/agent/llm_assist/__init__.py

@@ -0,0 +1,16 @@
+"""LLM-judged Doctor checks.
+
+This package adds an opt-in layer of Doctor checks that invoke a judge
+model (via Foundry's OpenAI client) to evaluate semantic signals -
+prompt quality, dataset PII risk, bias, bundle coverage. See
+``docs/doctor-explained.md`` for the full rationale.
+
+Entry point is :func:`run_llm_assist_check`. Everything else here is
+implementation detail; do not import from sub-modules directly.
+"""
+
+from __future__ import annotations
+
+from agentops.agent.llm_assist._engine import run_llm_assist_check
+
+__all__ = ["run_llm_assist_check"]

agentops/agent/llm_assist/_base.py

@@ -0,0 +1,124 @@
+"""Base helpers for individual LLM-judged rules.
+
+Every rule shares the same shape: a focused system prompt, a Pydantic
+schema for the verdict, and a small builder that converts a verdict
+into a :class:`Finding`. This module factors out the duplicate code.
+"""
+
+from __future__ import annotations
+
+import hashlib
+from dataclasses import dataclass
+from typing import Any, Dict, List, Optional
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from agentops.agent.findings import Category, Finding, Severity
+from agentops.agent.llm_assist._client import JudgementMeta
+
+
+class BaseVerdict(BaseModel):
+    """Minimum schema every judge response must satisfy."""
+
+    model_config = ConfigDict(extra="allow")
+    risk: str = Field(description="Low, Medium, or High")
+    confidence: float = Field(ge=0.0, le=1.0)
+    reasoning: str
+    suggestions: List[str] = Field(
+        default_factory=list,
+        description=(
+            "Two to four concrete, actionable fixes the user can apply, "
+            "tailored to what the judge actually observed."
+        ),
+    )
+
+
+def hash_text(*chunks: str) -> str:
+    h = hashlib.sha256()
+    for chunk in chunks:
+        h.update(chunk.encode("utf-8", errors="replace"))
+        h.update(b"\0")
+    return h.hexdigest()[:16]
+
+
+def normalised_risk(verdict: BaseVerdict) -> str:
+    """Return verdict.risk lower-cased and bounded to {low, medium, high}."""
+    raw = (getattr(verdict, "risk", "") or "").strip().lower()
+    if raw in {"low", "medium", "high"}:
+        return raw
+    if raw in {"none", "ok", "clean"}:
+        return "low"
+    if raw in {"warning", "moderate"}:
+        return "medium"
+    if raw in {"critical", "severe"}:
+        return "high"
+    return "low"
+
+
+def severity_for(risk: str) -> Severity:
+    # LLM findings cap at WARNING by design.
+    return Severity.WARNING if risk in {"medium", "high"} else Severity.INFO
+
+
+@dataclass
+class FindingBuilderArgs:
+    rule_id: str
+    title: str
+    category: Category
+    summary_template: str
+    recommendation: str
+    verdict: BaseVerdict
+    meta: JudgementMeta
+    extra_evidence: Dict[str, Any]
+
+
+def build_llm_finding(args: FindingBuilderArgs) -> Optional[Finding]:
+    risk = normalised_risk(args.verdict)
+    if risk == "low":
+        return None
+    severity = severity_for(risk)
+
+    # If the judge produced concrete suggestions, splice them into the
+    # recommendation so the user sees actionable, case-specific fixes
+    # right next to the canonical guidance.
+    suggestions: List[str] = []
+    for raw in getattr(args.verdict, "suggestions", []) or []:
+        text = str(raw).strip()
+        if text:
+            suggestions.append(text)
+    recommendation = args.recommendation
+    if suggestions:
+        bullets = "\n".join(f"- {s}" for s in suggestions[:6])
+        recommendation = (
+            f"{args.recommendation}\n\n"
+            f"**Concrete fixes the judge model suggested for this "
+            f"specific case:**\n{bullets}"
+        )
+
+    evidence: Dict[str, Any] = {
+        "confidence": round(args.verdict.confidence, 3),
+        "reasoning": args.verdict.reasoning,
+        "model_deployment": args.meta.model_deployment,
+        "cache_hit": args.meta.cache_hit,
+        "risk": risk,
+    }
+    if suggestions:
+        evidence["suggestions"] = suggestions
+    evidence.update(args.extra_evidence)
+    if args.meta.input_tokens or args.meta.output_tokens:
+        evidence["tokens"] = {
+            "input": args.meta.input_tokens,
+            "output": args.meta.output_tokens,
+        }
+    return Finding(
+        id=args.rule_id,
+        severity=severity,
+        category=args.category,
+        title=f"[LLM-judged] {args.title}",
+        summary=args.summary_template.format(
+            risk=risk, reasoning=args.verdict.reasoning
+        ),
+        recommendation=recommendation,
+        source="llm_judge",
+        evidence=evidence,
+    )

agentops/agent/llm_assist/_bundle_rule.py

@@ -0,0 +1,154 @@
+"""LLM-judged Operational Excellence check: evaluator-bundle coverage.
+
+Reads the project's evaluator bundle YAML and a short agent description
+excerpt, then asks the judge model whether the bundle covers the
+evaluators a project of that shape typically needs (e.g. a RAG agent
+without ``GroundednessEvaluator``).
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import List, Optional
+
+import yaml
+
+from agentops.agent.findings import Category, Finding
+from agentops.agent.llm_assist._base import (
+    BaseVerdict,
+    FindingBuilderArgs,
+    build_llm_finding,
+    hash_text,
+)
+from agentops.agent.llm_assist._client import LLMJudge
+from agentops.agent.sources.foundry_control import FoundryAgentSummary
+
+
+_COVERAGE_SYSTEM = """You audit a project's evaluator bundle for the
+Microsoft Well-Architected Framework for AI Operational Excellence
+pillar. You receive:
+
+1. The bundle YAML (evaluators list + thresholds).
+2. The agent's name + a short instructions excerpt that hints at its
+   use case (RAG, conversational, tool-using, etc.).
+
+Decide which Foundry / azure-ai-evaluation built-in evaluators are
+notably missing for that use case. Examples:
+
+* RAG agent without GroundednessEvaluator or RetrievalEvaluator.
+* Tool-using agent without ToolCallAccuracyEvaluator.
+* Customer-support chat agent without CoherenceEvaluator.
+* Any agent serving end-users without content-safety evaluators
+  (Violence, SelfHarm, Sexual, HateUnfairness).
+
+Respond as compact JSON. Do NOT recommend custom evaluators; stick to
+Foundry / azure-ai-evaluation built-ins.
+
+{"risk": "low|medium|high", "confidence": <0.0-1.0>,
+ "reasoning": "<one short paragraph>",
+ "suggestions": ["<fix 1>", "<fix 2>", "<fix 3>"],
+ "missing_evaluators": ["GroundednessEvaluator", ...]}
+"""
+
+
+class CoverageVerdict(BaseVerdict):
+    missing_evaluators: List[str] = []
+
+
+def _load_bundle(workspace: Path) -> Optional[str]:
+    bundles = workspace / ".agentops" / "bundles"
+    if not bundles.is_dir():
+        return None
+    yamls = sorted(bundles.glob("*.yaml"))
+    if not yamls:
+        return None
+    bundle_path = yamls[0]
+    try:
+        text = bundle_path.read_text(encoding="utf-8")
+    except OSError:
+        return None
+    return text
+
+
+def _agent_excerpt(agents: List[FoundryAgentSummary]) -> Optional[str]:
+    for agent in agents:
+        if not agent.instructions:
+            continue
+        excerpt = agent.instructions.strip()
+        if len(excerpt) > 800:
+            excerpt = excerpt[:800] + "..."
+        return (
+            f"Agent name: {agent.name or agent.agent_id}\n"
+            f"Model: {agent.model or 'unknown'}\n\n"
+            f"Instructions excerpt:\n{excerpt}"
+        )
+    if agents:
+        a = agents[0]
+        return (
+            f"Agent name: {a.name or a.agent_id}\nModel: "
+            f"{a.model or 'unknown'}\n(instructions unavailable)"
+        )
+    return None
+
+
+def check_bundle_coverage(
+    judge: LLMJudge,
+    workspace: Path,
+    agents: List[FoundryAgentSummary],
+    min_confidence: float,
+) -> List[Finding]:
+    bundle_text = _load_bundle(workspace)
+    if bundle_text is None:
+        return []
+    agent_excerpt = _agent_excerpt(agents)
+    if agent_excerpt is None:
+        return []
+
+    # Sanity check: skip when the YAML is unparseable.
+    try:
+        yaml.safe_load(bundle_text)
+    except yaml.YAMLError:
+        return []
+
+    ih = hash_text("bundle_coverage", bundle_text, agent_excerpt)
+    result = judge.call(
+        system=_COVERAGE_SYSTEM,
+        user=(
+            "Bundle YAML:\n```yaml\n"
+            f"{bundle_text}\n```\n\n"
+            f"Agent context:\n{agent_excerpt}"
+        ),
+        schema=CoverageVerdict,
+        inputs_hash=ih,
+    )
+    if result is None:
+        return []
+    verdict, meta = result
+    if verdict.confidence < min_confidence:
+        return []
+    finding = build_llm_finding(
+        FindingBuilderArgs(
+            rule_id="opex.llm.bundle_coverage",
+            title="Evaluator bundle may be missing built-ins for this agent",
+            category=Category.OPERATIONAL_EXCELLENCE,
+            summary_template=(
+                "The judge model identified built-in evaluators that "
+                "fit this agent's use case but are not in the bundle "
+                "(risk={risk}): {reasoning}"
+            ),
+            recommendation=(
+                "Review the suggested evaluators in this finding's "
+                "evidence and add them to `.agentops/bundles/*.yaml` "
+                "if they fit. Use the canonical names from "
+                "`docs/foundry-evaluation-sdk-built-in-evaluators.md`."
+            ),
+            verdict=verdict,
+            meta=meta,
+            extra_evidence={
+                "missing_evaluators": getattr(
+                    verdict, "missing_evaluators", []
+                ),
+            },
+        )
+    )
+    return [finding] if finding is not None else []