agentops-accelerator 0.3.0__py3-none-any.whl

agentops/agent/config.py

@@ -0,0 +1,240 @@
+"""Pydantic configuration model for the watchdog agent."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import List, Optional
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class ResultsHistorySourceConfig(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+    enabled: bool = True
+    path: str = ".agentops/results"
+    lookback_runs: int = Field(10, ge=2)
+
+
+class AzureMonitorSourceConfig(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+    enabled: bool = True
+    app_insights_resource_id: Optional[str] = None
+    log_analytics_workspace_id: Optional[str] = None
+
+
+class FoundryControlSourceConfig(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+    enabled: bool = True
+    project_endpoint: Optional[str] = None
+    project_endpoint_env: str = "AZURE_AI_FOUNDRY_PROJECT_ENDPOINT"
+    agent_ids: List[str] = Field(default_factory=list)
+
+
+class AzureResourcesSourceConfig(BaseModel):
+    """Read-only management-plane source for Azure resource posture audits.
+
+    Requires ``Reader`` (or stronger) RBAC on the resource group, and the
+    ``[agent]`` extra (which pulls in ``azure-mgmt-cognitiveservices`` and
+    ``azure-mgmt-monitor``).
+    """
+
+    model_config = ConfigDict(extra="forbid")
+    enabled: bool = True
+    subscription_id: Optional[str] = None
+    subscription_id_env: str = "AZURE_SUBSCRIPTION_ID"
+    resource_group: Optional[str] = None
+    cognitive_services_account: Optional[str] = None
+
+
+class SourcesConfig(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+    results_history: ResultsHistorySourceConfig = Field(
+        default_factory=ResultsHistorySourceConfig
+    )
+    azure_monitor: AzureMonitorSourceConfig = Field(
+        default_factory=AzureMonitorSourceConfig
+    )
+    foundry_control: FoundryControlSourceConfig = Field(
+        default_factory=FoundryControlSourceConfig
+    )
+    azure_resources: AzureResourcesSourceConfig = Field(
+        default_factory=AzureResourcesSourceConfig
+    )
+
+
+class RegressionCheckConfig(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+    metrics: List[str] = Field(
+        default_factory=lambda: [
+            "coherence",
+            "fluency",
+            "similarity",
+            "f1_score",
+            "groundedness",
+            "tool_call_accuracy",
+        ]
+    )
+    threshold_drop: float = Field(0.10, ge=0.0, le=1.0)
+    min_runs: int = Field(3, ge=2)
+
+
+class LatencyCheckConfig(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+    p95_threshold_seconds: float = Field(5.0, gt=0)
+
+
+class ErrorsCheckConfig(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+    rate_threshold: float = Field(0.05, ge=0.0, le=1.0)
+
+
+class SafetyCheckConfig(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+    severity_floor: str = "Medium"  # Low | Medium | High
+    min_runtime_hits: int = Field(1, ge=1)
+    runtime_critical_hits: int = Field(10, ge=1)
+
+
+class PostureCheckConfig(BaseModel):
+    """WAF-AI posture audit configuration.
+
+    The MVP rule set targets the **Security** pillar of the
+    Microsoft Well-Architected Framework for AI workloads.
+
+    The check is enabled by default. If the Azure resources source cannot
+    be discovered or read, it returns no findings and records an
+    actionable source diagnostic instead of failing the whole Doctor run.
+    """
+
+    model_config = ConfigDict(extra="forbid")
+    enabled: bool = True
+    pillar: str = "security"
+    exclude_rules: List[str] = Field(default_factory=list)
+
+
+class OpexCheckConfig(BaseModel):
+    """Operational-excellence (time-based) check configuration."""
+
+    model_config = ConfigDict(extra="forbid")
+    enabled: bool = True
+    stale_after_days: int = Field(14, ge=1)
+    min_runs_for_flaky: int = Field(5, ge=3)
+    flaky_cv_threshold: float = Field(0.30, gt=0.0, le=1.0)
+
+
+class LLMAssistCheckConfig(BaseModel):
+    """LLM-judged advisory checks.
+
+    Enabled by default - the Doctor auto-discovers a judge model from
+    the Foundry project on first use and reuses it on subsequent runs.
+    Set ``enabled: false`` to skip the suite entirely (e.g. in
+    ephemeral CI sandboxes that have no Foundry access).
+
+    The judge model is invoked via the Foundry project's OpenAI client.
+    No new credential flow.
+    """
+
+    model_config = ConfigDict(extra="forbid")
+    enabled: bool = True
+    deployment_name: Optional[str] = None
+    deployment_name_env: str = "AZURE_AI_MODEL_DEPLOYMENT_NAME"
+    project_endpoint_env: str = "AZURE_AI_FOUNDRY_PROJECT_ENDPOINT"
+    project_endpoint: Optional[str] = None
+    rules: List[str] = Field(default_factory=list)
+    max_dataset_rows: int = Field(50, ge=1, le=500)
+    min_confidence: float = Field(0.6, ge=0.0, le=1.0)
+    cache_ttl_days: int = Field(30, ge=0)
+
+
+class LLMSpecConformanceConfig(BaseModel):
+    """LLM gap-analysis sub-config for spec-conformance."""
+
+    model_config = ConfigDict(extra="forbid")
+    enabled: bool = False
+    severity_floor: float = Field(0.6, ge=0.0, le=1.0)
+    max_input_chars: int = Field(30_000, ge=1_000, le=200_000)
+    max_workspace_paths: int = Field(200, ge=10, le=2_000)
+
+
+class SpecConformanceCheckConfig(BaseModel):
+    """Spec-conformance sub-check under Operational Excellence.
+
+    The check inspects the workspace for spec-driven-development
+    artifacts (spec-kit ``.specify/``, ``AGENTS.md``, Copilot
+    instructions) and flags drift between the spec and the
+    implementation.
+    """
+
+    model_config = ConfigDict(extra="forbid")
+    enabled: bool = True
+    detectors: List[str] = Field(
+        default_factory=lambda: ["spec-kit", "agents-md"]
+    )
+    stale_after_days: int = Field(30, ge=1)
+    skip: List[str] = Field(default_factory=list)
+    llm_assist: LLMSpecConformanceConfig = Field(
+        default_factory=LLMSpecConformanceConfig
+    )
+
+
+class OperationalExcellenceCheckConfig(BaseModel):
+    """Container for Operational Excellence sub-checks."""
+
+    model_config = ConfigDict(extra="forbid")
+    spec_conformance: SpecConformanceCheckConfig = Field(
+        default_factory=SpecConformanceCheckConfig
+    )
+
+
+class ChecksConfig(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+    regression: RegressionCheckConfig = Field(default_factory=RegressionCheckConfig)
+    latency: LatencyCheckConfig = Field(default_factory=LatencyCheckConfig)
+    errors: ErrorsCheckConfig = Field(default_factory=ErrorsCheckConfig)
+    safety: SafetyCheckConfig = Field(default_factory=SafetyCheckConfig)
+    posture: PostureCheckConfig = Field(default_factory=PostureCheckConfig)
+    opex: OpexCheckConfig = Field(default_factory=OpexCheckConfig)
+    operational_excellence: OperationalExcellenceCheckConfig = Field(
+        default_factory=OperationalExcellenceCheckConfig
+    )
+    llm_assist: LLMAssistCheckConfig = Field(default_factory=LLMAssistCheckConfig)
+
+
+class ServerConfig(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+    github_app_client_id: Optional[str] = None
+
+
+class AgentConfig(BaseModel):
+    """Root config for ``.agentops/agent.yaml``."""
+
+    model_config = ConfigDict(extra="forbid")
+    version: int = 1
+    sources: SourcesConfig = Field(default_factory=SourcesConfig)
+    checks: ChecksConfig = Field(default_factory=ChecksConfig)
+    server: ServerConfig = Field(default_factory=ServerConfig)
+    lookback_days: int = Field(7, ge=1)
+
+
+def load_agent_config(path: Optional[Path]) -> AgentConfig:
+    """Load an :class:`AgentConfig` from a YAML file (or return defaults).
+
+    Legacy ``genaiops.*`` rule ids in ``checks.llm_assist.rules`` are
+    rewritten to their canonical ``opex.*`` equivalents with a one-shot
+    deprecation warning. See ``_legacy_ids.py`` for details.
+    """
+    if path is None or not path.exists():
+        return AgentConfig()
+
+    from agentops.utils.yaml import load_yaml
+
+    from agentops.agent._legacy_ids import canonicalize_id_list
+
+    raw = load_yaml(path)
+    if isinstance(raw, dict):
+        checks = raw.get("checks")
+        if isinstance(checks, dict):
+            llm = checks.get("llm_assist")
+            if isinstance(llm, dict) and isinstance(llm.get("rules"), list):
+                llm["rules"] = canonicalize_id_list(llm["rules"])
+    return AgentConfig.model_validate(raw)

agentops/agent/findings.py

@@ -0,0 +1,113 @@
+"""Severity-ranked findings produced by the watchdog agent."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Any, Dict
+
+
+class Category(str, Enum):
+    """High-level grouping for a finding.
+
+    Categories are stable user-facing buckets used for filtering and for
+    grouping the watchdog report. They are independent of severity:
+    a `quality` finding can be `critical`, `warning`, or `info`.
+
+    Categories mirror the **Microsoft Well-Architected Framework for AI**
+    pillars and are the single source of truth shared by doctor findings,
+    ``waf-checklist.csv`` (``pillar`` column), the cockpit rows, and
+    config skip-lists.
+
+    * ``quality``                - eval-driven signals (regression, content-safety)
+    * ``performance``            - latency / throughput signals
+    * ``reliability``            - error / failure signals
+    * ``operational_excellence`` - workspace + CI hygiene, Foundry config
+      audit, and spec-conformance signals (pinning, gates, drift,
+      versioning, spec-vs-implementation alignment)
+    * ``security``               - repo & identity surface beyond Foundry Compliance
+    * ``responsible_ai``         - prompt + eval-bundle heuristics for RAI practices
+    """
+
+    QUALITY = "quality"
+    PERFORMANCE = "performance"
+    RELIABILITY = "reliability"
+    OPERATIONAL_EXCELLENCE = "operational_excellence"
+    SECURITY = "security"
+    RESPONSIBLE_AI = "responsible_ai"
+
+
+class Severity(str, Enum):
+    """Severity level for a finding."""
+
+    INFO = "info"
+    WARNING = "warning"
+    CRITICAL = "critical"
+
+    @property
+    def rank(self) -> int:
+        return _SEVERITY_RANK[self]
+
+    def __lt__(self, other: object) -> bool:  # type: ignore[override]
+        if not isinstance(other, Severity):
+            return NotImplemented
+        return self.rank < other.rank
+
+    def __le__(self, other: object) -> bool:  # type: ignore[override]
+        if not isinstance(other, Severity):
+            return NotImplemented
+        return self.rank <= other.rank
+
+    def __gt__(self, other: object) -> bool:  # type: ignore[override]
+        if not isinstance(other, Severity):
+            return NotImplemented
+        return self.rank > other.rank
+
+    def __ge__(self, other: object) -> bool:  # type: ignore[override]
+        if not isinstance(other, Severity):
+            return NotImplemented
+        return self.rank >= other.rank
+
+
+_SEVERITY_RANK = {
+    Severity.INFO: 0,
+    Severity.WARNING: 1,
+    Severity.CRITICAL: 2,
+}
+
+
+_SEVERITY_EMOJI = {
+    Severity.INFO: "ℹ️",
+    Severity.WARNING: "⚠️",
+    Severity.CRITICAL: "🚨",
+}
+
+
+def severity_emoji(severity: Severity) -> str:
+    return _SEVERITY_EMOJI[severity]
+
+
+@dataclass
+class Finding:
+    """A single observation the watchdog agent surfaces."""
+
+    id: str
+    severity: Severity
+    title: str
+    summary: str
+    recommendation: str
+    source: str
+    category: Category = Category.QUALITY
+    evidence: Dict[str, Any] = field(default_factory=dict)
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "id": self.id,
+            "severity": self.severity.value,
+            "category": self.category.value,
+            "title": self.title,
+            "summary": self.summary,
+            "recommendation": self.recommendation,
+            "source": self.source,
+            "evidence": self.evidence,
+        }

agentops/agent/history.py

@@ -0,0 +1,142 @@
+"""Append-only analysis history for the watchdog agent.
+
+Each ``agentops doctor`` invocation appends one JSON record to
+``.agentops/agent/history.jsonl``. The file is the canonical local
+storage for the cockpit (``agentops cockpit``) and for any future
+trend-based checks. No Azure resource required.
+
+When OpenTelemetry tracing is configured, the same record is also
+emitted as a span (see :func:`agentops.utils.telemetry.agent_analyze_span`);
+the local JSONL remains authoritative because it works even when
+tracing is disabled.
+"""
+
+from __future__ import annotations
+
+import json
+from collections import Counter
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+from agentops.agent.findings import Category, Finding, Severity
+
+_HISTORY_REL_PATH = ".agentops/agent/history.jsonl"
+
+
+@dataclass
+class AnalysisRecord:
+    """A single watchdog analysis, captured for the cockpit and trend checks."""
+
+    timestamp: str
+    findings_total: int
+    findings_by_severity: Dict[str, int]
+    findings_by_category: Dict[str, int]
+    max_severity: Optional[str]
+    sources_enabled: List[str]
+    lookback_days: Optional[int]
+    duration_seconds: Optional[float]
+    findings: List[Dict[str, Any]] = field(default_factory=list)
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "timestamp": self.timestamp,
+            "findings_total": self.findings_total,
+            "findings_by_severity": self.findings_by_severity,
+            "findings_by_category": self.findings_by_category,
+            "max_severity": self.max_severity,
+            "sources_enabled": self.sources_enabled,
+            "lookback_days": self.lookback_days,
+            "duration_seconds": self.duration_seconds,
+            "findings": self.findings,
+        }
+
+    @classmethod
+    def from_dict(cls, payload: Dict[str, Any]) -> "AnalysisRecord":
+        return cls(
+            timestamp=str(payload.get("timestamp", "")),
+            findings_total=int(payload.get("findings_total", 0)),
+            findings_by_severity=dict(payload.get("findings_by_severity") or {}),
+            findings_by_category=dict(payload.get("findings_by_category") or {}),
+            max_severity=payload.get("max_severity"),
+            sources_enabled=list(payload.get("sources_enabled") or []),
+            lookback_days=payload.get("lookback_days"),
+            duration_seconds=payload.get("duration_seconds"),
+            findings=list(payload.get("findings") or []),
+        )
+
+
+def history_path(workspace: Path) -> Path:
+    """Return the absolute path to the analysis history file."""
+    return workspace / _HISTORY_REL_PATH
+
+
+def build_record(
+    findings: List[Finding],
+    *,
+    sources_enabled: List[str],
+    lookback_days: Optional[int],
+    duration_seconds: Optional[float],
+    timestamp: Optional[datetime] = None,
+) -> AnalysisRecord:
+    """Reduce a finding list into a serialisable :class:`AnalysisRecord`."""
+    now = timestamp or datetime.now(timezone.utc)
+    severity_counts = Counter(f.severity.value for f in findings)
+    category_counts = Counter(f.category.value for f in findings)
+    max_severity = max(findings, key=lambda f: f.severity.rank).severity.value if findings else None
+
+    return AnalysisRecord(
+        timestamp=now.isoformat(),
+        findings_total=len(findings),
+        findings_by_severity={s.value: severity_counts.get(s.value, 0) for s in Severity},
+        findings_by_category={c.value: category_counts.get(c.value, 0) for c in Category},
+        max_severity=max_severity,
+        sources_enabled=list(sources_enabled),
+        lookback_days=lookback_days,
+        duration_seconds=duration_seconds,
+        findings=[f.to_dict() for f in findings],
+    )
+
+
+def append_analysis(workspace: Path, record: AnalysisRecord) -> Path:
+    """Append a record to the workspace's history.jsonl. Returns the path."""
+    path = history_path(workspace)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("a", encoding="utf-8") as handle:
+        handle.write(json.dumps(record.to_dict(), ensure_ascii=False) + "\n")
+    return path
+
+
+def load_analysis_history(
+    workspace: Path,
+    *,
+    limit: Optional[int] = None,
+) -> List[AnalysisRecord]:
+    """Load all records (or the most recent ``limit``) from history.jsonl.
+
+    Returns an empty list when the file does not exist, so callers can
+    treat history as "best effort" without special-casing first runs.
+    Malformed lines are skipped silently rather than crashing the
+    cockpit or trend checks.
+    """
+    path = history_path(workspace)
+    if not path.exists():
+        return []
+
+    records: List[AnalysisRecord] = []
+    with path.open("r", encoding="utf-8") as handle:
+        for line in handle:
+            text = line.strip()
+            if not text:
+                continue
+            try:
+                payload = json.loads(text)
+            except ValueError:
+                continue
+            if isinstance(payload, dict):
+                records.append(AnalysisRecord.from_dict(payload))
+
+    if limit is not None and limit > 0:
+        records = records[-limit:]
+    return records

agentops/agent/knowledge/__init__.py

@@ -0,0 +1,182 @@
+"""WAF AI Landing Zones knowledge base for the Doctor agent.
+
+This package ships a CSV (`waf-checklist.csv`) that maps every Doctor
+finding id to the Microsoft Well-Architected Framework (WAF) for AI
+workloads pillar/area it belongs to.
+
+The shipped CSV is the **packaged baseline**. Users can override or
+extend it on a per-workspace basis by dropping a file at
+``.agentops/waf-checklist.csv`` (created by ``agentops init``). The
+loader merges the two: rows in the workspace file override packaged
+rows when ``doctor_check_id`` matches, and add new rows when the id
+is new. This keeps the toolkit's baseline updateable while letting
+teams version their own rules alongside the project.
+
+Strict scope rule: a row only exists if Doctor itself can produce
+that finding. Items that are exclusively visible through Foundry
+Operate -> Compliance (public network, private endpoints,
+content-filter attachment, custom subdomain) are intentionally *not*
+in the packaged CSV - they belong to Foundry's surface, not the
+Doctor's.
+
+The CSV's ``doctor_check_id`` column may carry either a fully
+qualified finding id (``opex.no_pr_gate``) or a prefix
+(``regression`` covers ``regression.coherence``,
+``regression.fluency``, ...). Lookups in :func:`find_waf_item` walk
+the dot segments from longest to shortest so the most specific row
+wins.
+"""
+
+from __future__ import annotations
+
+import csv
+import logging
+from dataclasses import dataclass
+from functools import lru_cache
+from importlib import resources
+from pathlib import Path
+from typing import Dict, List, Optional
+
+log = logging.getLogger(__name__)
+
+_CSV_NAME = "waf-checklist.csv"
+WORKSPACE_OVERRIDE_PATH = Path(".agentops") / "waf-checklist.csv"
+_EXPECTED_COLUMNS = (
+    "pillar",
+    "area",
+    "item_id",
+    "title",
+    "detection_source",
+    "detection_signal",
+    "doctor_check_id",
+    "status",
+    "reference_url",
+)
+
+
+@dataclass(frozen=True)
+class WAFItem:
+    """One row of the WAF checklist."""
+
+    pillar: str
+    area: str
+    item_id: str
+    title: str
+    detection_source: str
+    detection_signal: str
+    doctor_check_id: str
+    status: str
+    reference_url: str
+
+
+def _row_to_item(row: Dict[str, str]) -> Optional[WAFItem]:
+    check_id = (row.get("doctor_check_id") or "").strip()
+    if not check_id:
+        return None
+    return WAFItem(
+        pillar=(row.get("pillar") or "").strip(),
+        area=(row.get("area") or "").strip(),
+        item_id=(row.get("item_id") or "").strip(),
+        title=(row.get("title") or "").strip(),
+        detection_source=(row.get("detection_source") or "").strip(),
+        detection_signal=(row.get("detection_signal") or "").strip(),
+        doctor_check_id=check_id,
+        status=(row.get("status") or "").strip(),
+        reference_url=(row.get("reference_url") or "").strip(),
+    )
+
+
+def _parse_csv_text(label: str, text: str) -> List[WAFItem]:
+    reader = csv.DictReader(text.splitlines())
+    missing = [c for c in _EXPECTED_COLUMNS if c not in (reader.fieldnames or [])]
+    if missing:
+        log.warning("WAF checklist %s is missing columns: %s", label, missing)
+        return []
+    items: List[WAFItem] = []
+    for row in reader:
+        item = _row_to_item(row)
+        if item is not None:
+            items.append(item)
+    return items
+
+
+@lru_cache(maxsize=1)
+def _packaged_items() -> List[WAFItem]:
+    """Load the packaged baseline (cached - ships in the wheel)."""
+    try:
+        text = (
+            resources.files(__name__).joinpath(_CSV_NAME).read_text(encoding="utf-8")
+        )
+    except FileNotFoundError:
+        log.warning("Packaged WAF checklist not found")
+        return []
+    except OSError as exc:
+        log.warning("Packaged WAF checklist could not be read: %s", exc)
+        return []
+    return _parse_csv_text("packaged", text)
+
+
+def _workspace_items(workspace: Path) -> List[WAFItem]:
+    """Load the workspace override file. NOT cached - users edit it in place."""
+    override = workspace / WORKSPACE_OVERRIDE_PATH
+    if not override.is_file():
+        return []
+    try:
+        text = override.read_text(encoding="utf-8")
+    except OSError as exc:
+        log.warning("Workspace WAF checklist at %s unreadable: %s", override, exc)
+        return []
+    # Strip comment lines (#-prefixed) before parsing as CSV.
+    cleaned = "\n".join(
+        line for line in text.splitlines() if not line.lstrip().startswith("#")
+    )
+    return _parse_csv_text(f"workspace ({override})", cleaned)
+
+
+def load_waf_checklist(workspace: Optional[Path] = None) -> List[WAFItem]:
+    """Load the merged WAF checklist (packaged + optional workspace override).
+
+    Workspace rows override packaged rows by ``doctor_check_id`` and
+    add new ids that the packaged file does not carry. The packaged
+    baseline alone is returned when ``workspace`` is ``None`` or has
+    no override file.
+    """
+    items_by_id: Dict[str, WAFItem] = {
+        item.doctor_check_id: item for item in _packaged_items()
+    }
+    if workspace is not None:
+        for item in _workspace_items(workspace):
+            items_by_id[item.doctor_check_id] = item
+    return list(items_by_id.values())
+
+
+def waf_index_by_check_id(
+    workspace: Optional[Path] = None,
+) -> Dict[str, WAFItem]:
+    """Return a `{doctor_check_id: WAFItem}` map for quick reporter lookup."""
+    return {item.doctor_check_id: item for item in load_waf_checklist(workspace)}
+
+
+def find_waf_item(
+    finding_id: str, workspace: Optional[Path] = None
+) -> Optional[WAFItem]:
+    """Return the WAF row matching a finding id (most specific prefix wins).
+
+    Walks the dot segments from longest to shortest. For
+    ``safety.runtime.content_filter`` we try the full id first, then
+    ``safety.runtime``, then ``safety``. The first match is returned.
+
+    When ``workspace`` is provided, workspace-level overrides take
+    precedence (see :func:`load_waf_checklist`).
+    """
+    if not finding_id:
+        return None
+    index = waf_index_by_check_id(workspace)
+    parts = finding_id.split(".")
+    while parts:
+        candidate = ".".join(parts)
+        item = index.get(candidate)
+        if item is not None:
+            return item
+        parts.pop()
+    return None