agentops-accelerator 0.3.0__py3-none-any.whl

agentops/agent/checks/posture.py

@@ -0,0 +1,36 @@
+"""Posture check - runs the WAF-AI rule registry against the resource snapshot."""
+
+from __future__ import annotations
+
+from typing import List
+
+from agentops.agent.checks.posture_rules import RULE_REGISTRY
+from agentops.agent.config import PostureCheckConfig
+from agentops.agent.findings import Finding
+from agentops.agent.sources.azure_resources import AzureResourcesPayload
+
+SOURCE_NAME = "azure_resources"
+
+
+def run_posture_check(
+    resources: AzureResourcesPayload,
+    config: PostureCheckConfig,
+) -> List[Finding]:
+    if not config.enabled:
+        return []
+
+    diag = resources.diagnostics or {}
+    if diag.get("status") != "ok" or resources.account is None:
+        return []
+
+    excluded = {rid.strip() for rid in config.exclude_rules if rid and rid.strip()}
+
+    findings: List[Finding] = []
+    for rule_id, rule_fn in RULE_REGISTRY.items():
+        if rule_id in excluded:
+            continue
+        try:
+            findings.extend(rule_fn(resources, SOURCE_NAME))
+        except Exception:  # pragma: no cover - rules must be defensive
+            continue
+    return findings

agentops/agent/checks/posture_rules/__init__.py

@@ -0,0 +1,53 @@
+"""Rule registry for the WAF-AI posture check.
+
+Each rule is a small callable that receives the
+:class:`AzureResourcesPayload` and the source name, and returns a list
+of :class:`Finding`s (zero, one, or many). Rules are independent and
+pure.
+
+The ``posture`` check (see :mod:`agentops.agent.checks.posture`)
+iterates the rules registered here and aggregates the findings.
+
+To add a new rule:
+
+* Add a module under this package.
+* Implement ``def evaluate(payload, source_name) -> list[Finding]``.
+* Register it in :data:`RULE_REGISTRY` below.
+"""
+
+from __future__ import annotations
+
+from typing import Callable, Dict, List
+
+from agentops.agent.findings import Finding
+from agentops.agent.sources.azure_resources import AzureResourcesPayload
+
+RuleFn = Callable[[AzureResourcesPayload, str], List[Finding]]
+
+
+def _build_registry() -> Dict[str, RuleFn]:
+    # NOTE: two former rules (content_filter, network) were retired
+    # because Foundry's Operate -> Compliance surface now covers them
+    # natively (Guardrails tab + Security posture tab respectively).
+    # The watchdog's job in this area is the **complementary** half  - 
+    # runtime telemetry, identity scope, pipeline hygiene. The dropped
+    # rule modules are kept on disk so users with custom posture
+    # extensions importing them keep working.
+    from agentops.agent.checks.posture_rules.diagnostics import (
+        evaluate as diagnostics_rule,
+    )
+    from agentops.agent.checks.posture_rules.local_auth import (
+        evaluate as local_auth_rule,
+    )
+    from agentops.agent.checks.posture_rules.managed_identity import (
+        evaluate as managed_identity_rule,
+    )
+
+    return {
+        "waf.security.local_auth_disabled": local_auth_rule,
+        "waf.security.managed_identity": managed_identity_rule,
+        "waf.security.diagnostic_settings": diagnostics_rule,
+    }
+
+
+RULE_REGISTRY: Dict[str, RuleFn] = _build_registry()

agentops/agent/checks/posture_rules/content_filter.py

@@ -0,0 +1,59 @@
+"""WAF-AI Security: every model deployment needs a content filter (RAI policy).
+
+The WAF-AI Security pillar (Responsible AI subsection) requires that
+each Azure OpenAI / AI Foundry model deployment have a content filter
+applied. The default ``Microsoft.Default`` policy is acceptable; a
+deployment with no policy at all is not.
+
+This rule fires for **each** deployment that has no
+``rai_policy_name``.
+"""
+
+from __future__ import annotations
+
+from typing import List
+
+from agentops.agent.findings import Category, Finding, Severity
+from agentops.agent.sources.azure_resources import AzureResourcesPayload
+
+RULE_ID = "waf.security.content_filter"
+
+
+def evaluate(payload: AzureResourcesPayload, source_name: str) -> List[Finding]:
+    account = payload.account
+    if account is None or not payload.deployments:
+        return []
+
+    missing = [d for d in payload.deployments if not d.rai_policy_name]
+    if not missing:
+        return []
+
+    return [
+        Finding(
+            id=RULE_ID,
+            severity=Severity.CRITICAL,
+            category=Category.SECURITY,
+            title="One or more deployments have no content filter applied",
+            summary=(
+                f"{len(missing)} of {len(payload.deployments)} "
+                f"deployment(s) on account `{account.name}` have no "
+                "RAI / content-filter policy. The WAF-AI Security "
+                "pillar requires Responsible AI policies on every "
+                "model deployment."
+            ),
+            recommendation=(
+                "Apply a content-filter policy (start with "
+                "`Microsoft.Default`, then tune severity thresholds "
+                "for your workload) to every deployment listed below. "
+                "See https://learn.microsoft.com/azure/ai-services/openai/concepts/content-filter"
+            ),
+            source=source_name,
+            evidence={
+                "account": account.name,
+                "deployments_missing_filter": [
+                    {"name": d.name, "model": d.model} for d in missing
+                ],
+                "deployments_total": len(payload.deployments),
+            },
+        )
+    ]

agentops/agent/checks/posture_rules/diagnostics.py

@@ -0,0 +1,74 @@
+"""WAF-AI Security: diagnostic settings must be configured.
+
+Without diagnostic settings, audit logs and request traces from the
+Cognitive Services account never reach a Log Analytics workspace,
+storage account, or event hub - making incident investigation and
+content-safety auditing effectively impossible.
+
+The WAF-AI Security pillar recommends streaming diagnostic logs to
+Log Analytics for every AI account in production.
+
+This rule fires when **none** of the diagnostic settings on the
+account ship logs to a destination (workspace / storage / event hub).
+"""
+
+from __future__ import annotations
+
+from typing import List
+
+from agentops.agent.findings import Category, Finding, Severity
+from agentops.agent.sources.azure_resources import AzureResourcesPayload
+
+RULE_ID = "waf.security.diagnostic_settings"
+
+
+def evaluate(payload: AzureResourcesPayload, source_name: str) -> List[Finding]:
+    account = payload.account
+    if account is None:
+        return []
+
+    has_destination = any(
+        s.workspace_id or s.storage_account_id or s.event_hub_authorization_rule_id
+        for s in payload.diagnostic_settings
+    )
+    has_categories = any(s.enabled_log_categories for s in payload.diagnostic_settings)
+
+    if has_destination and has_categories:
+        return []
+
+    return [
+        Finding(
+            id=RULE_ID,
+            severity=Severity.WARNING,
+            category=Category.SECURITY,
+            title="Diagnostic settings are missing or incomplete",
+            summary=(
+                f"Cognitive Services account `{account.name}` has "
+                f"{len(payload.diagnostic_settings)} diagnostic "
+                "setting(s), but none route enabled log categories to "
+                "a Log Analytics workspace, storage account, or event "
+                "hub. Audit and content-safety logs are not being "
+                "captured."
+            ),
+            recommendation=(
+                "Create a diagnostic setting that ships the "
+                "`Audit`, `RequestResponse`, and `Trace` log categories "
+                "to a Log Analytics workspace. See "
+                "https://learn.microsoft.com/azure/ai-services/diagnostic-logging"
+            ),
+            source=source_name,
+            evidence={
+                "account": account.name,
+                "diagnostic_settings": [
+                    {
+                        "name": s.name,
+                        "workspace_id": s.workspace_id,
+                        "storage_account_id": s.storage_account_id,
+                        "event_hub_authorization_rule_id": s.event_hub_authorization_rule_id,
+                        "enabled_log_categories": s.enabled_log_categories,
+                    }
+                    for s in payload.diagnostic_settings
+                ],
+            },
+        )
+    ]

agentops/agent/checks/posture_rules/local_auth.py

@@ -0,0 +1,55 @@
+"""WAF-AI Security: local (key-based) authentication must be disabled.
+
+Cognitive Services / Azure OpenAI accounts ship with key-based auth
+enabled by default. Microsoft Entra ID is the recommended path for
+production AI workloads - keys can be exfiltrated, hard to rotate, and
+bypass conditional access policies.
+
+WAF-AI Security pillar reference:
+https://learn.microsoft.com/azure/ai-services/openai/how-to/managed-identity
+"""
+
+from __future__ import annotations
+
+from typing import List
+
+from agentops.agent.findings import Category, Finding, Severity
+from agentops.agent.sources.azure_resources import AzureResourcesPayload
+
+RULE_ID = "waf.security.local_auth_disabled"
+
+
+def evaluate(payload: AzureResourcesPayload, source_name: str) -> List[Finding]:
+    account = payload.account
+    if account is None:
+        return []
+    if account.disable_local_auth is True:
+        return []
+
+    return [
+        Finding(
+            id=RULE_ID,
+            severity=Severity.CRITICAL,
+            category=Category.SECURITY,
+            title="Local (API key) authentication is enabled",
+            summary=(
+                f"Cognitive Services account `{account.name}` has "
+                f"`disableLocalAuth={account.disable_local_auth}`. "
+                "Key-based authentication is enabled, which contradicts "
+                "the WAF-AI Security pillar guidance to use Microsoft "
+                "Entra ID exclusively."
+            ),
+            recommendation=(
+                "Set `disableLocalAuth: true` on the account, grant the "
+                "agent runtime the `Cognitive Services OpenAI User` "
+                "role via managed identity, and rotate any keys that "
+                "may have leaked. See "
+                "https://learn.microsoft.com/azure/ai-services/openai/how-to/managed-identity"
+            ),
+            source=source_name,
+            evidence={
+                "account": account.name,
+                "disable_local_auth": account.disable_local_auth,
+            },
+        )
+    ]

agentops/agent/checks/posture_rules/managed_identity.py

@@ -0,0 +1,59 @@
+"""WAF-AI Security: account must have a managed identity assigned.
+
+Cognitive Services / Azure OpenAI accounts call downstream Azure
+resources (Storage for fine-tuning data, Key Vault for customer keys,
+etc.). The WAF-AI Security pillar recommends using a managed identity
+for those calls instead of connection strings or keys.
+
+This rule fires when the account ``identity.type`` is missing or
+``None`` - i.e. neither system-assigned nor user-assigned managed
+identity is configured.
+"""
+
+from __future__ import annotations
+
+from typing import List
+
+from agentops.agent.findings import Category, Finding, Severity
+from agentops.agent.sources.azure_resources import AzureResourcesPayload
+
+RULE_ID = "waf.security.managed_identity"
+
+_NO_IDENTITY_VALUES = {"", "none", "null"}
+
+
+def evaluate(payload: AzureResourcesPayload, source_name: str) -> List[Finding]:
+    account = payload.account
+    if account is None:
+        return []
+    type_ = (account.identity_type or "").strip().lower()
+    if type_ and type_ not in _NO_IDENTITY_VALUES:
+        return []
+
+    return [
+        Finding(
+            id=RULE_ID,
+            severity=Severity.WARNING,
+            category=Category.SECURITY,
+            title="Account has no managed identity assigned",
+            summary=(
+                f"Cognitive Services account `{account.name}` has no "
+                "managed identity. The WAF-AI Security pillar "
+                "recommends assigning a system- or user-assigned MI so "
+                "downstream calls (Storage, Key Vault, Search) avoid "
+                "connection strings."
+            ),
+            recommendation=(
+                "Enable a system-assigned managed identity (or attach "
+                "a user-assigned one) on the account, and grant it the "
+                "minimum role it needs on each downstream resource. "
+                "See https://learn.microsoft.com/azure/ai-services/authentication"
+            ),
+            source=source_name,
+            evidence={
+                "account": account.name,
+                "identity_type": account.identity_type,
+                "user_assigned_identities": account.user_assigned_identities,
+            },
+        )
+    ]

agentops/agent/checks/posture_rules/network.py

@@ -0,0 +1,68 @@
+"""WAF-AI Security: restrict public network access to the AI account.
+
+Cognitive Services / Azure OpenAI accounts default to ``Enabled``
+public network access for convenience. For production AI workloads the
+WAF-AI Security pillar recommends restricting network access via
+private endpoints or a strict network ACL.
+
+This rule fires unless ONE of the following is true:
+
+* ``publicNetworkAccess == 'Disabled'``
+* At least one private endpoint connection is attached
+* Network ACLs default action is ``Deny``
+"""
+
+from __future__ import annotations
+
+from typing import List
+
+from agentops.agent.findings import Category, Finding, Severity
+from agentops.agent.sources.azure_resources import AzureResourcesPayload
+
+RULE_ID = "waf.security.public_network_access"
+
+
+def evaluate(payload: AzureResourcesPayload, source_name: str) -> List[Finding]:
+    account = payload.account
+    if account is None:
+        return []
+
+    pna = (account.public_network_access or "").lower()
+    has_private_endpoint = account.private_endpoint_count > 0
+    acl_default = (account.network_acls_default_action or "").lower()
+
+    if (
+        pna == "disabled"
+        or has_private_endpoint
+        or acl_default == "deny"
+    ):
+        return []
+
+    return [
+        Finding(
+            id=RULE_ID,
+            severity=Severity.WARNING,
+            category=Category.SECURITY,
+            title="Public network access is open and unrestricted",
+            summary=(
+                f"Cognitive Services account `{account.name}` allows "
+                "public network access without a deny-by-default ACL or "
+                "a private endpoint. The WAF-AI Security pillar "
+                "recommends restricting network access for production "
+                "AI workloads."
+            ),
+            recommendation=(
+                "Either set `publicNetworkAccess: Disabled` and attach "
+                "a private endpoint, or configure network ACLs with "
+                "`defaultAction: Deny` and an explicit allow list. See "
+                "https://learn.microsoft.com/azure/ai-services/cognitive-services-virtual-networks"
+            ),
+            source=source_name,
+            evidence={
+                "account": account.name,
+                "public_network_access": account.public_network_access,
+                "private_endpoint_count": account.private_endpoint_count,
+                "network_acls_default_action": account.network_acls_default_action,
+            },
+        )
+    ]

agentops/agent/checks/regression.py

@@ -0,0 +1,78 @@
+"""Regression check: detect metric drops vs a rolling baseline."""
+
+from __future__ import annotations
+
+from statistics import mean
+from typing import List
+
+from agentops.agent.config import RegressionCheckConfig
+from agentops.agent.findings import Category, Finding, Severity
+from agentops.agent.sources.results_history import ResultsHistory
+
+
+def run_regression_check(
+    history: ResultsHistory, config: RegressionCheckConfig
+) -> List[Finding]:
+    runs = history.runs
+    if len(runs) < config.min_runs:
+        return []
+
+    latest = runs[-1]
+    baseline_runs = runs[:-1]
+    if not baseline_runs:
+        return []
+
+    findings: List[Finding] = []
+    for metric in config.metrics:
+        baseline_values = [
+            r.metrics[metric] for r in baseline_runs if metric in r.metrics
+        ]
+        if not baseline_values:
+            continue
+        if metric not in latest.metrics:
+            continue
+
+        baseline = mean(baseline_values)
+        current = latest.metrics[metric]
+        if baseline <= 0:
+            continue
+
+        drop = (baseline - current) / baseline
+        if drop < config.threshold_drop:
+            continue
+
+        severity = (
+            Severity.CRITICAL
+            if drop >= max(config.threshold_drop * 2, 0.20)
+            else Severity.WARNING
+        )
+
+        findings.append(
+            Finding(
+                id=f"regression.{metric}",
+                severity=severity,
+                category=Category.QUALITY,
+                title=f"Regression detected on `{metric}`",
+                summary=(
+                    f"`{metric}` dropped {drop * 100:.1f}% in run "
+                    f"`{latest.run_id}` (current={current:.4f}, "
+                    f"baseline={baseline:.4f} over {len(baseline_values)} runs)."
+                ),
+                recommendation=(
+                    "Compare the latest run against the baseline runs in "
+                    "`.agentops/results/` or the Foundry Evaluations page, "
+                    "inspect prompt/model/dataset changes, and re-run the "
+                    "evaluation after the fix."
+                ),
+                source="results_history",
+                evidence={
+                    "metric": metric,
+                    "current": current,
+                    "baseline_avg": baseline,
+                    "drop_ratio": drop,
+                    "baseline_runs": len(baseline_values),
+                    "latest_run_id": latest.run_id,
+                },
+            )
+        )
+    return findings

agentops/agent/checks/release_readiness.py

@@ -0,0 +1,182 @@
+"""Production-readiness checks for the POC-to-production journey."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import List, Optional
+
+from agentops.agent.findings import Category, Finding, Severity
+from agentops.agent.sources.foundry_control import FoundryControlPayload
+from agentops.agent.sources.results_history import ResultsHistory
+
+SOURCE_NAME = "release_readiness"
+
+
+def run_release_readiness_check(
+    workspace: Path,
+    history: ResultsHistory,
+    foundry: Optional[FoundryControlPayload],
+) -> List[Finding]:
+    """Return findings that block or weaken production release evidence."""
+
+    if not _is_agentops_workspace(workspace, history):
+        return []
+
+    findings: List[Finding] = []
+    findings.extend(_check_latest_eval(history))
+    findings.extend(_check_baseline(workspace, history))
+    findings.extend(_check_trace_regression_dataset(workspace, history))
+    findings.extend(_check_foundry_online_evaluation(foundry))
+    return findings
+
+
+def _is_agentops_workspace(workspace: Path, history: ResultsHistory) -> bool:
+    return (
+        (workspace / "agentops.yaml").exists()
+        or bool(history.runs)
+        or (workspace / ".github" / "workflows" / "agentops-pr.yml").exists()
+        or (workspace / ".azuredevops" / "pipelines" / "agentops-pr.yml").exists()
+    )
+
+
+def _check_latest_eval(history: ResultsHistory) -> List[Finding]:
+    if not history.runs:
+        return [
+            Finding(
+                id="opex.release.no_eval_evidence",
+                severity=Severity.WARNING,
+                category=Category.OPERATIONAL_EXCELLENCE,
+                title="No evaluation evidence is available for release",
+                summary=(
+                    "AgentOps could not find a completed evaluation run in "
+                    "`.agentops/results/` or Foundry fallback history. A "
+                    "production promotion should have at least one recent eval "
+                    "result attached to the release evidence."
+                ),
+                recommendation=(
+                    "Run `agentops eval analyze`, fix any setup gaps, then run "
+                    "`agentops eval run` before promoting the agent."
+                ),
+                source=SOURCE_NAME,
+            )
+        ]
+
+    latest = history.runs[-1]
+    if latest.run_pass is False:
+        return [
+            Finding(
+                id="opex.release.latest_eval_failed",
+                severity=Severity.CRITICAL,
+                category=Category.OPERATIONAL_EXCELLENCE,
+                title="Latest evaluation run failed",
+                summary=(
+                    f"The latest eval run `{latest.run_id}` did not pass. "
+                    "A release with a failing quality gate should not be "
+                    "promoted to production."
+                ),
+                recommendation=(
+                    "Open the latest `report.md` or Foundry evaluation report, "
+                    "fix the failing rows or thresholds, and re-run the eval "
+                    "before generating release evidence again."
+                ),
+                source=SOURCE_NAME,
+                evidence={"run_id": latest.run_id, "run_pass": latest.run_pass},
+            )
+        ]
+    return []
+
+
+def _check_baseline(workspace: Path, history: ResultsHistory) -> List[Finding]:
+    if not history.runs:
+        return []
+    baseline = workspace / ".agentops" / "baseline" / "results.json"
+    if baseline.exists() or len(history.runs) >= 2:
+        return []
+    return [
+        Finding(
+            id="opex.release.no_baseline",
+            severity=Severity.WARNING,
+            category=Category.OPERATIONAL_EXCELLENCE,
+            title="No baseline result is available for regression gating",
+            summary=(
+                "AgentOps found an eval run, but no baseline or prior run to "
+                "compare against. The gate can say whether thresholds passed, "
+                "but not whether the candidate regressed from the last known "
+                "good behavior."
+            ),
+            recommendation=(
+                "After a known-good run, copy "
+                "`.agentops/results/latest/results.json` to "
+                "`.agentops/baseline/results.json` or keep historical runs so "
+                "`agentops eval run --baseline` can render deltas."
+            ),
+            source=SOURCE_NAME,
+        )
+    ]
+
+
+def _check_trace_regression_dataset(workspace: Path, history: ResultsHistory) -> List[Finding]:
+    if not history.runs:
+        return []
+    manifest = workspace / ".agentops" / "data" / "trace-regression-manifest.json"
+    if manifest.exists():
+        return []
+    return [
+        Finding(
+            id="opex.release.no_trace_regression_dataset",
+            severity=Severity.INFO,
+            category=Category.OPERATIONAL_EXCELLENCE,
+            title="Production traces are not feeding a regression dataset yet",
+            summary=(
+                "No trace-regression manifest was found under `.agentops/data/`. "
+                "This is acceptable for early exploration, but production "
+                "incidents and high-value conversations should become reviewed "
+                "regression rows over time."
+            ),
+            recommendation=(
+                "Export relevant App Insights / Foundry traces and run "
+                "`agentops eval promote-traces --source <traces.jsonl> --apply` "
+                "to create a reviewed production-derived regression dataset."
+            ),
+            source=SOURCE_NAME,
+            evidence={"manifest": str(manifest)},
+        )
+    ]
+
+
+def _check_foundry_online_evaluation(
+    foundry: Optional[FoundryControlPayload],
+) -> List[Finding]:
+    if foundry is None:
+        return []
+    diag = foundry.diagnostics or {}
+    if diag.get("status") != "ok":
+        return []
+    if "evaluation_rules_count" not in diag and "evaluation_rules_warning" not in diag:
+        return []
+    enabled = [rule for rule in foundry.evaluation_rules if rule.enabled is not False]
+    if enabled:
+        return []
+    return [
+        Finding(
+            id="opex.release.no_continuous_eval",
+            severity=Severity.WARNING,
+            category=Category.OPERATIONAL_EXCELLENCE,
+            title="No enabled Foundry continuous evaluation rule is attached",
+            summary=(
+                "The Foundry control plane was reachable, but AgentOps did not "
+                "detect an enabled continuous evaluation rule. Production "
+                "responses may not be sampled and scored after deployment."
+            ),
+            recommendation=(
+                "Enable Foundry continuous evaluation for the production agent "
+                "and include at least one safety or quality evaluator so runtime "
+                "traffic keeps producing quality evidence."
+            ),
+            source=SOURCE_NAME,
+            evidence={
+                "evaluation_rules_count": len(foundry.evaluation_rules),
+                "agents": [agent.agent_id for agent in foundry.agents],
+            },
+        )
+    ]