agent-control-evaluator-cisco 7.7.0__py3-none-any.whl

agent_control_evaluator_cisco/__init__.py

@@ -0,0 +1,2 @@
+__all__ = []
+

agent_control_evaluator_cisco/ai_defense/__init__.py

@@ -0,0 +1,5 @@
+from .config import CiscoAIDefenseConfig
+from .evaluator import CiscoAIDefenseEvaluator
+
+__all__ = ["CiscoAIDefenseEvaluator", "CiscoAIDefenseConfig"]
+

agent_control_evaluator_cisco/ai_defense/client.py

@@ -0,0 +1,101 @@
+from __future__ import annotations
+
+# Thin REST client for Cisco AI Defense Chat Inspection.
+# Uses httpx.AsyncClient and the OpenAPI-defined endpoint/header.
+from dataclasses import dataclass, field
+from typing import Any
+
+try:
+    import httpx
+
+    AI_DEFENSE_HTTPX_AVAILABLE = True
+except ImportError:  # Narrow to import error only
+    httpx = None  # type: ignore
+    AI_DEFENSE_HTTPX_AVAILABLE = False
+
+
+# Regions from ai_defense_api.json "servers" section
+REGION_BASE_URLS: dict[str, str] = {
+    "us": "https://us.api.inspect.aidefense.security.cisco.com",
+    "ap": "https://ap.api.inspect.aidefense.security.cisco.com",
+    "eu": "https://eu.api.inspect.aidefense.security.cisco.com",
+}
+
+
+def build_endpoint(base_url: str) -> str:
+    base = base_url.rstrip("/")
+    return f"{base}/api/v1/inspect/chat"
+
+
+@dataclass
+class AIDefenseClient:
+    """Minimal async client for Cisco AI Defense Chat Inspection.
+
+    Attributes:
+        api_key: API key used for authentication header
+        endpoint_url: Full URL to POST /api/v1/inspect/chat
+        timeout_s: Timeout in seconds
+    """
+
+    api_key: str = field(repr=False)
+    endpoint_url: str
+    timeout_s: float
+
+    _client: httpx.AsyncClient | None = field(  # type: ignore[name-defined]
+        default=None,
+        repr=False,
+        compare=False,
+    )
+
+    async def _get_client(self) -> httpx.AsyncClient:  # type: ignore[name-defined]
+        if not AI_DEFENSE_HTTPX_AVAILABLE:  # pragma: no cover
+            raise RuntimeError("httpx not installed; cannot call Cisco AI Defense REST API")
+        if self._client is None or self._client.is_closed:
+            self._client = httpx.AsyncClient(timeout=self.timeout_s)
+        return self._client
+
+    async def chat_inspect(
+        self,
+        messages: list[dict[str, str]],
+        metadata: dict[str, Any] | None = None,
+        inspect_config: dict[str, Any] | None = None,
+        headers: dict[str, str] | None = None,
+    ) -> dict[str, Any]:
+        client = await self._get_client()
+
+        req_headers: dict[str, str] = {
+            "X-Cisco-AI-Defense-API-Key": self.api_key,
+            "Content-Type": "application/json",
+            "Accept": "application/json",
+        }
+        if headers:
+            req_headers.update(headers)
+
+        payload: dict[str, Any] = {"messages": messages}
+        if metadata is not None:
+            payload["metadata"] = metadata
+        if inspect_config is not None:
+            payload["config"] = inspect_config
+
+        resp = await client.post(self.endpoint_url, json=payload, headers=req_headers)
+        resp.raise_for_status()
+        data = resp.json()
+        if not isinstance(data, dict):
+            raise RuntimeError("Invalid response payload: not a JSON object")
+        return data
+
+    async def aclose(self) -> None:
+        if self._client and not self._client.is_closed:
+            await self._client.aclose()
+
+    async def close(self) -> None:
+        """Close the HTTP client and release resources."""
+        await self.aclose()
+
+    async def __aenter__(self) -> AIDefenseClient:
+        """Async context manager entry."""
+        return self
+
+    async def __aexit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:
+        """Async context manager exit."""
+        await self.close()

agent_control_evaluator_cisco/ai_defense/config.py

@@ -0,0 +1,33 @@
+from __future__ import annotations
+
+from typing import Any, Literal
+
+from agent_control_evaluators import EvaluatorConfig
+from pydantic import Field
+
+
+class CiscoAIDefenseConfig(EvaluatorConfig):
+    """Configuration for Cisco AI Defense evaluator (REST).
+
+    Attributes:
+        api_key_env: Env var name for API key
+        region: Optional server region (us, ap, eu); ignored if api_url set
+        api_url: Optional full endpoint override
+        timeout_ms: Request timeout (milliseconds)
+        on_error: Error policy (allow=fail-open, deny=fail-closed)
+        payload_field: Force single-message role: input→user, output→assistant
+        messages_strategy: "single" (synthesize) or "history" (pass-through messages)
+        metadata: Optional metadata object to include (OpenAPI spec)
+        inspect_config: Optional Inspect API config passthrough (see OpenAPI spec)
+    """
+
+    api_key_env: str = "AI_DEFENSE_API_KEY"
+    region: Literal["us", "ap", "eu"] | None = "us"
+    api_url: str | None = None
+    timeout_ms: int = Field(default=15_000, ge=1)
+    on_error: Literal["allow", "deny"] = "allow"
+    payload_field: Literal["input", "output"] | None = None
+    messages_strategy: Literal["single", "history"] = "history"
+    metadata: dict[str, Any] | None = None
+    inspect_config: dict[str, Any] | None = None
+    include_raw_response: bool = False

agent_control_evaluator_cisco/ai_defense/evaluator.py

@@ -0,0 +1,229 @@
+from __future__ import annotations
+
+import json
+import os
+from importlib.metadata import PackageNotFoundError, version
+from typing import Any
+
+from agent_control_evaluators import (
+    Evaluator,
+    EvaluatorMetadata,
+    register_evaluator,
+)
+from agent_control_models import EvaluatorResult
+
+from .client import AI_DEFENSE_HTTPX_AVAILABLE, REGION_BASE_URLS, AIDefenseClient, build_endpoint
+from .config import CiscoAIDefenseConfig
+
+
+def _resolve_package_version() -> str:
+    """Return the installed package version, or a dev fallback during local imports."""
+    try:
+        return version("agent-control-evaluator-cisco")
+    except PackageNotFoundError:
+        return "0.0.0.dev"
+
+
+_PACKAGE_VERSION = _resolve_package_version()
+
+
+def _load_api_key(env_name: str) -> str:
+    key = os.getenv(env_name)
+    if not key:
+        raise RuntimeError(
+            f"Missing Cisco AI Defense API key in env '{env_name}'. Set it on the server."
+        )
+    return key
+
+
+def _build_messages(
+    data: Any,
+    strategy: str,
+    payload_field: str | None,
+) -> list[dict[str, str]]:
+    """Build Chat Inspection messages from selected data.
+
+    - history: pass-through if data has 'messages' list; else fallback to single
+    - single: synthesize one message with role based on payload_field
+    """
+    if strategy == "history":
+        if isinstance(data, dict) and isinstance(data.get("messages"), list):
+            msgs: list[dict[str, str]] = []
+            for m in data["messages"]:
+                if isinstance(m, dict) and "content" in m:
+                    role = str(m.get("role", "user"))
+                    content = str(m.get("content", ""))
+                    msgs.append({"role": role, "content": content})
+            if msgs:
+                return msgs
+        # Fallback to single
+
+    role = "assistant" if payload_field == "output" else "user"
+    content = _coerce_message_content(data, payload_field)
+    return [{"role": role, "content": content}]
+
+
+def _coerce_message_content(data: Any, payload_field: str | None) -> str:
+    if data is None:
+        return ""
+
+    if isinstance(data, dict):
+        candidate: Any = None
+        preferred_keys: list[str] = []
+        if payload_field is not None:
+            preferred_keys.append(payload_field)
+        if payload_field == "output":
+            preferred_keys.extend(["output", "content", "text", "message"])
+        else:
+            preferred_keys.extend(["input", "content", "text", "message"])
+
+        for key in preferred_keys:
+            if key in data and data[key] is not None:
+                candidate = data[key]
+                break
+
+        if candidate is None:
+            candidate = data
+        return _stringify_message_content(candidate)
+
+    return _stringify_message_content(data)
+
+
+def _stringify_message_content(value: Any) -> str:
+    if value is None:
+        return ""
+    if isinstance(value, str):
+        return value
+    if isinstance(value, (int, float, bool)):
+        return str(value)
+    try:
+        return json.dumps(value, ensure_ascii=False, sort_keys=True, default=str)
+    except TypeError:
+        return str(value)
+
+
+@register_evaluator
+class CiscoAIDefenseEvaluator(Evaluator[CiscoAIDefenseConfig]):
+    """Cisco AI Defense evaluator.
+
+    Maps InspectResponse.is_safe to EvaluatorResult.matched.
+    """
+
+    metadata = EvaluatorMetadata(
+        name="cisco.ai_defense",
+        version=_PACKAGE_VERSION,
+        description="Cisco AI Defense Chat Inspection integration",
+        requires_api_key=True,
+        timeout_ms=15000,
+    )
+
+    config_model = CiscoAIDefenseConfig
+
+    @classmethod
+    def is_available(cls) -> bool:
+        """Evaluator is available only if httpx dependency exists."""
+        return AI_DEFENSE_HTTPX_AVAILABLE
+
+    def __init__(self, config: CiscoAIDefenseConfig) -> None:
+        self.config = config
+
+        # Validate and resolve configuration eagerly to avoid per-call work.
+        # API key
+        try:
+            api_key = _load_api_key(self.config.api_key_env)
+        except RuntimeError as e:
+            # Fail fast during construction so misconfiguration is caught early.
+            raise ValueError(str(e)) from e
+
+        # Endpoint
+        if self.config.api_url:
+            endpoint_url = self.config.api_url
+        else:
+            base_url = REGION_BASE_URLS.get(self.config.region or "us", REGION_BASE_URLS["us"])
+            endpoint_url = build_endpoint(base_url)
+
+        # Timeout
+        timeout_s = float(self.config.timeout_ms) / 1000.0
+
+        # Create a single client instance for reuse.
+        self._client: AIDefenseClient = AIDefenseClient(
+            api_key=api_key,
+            endpoint_url=endpoint_url,
+            timeout_s=timeout_s,
+        )
+
+    async def evaluate(self, data: Any) -> EvaluatorResult:  # noqa: D401
+        # Null input: do not call external service; treat as no data
+        if data is None:
+            return EvaluatorResult(matched=False, confidence=1.0, message="No data")
+
+        messages = _build_messages(
+            data,
+            strategy=self.config.messages_strategy,
+            payload_field=self.config.payload_field,
+        )
+        if not messages:
+            return EvaluatorResult(matched=False, confidence=1.0, message="No data to inspect")
+
+        # Call REST API for Chat Inspection
+        try:
+            response: dict[str, Any] = await self._client.chat_inspect(
+                messages=messages,
+                metadata=self.config.metadata,
+                inspect_config=self.config.inspect_config,
+            )
+
+            # Map is_safe to matched
+            is_safe = response.get("is_safe")
+            if isinstance(is_safe, bool):
+                matched = not is_safe
+                msg = "Content is unsafe" if matched else "Content is safe"
+                meta: dict[str, Any] = {
+                    "severity": response.get("severity"),
+                    "classifications": response.get("classifications"),
+                    "rules": response.get("rules"),
+                    "attack_technique": response.get("attack_technique"),
+                    "event_id": response.get("event_id"),
+                }
+                if self.config.include_raw_response:
+                    meta["raw"] = response
+                return EvaluatorResult(
+                    matched=matched,
+                    confidence=1.0,
+                    message=msg,
+                    metadata=meta,
+                )
+
+            # If no boolean is present, consider it an evaluator error
+            fallback = self.config.on_error
+            matched = fallback == "deny"
+            error_message = "Cisco AI Defense response missing 'is_safe'"
+            meta2: dict[str, Any] = {"fallback_action": fallback}
+            if self.config.include_raw_response:
+                meta2["raw"] = response
+            return EvaluatorResult(
+                matched=matched,
+                confidence=0.0,
+                message=error_message,
+                metadata=meta2,
+                error=None if matched else error_message,
+            )
+        except Exception as e:  # noqa: BLE001
+            fallback = self.config.on_error
+            matched = fallback == "deny"
+            error_detail = str(e)
+            return EvaluatorResult(
+                matched=matched,
+                confidence=0.0,
+                message=f"Cisco AI Defense evaluation error: {error_detail}",
+                metadata={
+                    "error": error_detail,
+                    "error_type": type(e).__name__,
+                    "fallback_action": fallback,
+                },
+                error=None if matched else error_detail,
+            )
+
+    async def aclose(self) -> None:
+        """Close the underlying Cisco AI Defense client."""
+        await self._client.aclose()

agent_control_evaluator_cisco-7.7.0.dist-info/METADATA

@@ -0,0 +1,194 @@
+Metadata-Version: 2.4
+Name: agent-control-evaluator-cisco
+Version: 7.7.0
+Summary: Cisco AI Defense evaluator for agent-control
+Author: Cisco AI Defense Team
+License: Apache-2.0
+Requires-Python: >=3.12
+Requires-Dist: agent-control-evaluators>=7.7.0
+Requires-Dist: agent-control-models>=7.7.0
+Requires-Dist: httpx>=0.24.0
+Provides-Extra: dev
+Requires-Dist: mypy>=1.8.0; extra == 'dev'
+Requires-Dist: pytest-asyncio>=0.23.0; extra == 'dev'
+Requires-Dist: pytest-cov>=4.0.0; extra == 'dev'
+Requires-Dist: pytest>=8.0.0; extra == 'dev'
+Requires-Dist: ruff>=0.1.0; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# Agent Control Evaluator - Cisco AI Defense
+
+External evaluator that calls Cisco AI Defense Chat Inspection via REST and maps `InspectResponse.is_safe` to Agent Control decisions.
+
+- Entry point name: `cisco.ai_defense`
+- Transport: direct HTTP (httpx)
+
+## Installation
+
+Canonical install path:
+
+```bash
+pip install "agent-control-evaluators[cisco]"
+```
+
+Fallback direct wheel install:
+
+```bash
+pip install agent-control-evaluator-cisco
+```
+
+For local development:
+
+```bash
+uv pip install -e evaluators/contrib/cisco
+```
+
+- Build wheel from the repo root (contrib package only):
+
+  ```bash
+  make engine-build
+  (cd evaluators/contrib/cisco && make build)
+  ```
+
+To run the server with this evaluator enabled, see `examples/cisco_ai_defense/README.md` for setup and seeding instructions.
+
+## Configuration
+
+Set the `AI_DEFENSE_API_KEY` environment variable:
+
+```bash
+export AI_DEFENSE_API_KEY="<your_key>"
+```
+
+Evaluator config fields (all optional unless stated):
+
+- `api_key_env: str = "AI_DEFENSE_API_KEY"`
+- `region: "us" | "ap" | "eu" | None = "us"` (ignored if `api_url` set)
+- `api_url: str | None = None` (full endpoint override; e.g., `https://us.../api/v1/inspect/chat`)
+- `timeout_ms: int = 15000`
+- `on_error: "allow" | "deny" = "allow"` (fail-open or fail-closed on transport/response errors)
+- `payload_field: "input" | "output" | None = None`
+  - When set, synthesizes a single message from that field; `input` → `role=user`, `output` → `role=assistant`.
+- `messages_strategy: "single" | "history" = "history"`
+  - `history` forwards an existing `messages` list in the selected data if present; falls back to single otherwise.
+- `metadata: dict[str, Any] | None = None` (forwarded to API per OpenAPI spec)
+- `inspect_config: dict[str, Any] | None = None` (forwarded to API per OpenAPI spec)
+- `include_raw_response: bool = false` (when true, includes the full provider response under `metadata.raw`)
+
+## Available Evaluators
+
+| Name | Description |
+|------|-------------|
+| `cisco.ai_defense` | Cisco AI Defense Chat Inspection |
+
+Behavior mapping:
+
+- `is_safe == false` → `EvaluatorResult.matched = true` (e.g., a `deny` action will block)
+- `is_safe == true` → `matched = false`
+- Errors or invalid responses → `matched = (on_error == "deny")`; error details in `metadata` (no `error` field is set; engine honors `matched` per `on_error`)
+
+## Minimal server control configuration
+
+Example using `messages_strategy: "history"` (for inputs that already have a `messages` list):
+
+```
+{
+  "description": "Apply Cisco AI Defense Security, Safety, and Privacy guardrails",
+  "enabled": true,
+  "execution": "server",
+  "scope": { "step_types": ["llm"], "stages": ["pre", "post"] },
+  "condition": {
+    "selector": { "path": "input" },
+    "evaluator": {
+      "name": "cisco.ai_defense",
+      "config": {
+        "api_key_env": "AI_DEFENSE_API_KEY",
+        "region": "us",
+        "timeout_ms": 15000,
+        "on_error": "allow",
+        "messages_strategy": "history"
+      }
+    }
+  },
+  "action": { "decision": "deny" },
+  "tags": ["ai_defense", "safety"]
+}
+```
+
+```
+{
+  "description": "Apply Cisco AI Defense Security, Safety, and Privacy guardrails",
+  "enabled": true,
+  "execution": "server",
+  "scope": { "step_types": ["llm"], "stages": ["pre", "post"] },
+  "condition": {
+    "selector": { "path": "input" },
+    "evaluator": {
+      "name": "cisco.ai_defense",
+      "config": {
+        "api_key_env": "AI_DEFENSE_API_KEY",
+        "region": "us",
+        "timeout_ms": 15000,
+        "on_error": "allow",
+        "messages_strategy": "single",
+        "payload_field": "input"
+      }
+    }
+  },
+  "action": { "decision": "deny" },
+  "tags": ["ai_defense", "safety"]
+}
+```
+
+## Usage
+
+Once installed, the evaluator is automatically discovered:
+
+```python
+from agent_control_evaluators import discover_evaluators, get_evaluator
+
+discover_evaluators()
+CiscoAIDefenseEvaluator = get_evaluator("cisco.ai_defense")
+```
+
+Or import directly:
+
+```python
+import asyncio
+from agent_control_evaluator_cisco.ai_defense import CiscoAIDefenseEvaluator, CiscoAIDefenseConfig
+
+cfg = CiscoAIDefenseConfig(
+    region="us",
+    timeout_ms=15000,
+    on_error="allow",
+    messages_strategy="history",
+    payload_field="input",
+)
+ev = CiscoAIDefenseEvaluator(cfg)
+
+async def main():
+    data = {"messages": [{"role": "user", "content": "tell me how to hack wifi"}]}
+    print(await ev.evaluate(data))
+
+asyncio.run(main())
+```
+
+## Notes
+
+- Auth header: `X-Cisco-AI-Defense-API-Key: <AI_DEFENSE_API_KEY>`
+- Regions and endpoint path follow the Cisco AI Defense API spec
+- For custom deployments, set `api_url` to the full Chat Inspection endpoint.
+- The evaluator validates the API key at construction and raises if missing.
+- `is_available()` returns false if `httpx` is not installed; discovery will skip registration.
+- `messages_strategy: "history"` forwards the full message array when present; consider `messages_strategy: "single"` if payload size is a concern.
+
+## Documentation
+
+- Cisco AI Defense Inspection API reference: https://developer.cisco.com/docs/ai-defense-inspection/introduction/
+- Cisco Security Console (get API Key): https://security.cisco.com
+- Cisco AI Defense User Guide: https://securitydocs.cisco.com/docs/ai-def/user/97384.dita
+- Regional API base URLs used by this evaluator:
+  - US: `https://us.api.inspect.aidefense.security.cisco.com`
+  - AP: `https://ap.api.inspect.aidefense.security.cisco.com`
+  - EU: `https://eu.api.inspect.aidefense.security.cisco.com`
+  - Chat Inspection path: `/api/v1/inspect/chat`

agent_control_evaluator_cisco-7.7.0.dist-info/RECORD

@@ -0,0 +1,9 @@
+agent_control_evaluator_cisco/__init__.py,sha256=8DDvq0Peb__lGVcurhl4UvirKbOlgwleoZit4kOpYd4,14
+agent_control_evaluator_cisco/ai_defense/__init__.py,sha256=OqDF1J2GcagNASxsHSUD8O08rYQt7cmfkac_rSUtdcE,152
+agent_control_evaluator_cisco/ai_defense/client.py,sha256=qRHV5P8tneGdG-NylbSSR3WubteYOS55GP5vfVc-5Bc,3304
+agent_control_evaluator_cisco/ai_defense/config.py,sha256=54qIBL7QQAt5AvxildR7xPCZ6QN-Rw2ac1V_kIw-cfU,1373
+agent_control_evaluator_cisco/ai_defense/evaluator.py,sha256=HlKkZRmzTdqI0Q44kgJYeP-aMLeSgefwhPHnkgQqHEg,7870
+agent_control_evaluator_cisco-7.7.0.dist-info/METADATA,sha256=sACAcDeWRsVp5dZadVLRB8OD3t1gJGQfmPwBkHg3hBE,6175
+agent_control_evaluator_cisco-7.7.0.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
+agent_control_evaluator_cisco-7.7.0.dist-info/entry_points.txt,sha256=UxIxKYokYK7Bd7Cc18sAfwdKKEIrqUhaD9mZBadS5S8,111
+agent_control_evaluator_cisco-7.7.0.dist-info/RECORD,,

agent_control_evaluator_cisco-7.7.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.29.0
+Root-Is-Purelib: true
+Tag: py3-none-any

agent_control_evaluator_cisco-7.7.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[agent_control.evaluators]
+cisco.ai_defense = agent_control_evaluator_cisco.ai_defense:CiscoAIDefenseEvaluator