evalguard-python 1.1.0__py3-none-any.whl

evalguard/crewai.py

@@ -0,0 +1,189 @@
+"""CrewAI guardrail integration for EvalGuard.
+
+Usage::
+
+    from evalguard.crewai import EvalGuardGuardrail, guard_agent
+    from crewai import Crew, Agent, Task
+
+    # Option 1: Guard individual agents
+    agent = guard_agent(
+        Agent(role="researcher", goal="...", backstory="..."),
+        api_key="eg_...",
+    )
+
+    # Option 2: Use as a crew-level guardrail
+    guardrail = EvalGuardGuardrail(api_key="eg_...", project_id="proj_...")
+    crew = Crew(agents=[agent], tasks=[...])
+    # Call guardrail.check() before/after crew.kickoff()
+"""
+
+from __future__ import annotations
+
+import functools
+import time
+from typing import Any, Callable, Dict, List, Optional
+
+from .guardrails import GuardrailClient, GuardrailViolation
+
+
+class EvalGuardGuardrail:
+    """Standalone guardrail that can be used with CrewAI workflows.
+
+    Parameters
+    ----------
+    api_key:
+        EvalGuard API key.
+    project_id:
+        Optional project ID for trace grouping.
+    rules:
+        Guardrail rules for input checking.
+    block_on_violation:
+        If *True*, :meth:`check` raises on violation.
+    """
+
+    def __init__(
+        self,
+        api_key: str,
+        project_id: Optional[str] = None,
+        base_url: str = "https://api.evalguard.ai",
+        rules: Optional[List[str]] = None,
+        block_on_violation: bool = True,
+        timeout: float = 5.0,
+    ) -> None:
+        self._guard = GuardrailClient(
+            api_key=api_key,
+            base_url=base_url,
+            project_id=project_id,
+            timeout=timeout,
+        )
+        self._rules = rules
+        self._block = block_on_violation
+
+    def check(self, text: str, metadata: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
+        """Check text against guardrails.
+
+        Returns
+        -------
+        dict
+            ``{"allowed": bool, "violations": [...]}``
+
+        Raises
+        ------
+        GuardrailViolation
+            If ``block_on_violation`` is *True* and the check fails.
+        """
+        result = self._guard.check_input(text, rules=self._rules, metadata=metadata)
+        if not result.get("allowed", True) and self._block:
+            raise GuardrailViolation(result.get("violations", []))
+        return result
+
+    def log(self, data: Dict[str, Any]) -> None:
+        """Log a trace entry."""
+        self._guard.log_trace(data)
+
+    def wrap_function(self, fn: Callable[..., Any]) -> Callable[..., Any]:
+        """Decorator that guards a function's first string argument."""
+        @functools.wraps(fn)
+        def wrapper(*args: Any, **kwargs: Any) -> Any:
+            # Find the first string argument to check
+            text = ""
+            for arg in args:
+                if isinstance(arg, str):
+                    text = arg
+                    break
+            if not text:
+                for v in kwargs.values():
+                    if isinstance(v, str):
+                        text = v
+                        break
+
+            if text:
+                self.check(text, metadata={"function": fn.__name__})
+
+            start = time.monotonic()
+            result = fn(*args, **kwargs)
+            elapsed_ms = (time.monotonic() - start) * 1000
+
+            self._guard.log_trace({
+                "provider": "crewai",
+                "function": fn.__name__,
+                "input": text,
+                "output": str(result)[:2000] if result else "",
+                "llm_latency_ms": round(elapsed_ms, 2),
+            })
+            return result
+
+        return wrapper
+
+
+def guard_agent(
+    agent: Any,
+    *,
+    api_key: str,
+    project_id: Optional[str] = None,
+    base_url: str = "https://api.evalguard.ai",
+    rules: Optional[List[str]] = None,
+    block_on_violation: bool = True,
+    timeout: float = 5.0,
+) -> Any:
+    """Wrap a CrewAI ``Agent`` so that every task execution is guarded.
+
+    This patches the agent's ``execute_task`` method to run guardrail
+    checks before and trace logging after each task execution.
+
+    Parameters
+    ----------
+    agent:
+        A ``crewai.Agent`` instance.
+
+    Returns
+    -------
+    The same agent instance, with guardrails applied.
+    """
+    guard = GuardrailClient(
+        api_key=api_key,
+        base_url=base_url,
+        project_id=project_id,
+        timeout=timeout,
+    )
+    guardrail_rules = rules
+    block = block_on_violation
+
+    original_execute = getattr(agent, "execute_task", None)
+    if original_execute is None:
+        return agent
+
+    @functools.wraps(original_execute)
+    def guarded_execute(task: Any, *args: Any, **kwargs: Any) -> Any:
+        task_desc = getattr(task, "description", str(task)) if task else ""
+
+        # Pre-check
+        check = guard.check_input(
+            task_desc,
+            rules=guardrail_rules,
+            metadata={
+                "agent_role": getattr(agent, "role", "unknown"),
+                "framework": "crewai",
+            },
+        )
+        if not check.get("allowed", True) and block:
+            raise GuardrailViolation(check.get("violations", []))
+
+        # Execute
+        start = time.monotonic()
+        result = original_execute(task, *args, **kwargs)
+        elapsed_ms = (time.monotonic() - start) * 1000
+
+        # Trace
+        guard.log_trace({
+            "provider": "crewai",
+            "agent_role": getattr(agent, "role", "unknown"),
+            "input": task_desc[:2000],
+            "output": str(result)[:2000] if result else "",
+            "llm_latency_ms": round(elapsed_ms, 2),
+            "violations": check.get("violations", []),
+        })
+        return result
+
+    agent.execute_task = guarded_execute
+    return agent

evalguard/fastapi.py

@@ -0,0 +1,273 @@
+"""FastAPI middleware for EvalGuard.
+
+Usage::
+
+    from evalguard.fastapi import EvalGuardMiddleware
+    from fastapi import FastAPI
+
+    app = FastAPI()
+    app.add_middleware(
+        EvalGuardMiddleware,
+        api_key="eg_...",
+        project_id="proj_...",
+    )
+    # All matching endpoints are now guarded
+
+You can also use the route decorator for fine-grained control::
+
+    from evalguard.fastapi import guard_route
+
+    @app.post("/api/chat")
+    @guard_route(api_key="eg_...", rules=["prompt_injection"])
+    async def chat(request: Request):
+        ...
+"""
+
+from __future__ import annotations
+
+import functools
+import json
+import time
+from typing import Any, Callable, Dict, List, Optional, Set
+
+from .guardrails import GuardrailClient, GuardrailViolation
+
+
+class EvalGuardMiddleware:
+    """ASGI middleware that guards incoming requests.
+
+    By default, guards POST requests to paths containing ``/chat``,
+    ``/completions``, ``/generate``, or ``/invoke``.  Customize via
+    ``guarded_paths``.
+
+    Parameters
+    ----------
+    app:
+        The ASGI application.
+    api_key:
+        EvalGuard API key.
+    project_id:
+        Optional project ID.
+    rules:
+        Guardrail rules for input checking.
+    guarded_paths:
+        URL path substrings that trigger guardrail checks.
+    block_on_violation:
+        If *True*, return 403 when input is blocked.
+    """
+
+    _DEFAULT_PATHS = {"/chat", "/completions", "/generate", "/invoke", "/messages"}
+
+    def __init__(
+        self,
+        app: Any,
+        api_key: str,
+        project_id: Optional[str] = None,
+        base_url: str = "https://api.evalguard.ai",
+        rules: Optional[List[str]] = None,
+        guarded_paths: Optional[Set[str]] = None,
+        block_on_violation: bool = True,
+        timeout: float = 5.0,
+    ) -> None:
+        self.app = app
+        self._guard = GuardrailClient(
+            api_key=api_key,
+            base_url=base_url,
+            project_id=project_id,
+            timeout=timeout,
+        )
+        self._rules = rules
+        self._paths = guarded_paths or self._DEFAULT_PATHS
+        self._block = block_on_violation
+
+    async def __call__(self, scope: Dict[str, Any], receive: Any, send: Any) -> None:
+        if scope["type"] != "http":
+            await self.app(scope, receive, send)
+            return
+
+        path: str = scope.get("path", "")
+        method: str = scope.get("method", "GET")
+
+        # Only guard POST/PUT requests to matching paths
+        if method not in ("POST", "PUT") or not self._should_guard(path):
+            await self.app(scope, receive, send)
+            return
+
+        # Read the request body
+        body_chunks: list[bytes] = []
+        request_complete = False
+
+        async def receive_wrapper() -> Dict[str, Any]:
+            nonlocal request_complete
+            if body_chunks and request_complete:
+                # Replay the body for the downstream app
+                return {"type": "http.request", "body": b"".join(body_chunks), "more_body": False}
+            message = await receive()
+            if message["type"] == "http.request":
+                body_chunks.append(message.get("body", b""))
+                if not message.get("more_body", False):
+                    request_complete = True
+            return message
+
+        # Consume the body first
+        while not request_complete:
+            await receive_wrapper()
+
+        body_bytes = b"".join(body_chunks)
+        prompt_text = _extract_body_text(body_bytes)
+
+        if prompt_text:
+            start = time.monotonic()
+            check = self._guard.check_input(
+                prompt_text,
+                rules=self._rules,
+                metadata={"path": path, "method": method, "framework": "fastapi"},
+            )
+            guard_ms = (time.monotonic() - start) * 1000
+
+            if not check.get("allowed", True) and self._block:
+                # Return 403 Forbidden
+                response_body = json.dumps({
+                    "error": "Blocked by EvalGuard guardrail",
+                    "violations": check.get("violations", []),
+                }).encode()
+                await send({
+                    "type": "http.response.start",
+                    "status": 403,
+                    "headers": [
+                        [b"content-type", b"application/json"],
+                        [b"content-length", str(len(response_body)).encode()],
+                    ],
+                })
+                await send({
+                    "type": "http.response.body",
+                    "body": response_body,
+                })
+                return
+
+        # Pass through to the app with replayed body
+        request_complete = True  # ensure replay mode
+        start = time.monotonic()
+        await self.app(scope, receive_wrapper, send)
+        llm_ms = (time.monotonic() - start) * 1000
+
+        # Best-effort trace log
+        self._guard.log_trace({
+            "provider": "fastapi",
+            "path": path,
+            "input": prompt_text[:2000] if prompt_text else "",
+            "guard_latency_ms": round(guard_ms, 2) if prompt_text else 0,
+            "request_latency_ms": round(llm_ms, 2),
+        })
+
+    def _should_guard(self, path: str) -> bool:
+        return any(p in path for p in self._paths)
+
+
+def guard_route(
+    *,
+    api_key: str,
+    project_id: Optional[str] = None,
+    base_url: str = "https://api.evalguard.ai",
+    rules: Optional[List[str]] = None,
+    block_on_violation: bool = True,
+    timeout: float = 5.0,
+) -> Callable:
+    """Decorator for guarding individual FastAPI route handlers.
+
+    Usage::
+
+        @app.post("/api/chat")
+        @guard_route(api_key="eg_...")
+        async def chat(request: Request):
+            body = await request.json()
+            ...
+    """
+    guard = GuardrailClient(
+        api_key=api_key,
+        base_url=base_url,
+        project_id=project_id,
+        timeout=timeout,
+    )
+
+    def decorator(fn: Callable) -> Callable:
+        @functools.wraps(fn)
+        async def wrapper(*args: Any, **kwargs: Any) -> Any:
+            # Try to find the Request object
+            request = None
+            for arg in args:
+                if hasattr(arg, "json") and hasattr(arg, "method"):
+                    request = arg
+                    break
+            for v in kwargs.values():
+                if hasattr(v, "json") and hasattr(v, "method"):
+                    request = v
+                    break
+
+            if request:
+                try:
+                    body = await request.json()
+                    prompt_text = _extract_dict_text(body)
+                    if prompt_text:
+                        check = guard.check_input(prompt_text, rules=rules)
+                        if not check.get("allowed", True) and block_on_violation:
+                            # Import here to avoid hard dependency
+                            try:
+                                from fastapi.responses import JSONResponse
+                                return JSONResponse(
+                                    status_code=403,
+                                    content={
+                                        "error": "Blocked by EvalGuard guardrail",
+                                        "violations": check.get("violations", []),
+                                    },
+                                )
+                            except ImportError:
+                                raise GuardrailViolation(check.get("violations", []))
+                except Exception:
+                    pass  # fail-open
+
+            return await fn(*args, **kwargs)
+
+        return wrapper
+
+    return decorator
+
+
+def _extract_body_text(body: bytes) -> str:
+    """Extract prompt text from a JSON request body."""
+    try:
+        data = json.loads(body)
+        return _extract_dict_text(data)
+    except (json.JSONDecodeError, UnicodeDecodeError):
+        return ""
+
+
+def _extract_dict_text(data: Any) -> str:
+    """Extract prompt/message text from a parsed JSON body."""
+    if not isinstance(data, dict):
+        return ""
+
+    # Direct prompt field
+    if "prompt" in data:
+        return data["prompt"] if isinstance(data["prompt"], str) else str(data["prompt"])
+
+    # OpenAI-style messages
+    messages = data.get("messages", [])
+    if messages and isinstance(messages, list):
+        parts: list[str] = []
+        for msg in messages:
+            if isinstance(msg, dict):
+                content = msg.get("content", "")
+                if isinstance(content, str):
+                    parts.append(content)
+        return "\n".join(parts)
+
+    # Input field (common in many APIs)
+    if "input" in data:
+        return data["input"] if isinstance(data["input"], str) else str(data["input"])
+
+    # Query field
+    if "query" in data:
+        return data["query"] if isinstance(data["query"], str) else str(data["query"])
+
+    return ""

evalguard/guardrails.py

@@ -0,0 +1,160 @@
+"""Core guardrail client shared by all framework integrations.
+
+Provides pre-LLM input checking (prompt injection, PII redaction) and
+post-LLM trace logging.  Every framework wrapper delegates to a single
+:class:`GuardrailClient` instance so configuration is consistent.
+"""
+
+from __future__ import annotations
+
+import logging
+import time
+from typing import Any, Dict, List, Optional
+
+import requests
+
+logger = logging.getLogger("evalguard.guardrails")
+
+_DEFAULT_RULES: List[str] = ["prompt_injection", "pii_redact"]
+
+
+class GuardrailClient:
+    """Lightweight HTTP client for the EvalGuard guardrail & trace APIs.
+
+    Parameters
+    ----------
+    api_key:
+        EvalGuard API key (``eg_live_...`` or ``eg_test_...``).
+    base_url:
+        API base URL.  Override for self-hosted deployments.
+    project_id:
+        Optional project ID attached to every trace.
+    timeout:
+        HTTP request timeout in seconds.  Keep low (default 5 s) so the
+        guardrail check never dominates end-to-end latency.
+    fail_open:
+        If *True*, network / server errors allow the request through
+        instead of raising.  Defaults to *False*.
+    """
+
+    def __init__(
+        self,
+        api_key: str,
+        base_url: str = "https://api.evalguard.ai",
+        project_id: Optional[str] = None,
+        timeout: float = 5.0,
+        fail_open: bool = False,
+    ) -> None:
+        self._api_key = api_key
+        self._base_url = base_url.rstrip("/")
+        self._project_id = project_id
+        self._timeout = timeout
+        self._fail_open = fail_open
+        self._session = requests.Session()
+        self._session.headers.update(
+            {
+                "Authorization": f"Bearer {api_key}",
+                "Content-Type": "application/json",
+                "User-Agent": "evalguard-python/1.0.0",
+            }
+        )
+
+    # ── Public API ────────────────────────────────────────────────────
+
+    def check_input(
+        self,
+        text: str,
+        rules: Optional[List[str]] = None,
+        metadata: Optional[Dict[str, Any]] = None,
+    ) -> Dict[str, Any]:
+        """Pre-LLM guard check.
+
+        Returns
+        -------
+        dict
+            ``{"allowed": bool, "violations": [...], "sanitized": str | None}``
+        """
+        payload: Dict[str, Any] = {
+            "input": text,
+            "rules": rules or _DEFAULT_RULES,
+        }
+        if self._project_id:
+            payload["project_id"] = self._project_id
+        if metadata:
+            payload["metadata"] = metadata
+        try:
+            resp = self._session.post(
+                f"{self._base_url}/api/v1/guardrails",
+                json=payload,
+                timeout=self._timeout,
+            )
+            resp.raise_for_status()
+            data = resp.json()
+            return data.get("data", {"allowed": True, "violations": [], "sanitized": None})
+        except Exception:
+            if self._fail_open:
+                logger.debug("Guardrail check failed; fail-open allowing request", exc_info=True)
+                return {"allowed": True, "violations": [], "sanitized": None}
+            raise
+
+    def check_output(
+        self,
+        text: str,
+        rules: Optional[List[str]] = None,
+        metadata: Optional[Dict[str, Any]] = None,
+    ) -> Dict[str, Any]:
+        """Post-LLM output check.
+
+        Returns
+        -------
+        dict
+            ``{"allowed": bool, "violations": [...], "sanitized": str | None}``
+        """
+        payload: Dict[str, Any] = {
+            "output": text,
+            "rules": rules or ["toxic_content", "pii_redact"],
+        }
+        if self._project_id:
+            payload["project_id"] = self._project_id
+        if metadata:
+            payload["metadata"] = metadata
+        try:
+            resp = self._session.post(
+                f"{self._base_url}/api/v1/guardrails/output",
+                json=payload,
+                timeout=self._timeout,
+            )
+            resp.raise_for_status()
+            data = resp.json()
+            return data.get("data", {"allowed": True, "violations": [], "sanitized": None})
+        except Exception:
+            if self._fail_open:
+                logger.debug("Output check failed; fail-open allowing response", exc_info=True)
+                return {"allowed": True, "violations": [], "sanitized": None}
+            raise
+
+    def log_trace(self, data: Dict[str, Any]) -> None:
+        """Fire-and-forget trace logging.  Errors are silently swallowed."""
+        payload = {**data}
+        if self._project_id:
+            payload.setdefault("project_id", self._project_id)
+        try:
+            self._session.post(
+                f"{self._base_url}/api/v1/traces",
+                json=payload,
+                timeout=self._timeout,
+            )
+        except Exception:
+            logger.debug("Trace log failed", exc_info=True)
+
+    def close(self) -> None:
+        """Close the underlying HTTP session."""
+        self._session.close()
+
+
+class GuardrailViolation(Exception):
+    """Raised when a guardrail check blocks a request."""
+
+    def __init__(self, violations: List[Dict[str, Any]], message: str = "Blocked by EvalGuard guardrail"):
+        super().__init__(message)
+        self.violations = violations