evalguard-python 1.1.0__py3-none-any.whl

evalguard/langchain.py

@@ -0,0 +1,218 @@
+"""LangChain callback handler for EvalGuard.
+
+Usage::
+
+    from evalguard.langchain import EvalGuardCallback
+    from langchain_openai import ChatOpenAI
+
+    callback = EvalGuardCallback(api_key="eg_...", project_id="proj_...")
+    llm = ChatOpenAI(callbacks=[callback])
+    # Every LLM call is now guarded and traced
+    llm.invoke("Hello, world!")
+
+Works with any LangChain LLM, chat model, or chain that supports callbacks.
+"""
+
+from __future__ import annotations
+
+import time
+import uuid
+from typing import Any, Dict, List, Optional, Sequence, Union
+
+from .guardrails import GuardrailClient, GuardrailViolation
+
+# LangChain callback protocol: we implement the methods directly rather
+# than inheriting from BaseCallbackHandler so the SDK has zero hard
+# dependencies on LangChain.  When LangChain is installed, the duck-typed
+# interface is fully compatible.
+
+
+class EvalGuardCallback:
+    """LangChain callback that guards every LLM call via EvalGuard.
+
+    This class implements the LangChain callback protocol without importing
+    LangChain, so it works with *any* version (0.1.x, 0.2.x, 0.3.x).
+
+    Parameters
+    ----------
+    api_key:
+        EvalGuard API key.
+    project_id:
+        Optional project ID for trace grouping.
+    rules:
+        Guardrail rules for input checking.
+    block_on_violation:
+        Raise :class:`GuardrailViolation` when input is blocked.
+    """
+
+    def __init__(
+        self,
+        api_key: str,
+        project_id: Optional[str] = None,
+        base_url: str = "https://api.evalguard.ai",
+        rules: Optional[List[str]] = None,
+        block_on_violation: bool = True,
+        timeout: float = 5.0,
+    ) -> None:
+        self._guard = GuardrailClient(
+            api_key=api_key,
+            base_url=base_url,
+            project_id=project_id,
+            timeout=timeout,
+        )
+        self._rules = rules
+        self._block = block_on_violation
+        # Per-run state keyed by run_id
+        self._runs: Dict[str, Dict[str, Any]] = {}
+
+    # ── LangChain callback protocol ──────────────────────────────────
+
+    def on_llm_start(
+        self,
+        serialized: Dict[str, Any],
+        prompts: List[str],
+        *,
+        run_id: Optional[Any] = None,
+        parent_run_id: Optional[Any] = None,
+        tags: Optional[List[str]] = None,
+        metadata: Optional[Dict[str, Any]] = None,
+        **kwargs: Any,
+    ) -> None:
+        """Pre-LLM: check for prompt injection, PII, etc."""
+        rid = str(run_id or uuid.uuid4())
+        prompt_text = "\n".join(prompts)
+        model_name = serialized.get("name", serialized.get("id", ["unknown"])[-1] if isinstance(serialized.get("id"), list) else "unknown")
+
+        start = time.monotonic()
+        check = self._guard.check_input(
+            prompt_text,
+            rules=self._rules,
+            metadata={"model": model_name, "framework": "langchain"},
+        )
+        guard_ms = (time.monotonic() - start) * 1000
+
+        self._runs[rid] = {
+            "model": model_name,
+            "input": prompt_text,
+            "guard_ms": guard_ms,
+            "violations": check.get("violations", []),
+            "start": time.monotonic(),
+        }
+
+        if not check.get("allowed", True) and self._block:
+            raise GuardrailViolation(check.get("violations", []))
+
+    def on_chat_model_start(
+        self,
+        serialized: Dict[str, Any],
+        messages: List[Any],
+        *,
+        run_id: Optional[Any] = None,
+        parent_run_id: Optional[Any] = None,
+        tags: Optional[List[str]] = None,
+        metadata: Optional[Dict[str, Any]] = None,
+        **kwargs: Any,
+    ) -> None:
+        """Pre-chat-model: extract text from message objects and check."""
+        prompts = []
+        for message_group in messages:
+            for msg in (message_group if isinstance(message_group, list) else [message_group]):
+                content = getattr(msg, "content", "") if not isinstance(msg, dict) else msg.get("content", "")
+                if isinstance(content, str):
+                    prompts.append(content)
+        self.on_llm_start(serialized, prompts, run_id=run_id, parent_run_id=parent_run_id, tags=tags, metadata=metadata, **kwargs)
+
+    def on_llm_end(
+        self,
+        response: Any,
+        *,
+        run_id: Optional[Any] = None,
+        parent_run_id: Optional[Any] = None,
+        **kwargs: Any,
+    ) -> None:
+        """Post-LLM: log the complete trace."""
+        rid = str(run_id or "")
+        run_data = self._runs.pop(rid, {})
+        llm_ms = (time.monotonic() - run_data.get("start", time.monotonic())) * 1000
+
+        output_text = _extract_lc_response(response)
+        self._guard.log_trace(
+            {
+                "provider": "langchain",
+                "model": run_data.get("model", "unknown"),
+                "input": run_data.get("input", ""),
+                "output": output_text,
+                "guard_latency_ms": round(run_data.get("guard_ms", 0), 2),
+                "llm_latency_ms": round(llm_ms, 2),
+                "violations": run_data.get("violations", []),
+            }
+        )
+
+    def on_llm_error(
+        self,
+        error: BaseException,
+        *,
+        run_id: Optional[Any] = None,
+        parent_run_id: Optional[Any] = None,
+        **kwargs: Any,
+    ) -> None:
+        """Log error traces for failed LLM calls."""
+        rid = str(run_id or "")
+        run_data = self._runs.pop(rid, {})
+        self._guard.log_trace(
+            {
+                "provider": "langchain",
+                "model": run_data.get("model", "unknown"),
+                "input": run_data.get("input", ""),
+                "output": "",
+                "error": str(error),
+                "violations": run_data.get("violations", []),
+            }
+        )
+
+    # ── Chain-level callbacks (no-ops, present for compatibility) ────
+
+    def on_chain_start(self, serialized: Dict[str, Any], inputs: Dict[str, Any], **kwargs: Any) -> None:
+        pass
+
+    def on_chain_end(self, outputs: Dict[str, Any], **kwargs: Any) -> None:
+        pass
+
+    def on_chain_error(self, error: BaseException, **kwargs: Any) -> None:
+        pass
+
+    def on_tool_start(self, serialized: Dict[str, Any], input_str: str, **kwargs: Any) -> None:
+        pass
+
+    def on_tool_end(self, output: str, **kwargs: Any) -> None:
+        pass
+
+    def on_tool_error(self, error: BaseException, **kwargs: Any) -> None:
+        pass
+
+    def on_text(self, text: str, **kwargs: Any) -> None:
+        pass
+
+    def on_retry(self, retry_state: Any, **kwargs: Any) -> None:
+        pass
+
+
+def _extract_lc_response(response: Any) -> str:
+    """Extract text from a LangChain LLMResult or ChatResult."""
+    try:
+        # LLMResult / ChatResult have .generations
+        generations = getattr(response, "generations", None)
+        if generations:
+            parts: list[str] = []
+            for gen_list in generations:
+                for gen in gen_list:
+                    # ChatGeneration has .message.content; Generation has .text
+                    msg = getattr(gen, "message", None)
+                    if msg:
+                        parts.append(getattr(msg, "content", str(msg)))
+                    else:
+                        parts.append(getattr(gen, "text", str(gen)))
+            return "\n".join(parts)
+    except Exception:
+        pass
+    return str(response) if response else ""

evalguard/nemoclaw.py

@@ -0,0 +1,251 @@
+"""NVIDIA NeMo Guardrails / OpenClaw agent integration for EvalGuard.
+
+Usage::
+
+    from evalguard.nemoclaw import EvalGuardAgent
+
+    agent = EvalGuardAgent(api_key="eg_...", agent_name="support-bot")
+
+    # Guard any LLM call regardless of provider
+    result = agent.guarded_call(
+        provider="openai",
+        messages=[{"role": "user", "content": "Hello"}],
+        llm_fn=lambda: openai_client.chat.completions.create(
+            model="gpt-4", messages=[{"role": "user", "content": "Hello"}]
+        ),
+    )
+
+    # Or use as a context manager for multi-step agent workflows
+    with agent.session("ticket-123") as session:
+        session.check("User says: reset my password")
+        result = do_llm_call(...)
+        session.log_step("password_reset", input="...", output=str(result))
+"""
+
+from __future__ import annotations
+
+import time
+import uuid
+from contextlib import contextmanager
+from typing import Any, Callable, Dict, Generator, List, Optional
+
+from .guardrails import GuardrailClient, GuardrailViolation
+
+
+class EvalGuardAgent:
+    """Agent-level guardrail wrapper for NeMo/OpenClaw-style agent systems.
+
+    Parameters
+    ----------
+    api_key:
+        EvalGuard API key.
+    agent_name:
+        A human-readable name for this agent (used in traces).
+    project_id:
+        Optional project ID.
+    rules:
+        Default guardrail rules.
+    block_on_violation:
+        Raise on violation if *True*.
+    """
+
+    def __init__(
+        self,
+        api_key: str,
+        agent_name: str = "default",
+        project_id: Optional[str] = None,
+        base_url: str = "https://api.evalguard.ai",
+        rules: Optional[List[str]] = None,
+        block_on_violation: bool = True,
+        timeout: float = 5.0,
+    ) -> None:
+        self._guard = GuardrailClient(
+            api_key=api_key,
+            base_url=base_url,
+            project_id=project_id,
+            timeout=timeout,
+        )
+        self._agent_name = agent_name
+        self._rules = rules
+        self._block = block_on_violation
+
+    def guarded_call(
+        self,
+        provider: str,
+        messages: List[Dict[str, str]],
+        llm_fn: Callable[[], Any],
+        *,
+        rules: Optional[List[str]] = None,
+        metadata: Optional[Dict[str, Any]] = None,
+    ) -> Any:
+        """Execute an LLM call with pre/post guardrail checks.
+
+        Parameters
+        ----------
+        provider:
+            LLM provider name (``"openai"``, ``"anthropic"``, etc.).
+        messages:
+            The messages being sent (for guardrail input checking).
+        llm_fn:
+            A zero-argument callable that performs the actual LLM call.
+        rules:
+            Override default rules for this call.
+        metadata:
+            Additional metadata for the trace.
+
+        Returns
+        -------
+        The result of ``llm_fn()``.
+        """
+        prompt_text = "\n".join(
+            msg.get("content", "") for msg in messages if isinstance(msg, dict)
+        )
+
+        # ── Pre-LLM check ────────────────────────────────────────────
+        start = time.monotonic()
+        check = self._guard.check_input(
+            prompt_text,
+            rules=rules or self._rules,
+            metadata={
+                "agent": self._agent_name,
+                "provider": provider,
+                "framework": "nemoclaw",
+                **(metadata or {}),
+            },
+        )
+        guard_ms = (time.monotonic() - start) * 1000
+
+        if not check.get("allowed", True) and self._block:
+            raise GuardrailViolation(check.get("violations", []))
+
+        # ── Execute LLM call ─────────────────────────────────────────
+        start = time.monotonic()
+        result = llm_fn()
+        llm_ms = (time.monotonic() - start) * 1000
+
+        # ── Post-LLM trace ───────────────────────────────────────────
+        output_text = str(result)[:2000] if result else ""
+        self._guard.log_trace(
+            {
+                "provider": provider,
+                "agent": self._agent_name,
+                "framework": "nemoclaw",
+                "input": prompt_text,
+                "output": output_text,
+                "guard_latency_ms": round(guard_ms, 2),
+                "llm_latency_ms": round(llm_ms, 2),
+                "violations": check.get("violations", []),
+                **(metadata or {}),
+            }
+        )
+
+        return result
+
+    @contextmanager
+    def session(self, session_id: Optional[str] = None) -> Generator["_AgentSession", None, None]:
+        """Create a guarded session for multi-step agent workflows.
+
+        Usage::
+
+            with agent.session("ticket-123") as s:
+                s.check("user input here")
+                result = my_llm_call()
+                s.log_step("step_name", input="...", output="...")
+        """
+        sid = session_id or str(uuid.uuid4())
+        sess = _AgentSession(self._guard, self._agent_name, sid, self._rules, self._block)
+        try:
+            yield sess
+        finally:
+            sess._finalize()
+
+
+class _AgentSession:
+    """A multi-step guarded session within an agent."""
+
+    __slots__ = ("_guard", "_agent_name", "_session_id", "_rules", "_block", "_steps")
+
+    def __init__(
+        self,
+        guard: GuardrailClient,
+        agent_name: str,
+        session_id: str,
+        rules: Optional[List[str]],
+        block: bool,
+    ) -> None:
+        self._guard = guard
+        self._agent_name = agent_name
+        self._session_id = session_id
+        self._rules = rules
+        self._block = block
+        self._steps: List[Dict[str, Any]] = []
+
+    def check(self, text: str, *, rules: Optional[List[str]] = None) -> Dict[str, Any]:
+        """Run a guardrail check within this session."""
+        result = self._guard.check_input(
+            text,
+            rules=rules or self._rules,
+            metadata={
+                "agent": self._agent_name,
+                "session_id": self._session_id,
+                "framework": "nemoclaw",
+            },
+        )
+        if not result.get("allowed", True) and self._block:
+            raise GuardrailViolation(result.get("violations", []))
+        return result
+
+    def log_step(
+        self,
+        step_name: str,
+        *,
+        input: str = "",
+        output: str = "",
+        metadata: Optional[Dict[str, Any]] = None,
+    ) -> None:
+        """Log a single step in the agent workflow."""
+        step = {
+            "step": step_name,
+            "input": input[:2000],
+            "output": output[:2000],
+            **(metadata or {}),
+        }
+        self._steps.append(step)
+        self._guard.log_trace(
+            {
+                "provider": "nemoclaw",
+                "agent": self._agent_name,
+                "session_id": self._session_id,
+                "framework": "nemoclaw",
+                **step,
+            }
+        )
+
+    def _finalize(self) -> None:
+        """Log session summary on exit."""
+        self._guard.log_trace(
+            {
+                "provider": "nemoclaw",
+                "agent": self._agent_name,
+                "session_id": self._session_id,
+                "framework": "nemoclaw",
+                "event": "session_end",
+                "total_steps": len(self._steps),
+            }
+        )
+
+
+# Convenience alias
+def init(
+    api_key: str,
+    agent_name: str = "default",
+    **kwargs: Any,
+) -> EvalGuardAgent:
+    """Shorthand for creating an EvalGuardAgent.
+
+    Usage::
+
+        from evalguard.nemoclaw import init
+        agent = init(api_key="eg_...", agent_name="support-bot")
+    """
+    return EvalGuardAgent(api_key=api_key, agent_name=agent_name, **kwargs)

evalguard/openai.py

@@ -0,0 +1,194 @@
+"""Drop-in OpenAI wrapper with EvalGuard guardrails.
+
+Usage::
+
+    from evalguard.openai import wrap
+    from openai import OpenAI
+
+    client = wrap(OpenAI(), api_key="eg_...", project_id="proj_...")
+    # Use exactly like normal -- guardrails are automatic
+    response = client.chat.completions.create(
+        model="gpt-4",
+        messages=[{"role": "user", "content": "Hello"}],
+    )
+"""
+
+from __future__ import annotations
+
+import time
+from typing import Any, List, Optional
+
+from .guardrails import GuardrailClient, GuardrailViolation
+
+
+def wrap(
+    client: Any,
+    *,
+    api_key: str,
+    project_id: Optional[str] = None,
+    base_url: str = "https://api.evalguard.ai",
+    rules: Optional[List[str]] = None,
+    block_on_violation: bool = True,
+    timeout: float = 5.0,
+) -> "_OpenAIProxy":
+    """Wrap an ``openai.OpenAI`` (or ``AsyncOpenAI``) client with guardrails.
+
+    Parameters
+    ----------
+    client:
+        An instantiated ``openai.OpenAI`` or ``openai.AsyncOpenAI`` client.
+    api_key:
+        EvalGuard API key.
+    project_id:
+        Optional EvalGuard project ID for trace grouping.
+    rules:
+        Guardrail rules to apply.  Defaults to prompt-injection + PII.
+    block_on_violation:
+        If *True*, raise :class:`GuardrailViolation` when input is blocked.
+        If *False*, violations are logged but the request proceeds.
+    """
+    guard = GuardrailClient(
+        api_key=api_key,
+        base_url=base_url,
+        project_id=project_id,
+        timeout=timeout,
+    )
+    return _OpenAIProxy(client, guard, rules, block_on_violation)
+
+
+class _OpenAIProxy:
+    """Transparent proxy that intercepts ``chat.completions.create``."""
+
+    __slots__ = ("_client", "_guard", "_rules", "_block")
+
+    def __init__(
+        self,
+        client: Any,
+        guard: GuardrailClient,
+        rules: Optional[List[str]],
+        block: bool,
+    ) -> None:
+        self._client = client
+        self._guard = guard
+        self._rules = rules
+        self._block = block
+
+    def __getattr__(self, name: str) -> Any:
+        attr = getattr(self._client, name)
+        if name == "chat":
+            return _ChatProxy(attr, self._guard, self._rules, self._block)
+        return attr
+
+
+class _ChatProxy:
+    __slots__ = ("_chat", "_guard", "_rules", "_block")
+
+    def __init__(self, chat: Any, guard: GuardrailClient, rules: Optional[List[str]], block: bool) -> None:
+        self._chat = chat
+        self._guard = guard
+        self._rules = rules
+        self._block = block
+
+    def __getattr__(self, name: str) -> Any:
+        attr = getattr(self._chat, name)
+        if name == "completions":
+            return _CompletionsProxy(attr, self._guard, self._rules, self._block)
+        return attr
+
+
+class _CompletionsProxy:
+    __slots__ = ("_completions", "_guard", "_rules", "_block")
+
+    def __init__(self, completions: Any, guard: GuardrailClient, rules: Optional[List[str]], block: bool) -> None:
+        self._completions = completions
+        self._guard = guard
+        self._rules = rules
+        self._block = block
+
+    def __getattr__(self, name: str) -> Any:
+        if name == "create":
+            return self._guarded_create
+        return getattr(self._completions, name)
+
+    def _guarded_create(self, **kwargs: Any) -> Any:
+        messages = kwargs.get("messages", [])
+        prompt_text = _extract_prompt(messages)
+        model = kwargs.get("model", "unknown")
+
+        # ── Pre-LLM check ────────────────────────────────────────────
+        start = time.monotonic()
+        check = self._guard.check_input(
+            prompt_text,
+            rules=self._rules,
+            metadata={"model": model, "framework": "openai"},
+        )
+        guard_ms = (time.monotonic() - start) * 1000
+
+        if not check.get("allowed", True):
+            if self._block:
+                raise GuardrailViolation(check.get("violations", []))
+            # Non-blocking: log but continue
+
+        # ── Call OpenAI ───────────────────────────────────────────────
+        start = time.monotonic()
+        response = self._completions.create(**kwargs)
+        llm_ms = (time.monotonic() - start) * 1000
+
+        # ── Post-LLM trace ───────────────────────────────────────────
+        output_text = _extract_response(response)
+        self._guard.log_trace(
+            {
+                "provider": "openai",
+                "model": model,
+                "input": prompt_text,
+                "output": output_text,
+                "guard_latency_ms": round(guard_ms, 2),
+                "llm_latency_ms": round(llm_ms, 2),
+                "violations": check.get("violations", []),
+                "token_usage": _extract_usage(response),
+            }
+        )
+
+        return response
+
+
+def _extract_prompt(messages: list) -> str:
+    """Join all message contents into a single string for guardrail checking."""
+    parts: list[str] = []
+    for msg in messages:
+        content = msg.get("content", "") if isinstance(msg, dict) else getattr(msg, "content", "")
+        if isinstance(content, str):
+            parts.append(content)
+        elif isinstance(content, list):
+            # Multi-modal messages: extract text parts
+            for part in content:
+                if isinstance(part, dict) and part.get("type") == "text":
+                    parts.append(part.get("text", ""))
+    return "\n".join(parts)
+
+
+def _extract_response(response: Any) -> str:
+    """Extract text from an OpenAI ChatCompletion response."""
+    try:
+        choices = response.choices if hasattr(response, "choices") else response.get("choices", [])
+        if choices:
+            msg = choices[0].message if hasattr(choices[0], "message") else choices[0].get("message", {})
+            return msg.content if hasattr(msg, "content") else msg.get("content", "")
+    except Exception:
+        pass
+    return ""
+
+
+def _extract_usage(response: Any) -> Optional[dict]:
+    """Extract token usage from response."""
+    try:
+        usage = response.usage if hasattr(response, "usage") else response.get("usage")
+        if usage:
+            return {
+                "prompt_tokens": getattr(usage, "prompt_tokens", None) or usage.get("prompt_tokens"),
+                "completion_tokens": getattr(usage, "completion_tokens", None) or usage.get("completion_tokens"),
+                "total_tokens": getattr(usage, "total_tokens", None) or usage.get("total_tokens"),
+            }
+    except Exception:
+        pass
+    return None