evalguard-python 1.1.0__py3-none-any.whl

evalguard/types.py

@@ -0,0 +1,142 @@
+"""EvalGuard Python SDK — Domain types matching the TypeScript definitions."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional
+
+
+# ── Enums as string literals ──
+
+EvalStatus = str  # "pending" | "running" | "passed" | "failed" | "error"
+Severity = str  # "critical" | "high" | "medium" | "low" | "info"
+PlanTier = str  # "free" | "pro" | "team" | "enterprise"
+
+
+@dataclass
+class TokenUsage:
+    prompt: int
+    completion: int
+    total: int
+
+
+@dataclass
+class EvalCase:
+    id: str
+    eval_run_id: str
+    input: str
+    expected_output: Optional[str] = None
+    actual_output: Optional[str] = None
+    score: Optional[float] = None
+    passed: Optional[bool] = None
+    latency: Optional[float] = None
+    token_usage: Optional[TokenUsage] = None
+    metadata: Dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class EvalRun:
+    id: str
+    project_id: str
+    name: str
+    status: EvalStatus
+    score: Optional[float]
+    max_score: float
+    duration: Optional[float]
+    created_at: str
+    completed_at: Optional[str] = None
+    metadata: Dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class CaseResult:
+    input: str
+    actual_output: str
+    score: float
+    passed: bool
+    latency: float
+    expected_output: Optional[str] = None
+    scorer_results: Dict[str, Any] = field(default_factory=dict)
+    token_usage: Optional[TokenUsage] = None
+
+
+@dataclass
+class EvalResult:
+    cases: List[CaseResult]
+    score: float
+    max_score: float
+    pass_rate: float
+    total_latency: float
+    total_tokens: int
+
+
+@dataclass
+class SecurityFinding:
+    id: str
+    scan_id: str
+    type: str
+    severity: Severity
+    title: str
+    description: str
+    input: str
+    output: str
+    passed: bool = True
+    plugin_id: Optional[str] = None
+    strategy_id: Optional[str] = None
+    metadata: Dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class SecurityScanResult:
+    findings: List[SecurityFinding]
+    pass_rate: float
+    critical_count: int
+    high_count: int
+    medium_count: int
+    low_count: int
+    total_tests: int
+    duration: float
+
+
+@dataclass
+class FirewallResult:
+    action: str  # "allow" | "block" | "flag"
+    reasons: List[Dict[str, Any]]
+    latency_ms: float
+
+
+@dataclass
+class FirewallRule:
+    id: str
+    name: str
+    type: str  # "pii" | "injection" | "toxic" | "topic" | "custom"
+    enabled: bool
+    config: Dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class ComplianceReport:
+    framework: str
+    total_controls: int
+    tested_controls: int
+    passed_controls: int
+    failed_controls: int
+    coverage: float
+    findings: List[Dict[str, Any]]
+
+
+@dataclass
+class DriftReport:
+    has_drift: bool
+    overall_delta: float
+    metric_deltas: List[Dict[str, Any]]
+    alerts: List[str]
+
+
+@dataclass
+class BenchmarkResult:
+    suite: str
+    model: str
+    score: float
+    cases: List[Dict[str, Any]]
+    duration: float

evalguard_python-1.1.0.dist-info/METADATA

@@ -0,0 +1,362 @@
+Metadata-Version: 2.4
+Name: evalguard-python
+Version: 1.1.0
+Summary: Python SDK for EvalGuard -- evaluate, red-team, and guard LLM applications with drop-in framework integrations
+Home-page: https://github.com/EvalGuardAi/evalguard
+Author: EvalGuard
+Author-email: EvalGuard <support@evalguard.ai>
+License: MIT
+Project-URL: Homepage, https://evalguard.ai
+Project-URL: Repository, https://github.com/EvalGuardAi/evalguard
+Project-URL: Documentation, https://docs.evalguard.ai/python-sdk
+Project-URL: Issues, https://github.com/EvalGuardAi/evalguard/issues
+Project-URL: Changelog, https://github.com/EvalGuardAi/evalguard/releases
+Keywords: llm,evaluation,ai,security,red-team,prompt-injection,guardrails,ai-safety,llm-security,agent-evaluation,monitoring,evalguard,openai,anthropic,langchain,bedrock,crewai,fastapi
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Security
+Classifier: Topic :: Software Development :: Testing
+Classifier: Typing :: Typed
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+Requires-Dist: requests>=2.28.0
+Provides-Extra: openai
+Requires-Dist: openai>=1.0.0; extra == "openai"
+Provides-Extra: anthropic
+Requires-Dist: anthropic>=0.18.0; extra == "anthropic"
+Provides-Extra: langchain
+Requires-Dist: langchain-core>=0.1.0; extra == "langchain"
+Provides-Extra: crewai
+Requires-Dist: crewai>=0.1.0; extra == "crewai"
+Provides-Extra: bedrock
+Requires-Dist: boto3>=1.28.0; extra == "bedrock"
+Provides-Extra: fastapi
+Requires-Dist: fastapi>=0.100.0; extra == "fastapi"
+Provides-Extra: all
+Requires-Dist: openai>=1.0.0; extra == "all"
+Requires-Dist: anthropic>=0.18.0; extra == "all"
+Requires-Dist: langchain-core>=0.1.0; extra == "all"
+Requires-Dist: crewai>=0.1.0; extra == "all"
+Requires-Dist: boto3>=1.28.0; extra == "all"
+Requires-Dist: fastapi>=0.100.0; extra == "all"
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == "dev"
+Requires-Dist: pytest-mock>=3.10; extra == "dev"
+Requires-Dist: responses>=0.23; extra == "dev"
+Dynamic: author
+Dynamic: home-page
+Dynamic: requires-python
+
+# evalguard-python
+
+[![PyPI version](https://img.shields.io/pypi/v/evalguard-python.svg)](https://pypi.org/project/evalguard-python/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+[![Python 3.9+](https://img.shields.io/badge/python-3.9+-blue.svg)](https://www.python.org/downloads/)
+
+Python SDK for [EvalGuard](https://evalguard.ai) -- evaluate, red-team, and guard LLM applications with **drop-in framework integrations**.
+
+## Installation
+
+```bash
+# Core SDK
+pip install evalguard-python
+
+# With framework extras
+pip install evalguard-python[openai]
+pip install evalguard-python[anthropic]
+pip install evalguard-python[langchain]
+pip install evalguard-python[bedrock]
+pip install evalguard-python[crewai]
+pip install evalguard-python[fastapi]
+
+# Everything
+pip install evalguard-python[all]
+```
+
+## Quick Start
+
+```python
+from evalguard import EvalGuardClient
+
+client = EvalGuardClient(api_key="eg_live_...")
+
+# Run an evaluation
+result = client.run_eval({
+    "model": "gpt-4o",
+    "prompt": "Answer: {{input}}",
+    "cases": [
+        {"input": "What is 2+2?", "expectedOutput": "4"},
+    ],
+    "scorers": ["exact-match", "contains"],
+})
+print(f"Score: {result['score']}, Pass rate: {result['passRate']}")
+
+# Check the firewall
+fw = client.check_firewall("Ignore all previous instructions")
+print(f"Action: {fw['action']}")  # "block"
+```
+
+---
+
+## Framework Integrations
+
+Every integration is a **drop-in wrapper** -- add two lines and your existing code gets automatic guardrails, traces, and observability.
+
+### OpenAI
+
+```python
+from evalguard.openai import wrap
+from openai import OpenAI
+
+client = wrap(OpenAI(), api_key="eg_...", project_id="proj_...")
+
+# Use exactly like normal -- guardrails are automatic
+response = client.chat.completions.create(
+    model="gpt-4o",
+    messages=[{"role": "user", "content": "Hello, world!"}],
+)
+print(response.choices[0].message.content)
+```
+
+All calls to `chat.completions.create()` are intercepted:
+- **Pre-LLM**: Input is checked for prompt injection, PII, etc.
+- **Post-LLM**: Response + latency + token usage are traced to EvalGuard.
+- **Violations**: Raise `GuardrailViolation` (or log-only with `block_on_violation=False`).
+
+### Anthropic
+
+```python
+from evalguard.anthropic import wrap
+from anthropic import Anthropic
+
+client = wrap(Anthropic(), api_key="eg_...", project_id="proj_...")
+
+response = client.messages.create(
+    model="claude-sonnet-4-20250514",
+    max_tokens=1024,
+    messages=[{"role": "user", "content": "Explain quantum computing"}],
+)
+print(response.content[0].text)
+```
+
+Intercepts `messages.create()` with the same pre/post guardrail pattern.
+
+### LangChain
+
+```python
+from evalguard.langchain import EvalGuardCallback
+from langchain_openai import ChatOpenAI
+
+callback = EvalGuardCallback(api_key="eg_...", project_id="proj_...")
+
+llm = ChatOpenAI(model="gpt-4o", callbacks=[callback])
+result = llm.invoke("What is the capital of France?")
+```
+
+Works with **any** LangChain LLM, chat model, or chain that supports callbacks. The callback implements the full LangChain callback protocol without importing LangChain, so it is compatible with all versions (0.1.x through 0.3.x).
+
+Traced events:
+- `on_llm_start` / `on_chat_model_start` -- pre-check input
+- `on_llm_end` -- log output trace
+- `on_llm_error` -- log error trace
+
+### AWS Bedrock
+
+```python
+from evalguard.bedrock import wrap
+import boto3
+
+bedrock = boto3.client("bedrock-runtime", region_name="us-east-1")
+client = wrap(bedrock, api_key="eg_...", project_id="proj_...")
+
+# invoke_model (all Bedrock model families supported)
+import json
+response = client.invoke_model(
+    modelId="anthropic.claude-3-sonnet-20240229-v1:0",
+    body=json.dumps({
+        "messages": [{"role": "user", "content": "Hello"}],
+        "max_tokens": 256,
+        "anthropic_version": "bedrock-2023-05-31",
+    }),
+)
+
+# Converse API
+response = client.converse(
+    modelId="anthropic.claude-3-sonnet-20240229-v1:0",
+    messages=[{"role": "user", "content": [{"text": "Hello"}]}],
+)
+```
+
+Supports all Bedrock model families: Anthropic Claude, Amazon Titan, Meta Llama, Cohere, AI21, and Mistral. Both `invoke_model` and `converse` APIs are guarded.
+
+### CrewAI
+
+```python
+from evalguard.crewai import guard_agent, EvalGuardGuardrail
+from crewai import Agent, Task, Crew
+
+# Guard individual agents
+agent = Agent(role="researcher", goal="...", backstory="...")
+agent = guard_agent(agent, api_key="eg_...")
+
+# Or use the standalone guardrail
+guardrail = EvalGuardGuardrail(api_key="eg_...", project_id="proj_...")
+result = guardrail.check("User input to validate")
+
+# Wrap arbitrary functions
+@guardrail.wrap_function
+def my_tool(query: str) -> str:
+    return do_search(query)
+```
+
+### FastAPI Middleware
+
+```python
+from evalguard.fastapi import EvalGuardMiddleware
+from fastapi import FastAPI
+
+app = FastAPI()
+app.add_middleware(
+    EvalGuardMiddleware,
+    api_key="eg_...",
+    project_id="proj_...",
+)
+
+@app.post("/api/chat")
+async def chat(request: dict):
+    # Automatically guarded -- prompt injection blocked with 403
+    return {"response": "..."}
+```
+
+By default, POST requests to paths containing `/chat`, `/completions`, `/generate`, `/invoke`, or `/messages` are guarded. Customize with `guarded_paths`:
+
+```python
+app.add_middleware(
+    EvalGuardMiddleware,
+    api_key="eg_...",
+    guarded_paths={"/api/v1/chat", "/api/v1/generate"},
+)
+```
+
+For per-route control:
+
+```python
+from evalguard.fastapi import guard_route
+
+@app.post("/api/chat")
+@guard_route(api_key="eg_...", rules=["prompt_injection"])
+async def chat(request: Request):
+    body = await request.json()
+    ...
+```
+
+### NeMo / Agent Workflows
+
+```python
+from evalguard.nemoclaw import EvalGuardAgent
+
+agent = EvalGuardAgent(api_key="eg_...", agent_name="support-bot")
+
+# Guard any LLM call
+result = agent.guarded_call(
+    provider="openai",
+    messages=[{"role": "user", "content": "Reset my password"}],
+    llm_fn=lambda: openai_client.chat.completions.create(
+        model="gpt-4", messages=[{"role": "user", "content": "Reset my password"}]
+    ),
+)
+
+# Multi-step agent sessions
+with agent.session("ticket-123") as session:
+    session.check("User says: reset my password")
+    result = do_llm_call(...)
+    session.log_step("password_reset", input="...", output=str(result))
+```
+
+---
+
+## Core Guardrail Client
+
+All framework integrations share the same underlying `GuardrailClient`:
+
+```python
+from evalguard.guardrails import GuardrailClient
+
+guard = GuardrailClient(
+    api_key="eg_...",
+    project_id="proj_...",
+    timeout=5.0,       # keep low to avoid latency
+    fail_open=True,    # allow on error (default)
+)
+
+# Pre-LLM check
+result = guard.check_input("user prompt here", rules=["prompt_injection", "pii_redact"])
+if not result["allowed"]:
+    print("Blocked:", result["violations"])
+
+# Post-LLM check
+result = guard.check_output("model response here", rules=["toxic_content"])
+
+# Fire-and-forget trace
+guard.log_trace({"model": "gpt-4", "input": "...", "output": "...", "latency_ms": 120})
+```
+
+## Error Handling
+
+All integrations use **fail-open** semantics by default: if the EvalGuard API is unreachable, requests pass through rather than blocking your application.
+
+To fail-closed:
+
+```python
+# Framework wrappers
+client = wrap(OpenAI(), api_key="eg_...", block_on_violation=True)
+
+# Core client
+guard = GuardrailClient(api_key="eg_...", fail_open=False)
+```
+
+Catch violations explicitly:
+
+```python
+from evalguard import GuardrailViolation
+
+try:
+    response = client.chat.completions.create(...)
+except GuardrailViolation as e:
+    print(f"Blocked: {e.violations}")
+```
+
+## All SDK Methods
+
+| Method | Description |
+|---|---|
+| `client.run_eval(config)` | Run an evaluation with scorers and test cases |
+| `client.get_eval(run_id)` | Fetch a specific eval run by ID |
+| `client.list_evals(project_id=None)` | List eval runs, optionally filtered by project |
+| `client.run_scan(config)` | Run a red-team security scan against a model |
+| `client.get_scan(scan_id)` | Fetch a specific security scan by ID |
+| `client.list_scorers()` | List all available evaluation scorers |
+| `client.list_plugins()` | List all available security plugins |
+| `client.check_firewall(input_text, rules=None)` | Check input against firewall rules |
+| `client.run_benchmarks(suites, model)` | Run benchmark suites against a model |
+| `client.export_dpo(run_id)` | Export eval results as DPO training data (JSONL) |
+| `client.export_burp(scan_id)` | Export scan results as Burp Suite XML |
+| `client.get_compliance_report(scan_id, framework)` | Map scan results to a compliance framework |
+| `client.detect_drift(config)` | Detect performance drift between eval runs |
+| `client.generate_guardrails(config)` | Auto-generate firewall rules from scan findings |
+
+## Documentation
+
+Full documentation at [docs.evalguard.ai/python-sdk](https://docs.evalguard.ai/python-sdk).
+
+## License
+
+MIT -- see [LICENSE](./LICENSE) for details.

evalguard_python-1.1.0.dist-info/RECORD

@@ -0,0 +1,15 @@
+evalguard/__init__.py,sha256=IAIMbOw7JydE_MzegoSDFXVmQj8jkzrkqXwCLAuU-78,895
+evalguard/anthropic.py,sha256=XlPho-PKuniJGcGMEJQh9qAxO5b2kn90-TNpORaygRw,6490
+evalguard/bedrock.py,sha256=6_uGpZOW6mATekUIo2tu4zRIc7pXyYhEljTGjECtqFQ,10050
+evalguard/client.py,sha256=txQ3IUD9-ltbhyzU22Aije9LZ8AP20Ze-pT05pkXI7E,25594
+evalguard/crewai.py,sha256=RurHWDl0t0S615Zv5MSHZn2V1MZmUFdRvokOs_h4CLo,5643
+evalguard/fastapi.py,sha256=RedY-i2a-UWw8AcoGgX1rQYKYgZZWTOBsqSKz7nStp0,8921
+evalguard/guardrails.py,sha256=pNf0nSe5gI2DzraPuySTMjaQFun_fXsTSu8lOlTtbSc,5376
+evalguard/langchain.py,sha256=sIO3lP9_eNRIVz_y0b_owkU2Vsy_Jpmjk77hJkBBwag,7482
+evalguard/nemoclaw.py,sha256=DxGHXHvuaPVZWvutvJHca-zdwiu99OLyV2yTsp-jt8Q,7919
+evalguard/openai.py,sha256=SLpnS__233jY6xyhSXXSdqARoeSBZBLa_nmhCvtAZsg,6753
+evalguard/types.py,sha256=6y2L9dC9OhEZPQulSkewZAARo0R5pknI40Ek-7mlbeI,3139
+evalguard_python-1.1.0.dist-info/METADATA,sha256=nwc5Jx-bgSa9Aob_7CMqhDrmQLKWQnAxZe-nbtGvr84,11507
+evalguard_python-1.1.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+evalguard_python-1.1.0.dist-info/top_level.txt,sha256=BUIQhUkmgN0RSompUln4ahg7wOdGcO8h06-vwk_I5xA,10
+evalguard_python-1.1.0.dist-info/RECORD,,

evalguard_python-1.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

evalguard_python-1.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+evalguard