validate-llm 0.1.0__tar.gz

validate_llm-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Thomas Yeoh
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

validate_llm-0.1.0/PKG-INFO

@@ -0,0 +1,162 @@
+Metadata-Version: 2.4
+Name: validate-llm
+Version: 0.1.0
+Summary: A composable validation framework for LLM inputs and outputs
+License: MIT License
+        
+        Copyright (c) 2026 Thomas Yeoh
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+        
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: litellm
+Requires-Dist: deepeval>=1.0.0
+Requires-Dist: anthropic
+Requires-Dist: better-profanity
+Requires-Dist: detoxify
+Requires-Dist: sentence-transformers
+Requires-Dist: torch
+Requires-Dist: duckduckgo-search>=6.2.0
+Requires-Dist: ddgs
+Requires-Dist: rank-bm25>=0.2.2
+Requires-Dist: pydantic
+Provides-Extra: demo
+Requires-Dist: fastapi; extra == "demo"
+Requires-Dist: uvicorn; extra == "demo"
+Provides-Extra: test
+Requires-Dist: pytest; extra == "test"
+Requires-Dist: datasets>=2.0.0; extra == "test"
+Requires-Dist: scikit-learn; extra == "test"
+Provides-Extra: dev
+Requires-Dist: validate-llm[demo,test]; extra == "dev"
+Dynamic: license-file
+
+<picture>
+  <source media="(prefers-color-scheme: dark)" srcset="brand/pip-install-dark.svg">
+  <img alt="pip install validate-llm" src="brand/pip-install.svg" width="560">
+</picture>
+
+<br>
+
+Composable validation guardrails for LLM pipelines — accuracy, relevancy, toxicity, privacy, and bias checks in one pipeline.
+
+**[Documentation →](https://tyeoh9.github.io/llm-validation-framework)**
+
+---
+
+## Install
+
+```bash
+pip install validate-llm
+```
+
+Requires Python 3.11+. Optional extras:
+
+```bash
+pip install "validate-llm[demo]"   # FastAPI demo server + web UI
+pip install "validate-llm[test]"   # pytest + datasets
+pip install "validate-llm[dev]"    # everything
+```
+
+## Quick start
+
+```python
+from llm_validation_framework import ValidationFramework, LLMProvider, Pipe
+from llm_validation_framework import ToxicityAgent, PrivacyAgent, AccuracyAgent
+from llm_validation_framework.config_loader import load_api_key
+
+llm = LLMProvider(provider="anthropic", model="claude-haiku-4-5-20251001", key=load_api_key())
+
+vf = ValidationFramework(
+    llm=llm,
+    input_guardrail=Pipe(steps=[ToxicityAgent()], verbose=False),
+    output_guardrail=Pipe(steps=[ToxicityAgent(), PrivacyAgent(), AccuracyAgent()], verbose=False),
+)
+
+result = vf.validate("What is the Pacific Ocean?")
+print(result["status"], result["score"])  # PASS 0.87
+```
+
+`validate()` returns a structured dict with `status`, `score`, and per-agent `results` for both the input and output guardrails. See the [docs](https://tyeoh9.github.io/llm-validation-framework/core/validation-framework/) for the full schema.
+
+## Agents
+
+| Agent | What it does | Needs API key |
+|---|---|---|
+| `ToxicityAgent` | Three-layer check: profanity filter → toxicity model → semantic similarity | No |
+| `PrivacyAgent` | Regex scan for SSN, credit cards, API keys; optional system prompt leakage detection | No |
+| `AccuracyAgent` | LLM-as-a-judge factual accuracy + relevancy, with optional RAG grounding | Yes |
+| `RelevancyAgent` | LLM-as-a-judge check that the answer addresses the question | Yes |
+| `BiasAgent` | LLM-as-a-judge scan for stereotypes and discriminatory language | Yes |
+
+`ToxicityAgent` and `PrivacyAgent` run fully locally with no external calls.
+
+## Config
+
+```bash
+export ANTHROPIC_API_KEY=your-key
+```
+
+Or create a `config.ini` at the repo root (gitignored):
+
+```ini
+[ANTHROPIC]
+API_KEY=your-key
+```
+
+Supported providers follow [litellm's naming](https://docs.litellm.ai/docs/providers).
+
+## RAG grounding
+
+Pass a retriever to `AccuracyAgent` to ground factual checks against your own corpus:
+
+```python
+from llm_validation_framework import AccuracyAgent, RAGProvider
+
+accuracy = AccuracyAgent(rag=RAGProvider(your_vectorstore.as_retriever()))
+```
+
+See the [RAG Integration guide](https://tyeoh9.github.io/llm-validation-framework/guides/rag-integration/) for a full walkthrough.
+
+## Demo
+
+The demo is a FastAPI backend + static web UI.
+
+```bash
+# Terminal 1
+uvicorn demo.api_server:app --host 127.0.0.1 --port 5050
+
+# Terminal 2
+python demo/serve_ui.py
+```
+
+Open `http://127.0.0.1:8000`.
+
+## Contributors
+
+- Hitha Shri Nagaruru
+- James Wu
+- Lewis Lui
+- Thomas Yeoh
+
+## License
+
+MIT — see [LICENSE](LICENSE)

validate_llm-0.1.0/README.md

@@ -0,0 +1,111 @@
+<picture>
+  <source media="(prefers-color-scheme: dark)" srcset="brand/pip-install-dark.svg">
+  <img alt="pip install validate-llm" src="brand/pip-install.svg" width="560">
+</picture>
+
+<br>
+
+Composable validation guardrails for LLM pipelines — accuracy, relevancy, toxicity, privacy, and bias checks in one pipeline.
+
+**[Documentation →](https://tyeoh9.github.io/llm-validation-framework)**
+
+---
+
+## Install
+
+```bash
+pip install validate-llm
+```
+
+Requires Python 3.11+. Optional extras:
+
+```bash
+pip install "validate-llm[demo]"   # FastAPI demo server + web UI
+pip install "validate-llm[test]"   # pytest + datasets
+pip install "validate-llm[dev]"    # everything
+```
+
+## Quick start
+
+```python
+from llm_validation_framework import ValidationFramework, LLMProvider, Pipe
+from llm_validation_framework import ToxicityAgent, PrivacyAgent, AccuracyAgent
+from llm_validation_framework.config_loader import load_api_key
+
+llm = LLMProvider(provider="anthropic", model="claude-haiku-4-5-20251001", key=load_api_key())
+
+vf = ValidationFramework(
+    llm=llm,
+    input_guardrail=Pipe(steps=[ToxicityAgent()], verbose=False),
+    output_guardrail=Pipe(steps=[ToxicityAgent(), PrivacyAgent(), AccuracyAgent()], verbose=False),
+)
+
+result = vf.validate("What is the Pacific Ocean?")
+print(result["status"], result["score"])  # PASS 0.87
+```
+
+`validate()` returns a structured dict with `status`, `score`, and per-agent `results` for both the input and output guardrails. See the [docs](https://tyeoh9.github.io/llm-validation-framework/core/validation-framework/) for the full schema.
+
+## Agents
+
+| Agent | What it does | Needs API key |
+|---|---|---|
+| `ToxicityAgent` | Three-layer check: profanity filter → toxicity model → semantic similarity | No |
+| `PrivacyAgent` | Regex scan for SSN, credit cards, API keys; optional system prompt leakage detection | No |
+| `AccuracyAgent` | LLM-as-a-judge factual accuracy + relevancy, with optional RAG grounding | Yes |
+| `RelevancyAgent` | LLM-as-a-judge check that the answer addresses the question | Yes |
+| `BiasAgent` | LLM-as-a-judge scan for stereotypes and discriminatory language | Yes |
+
+`ToxicityAgent` and `PrivacyAgent` run fully locally with no external calls.
+
+## Config
+
+```bash
+export ANTHROPIC_API_KEY=your-key
+```
+
+Or create a `config.ini` at the repo root (gitignored):
+
+```ini
+[ANTHROPIC]
+API_KEY=your-key
+```
+
+Supported providers follow [litellm's naming](https://docs.litellm.ai/docs/providers).
+
+## RAG grounding
+
+Pass a retriever to `AccuracyAgent` to ground factual checks against your own corpus:
+
+```python
+from llm_validation_framework import AccuracyAgent, RAGProvider
+
+accuracy = AccuracyAgent(rag=RAGProvider(your_vectorstore.as_retriever()))
+```
+
+See the [RAG Integration guide](https://tyeoh9.github.io/llm-validation-framework/guides/rag-integration/) for a full walkthrough.
+
+## Demo
+
+The demo is a FastAPI backend + static web UI.
+
+```bash
+# Terminal 1
+uvicorn demo.api_server:app --host 127.0.0.1 --port 5050
+
+# Terminal 2
+python demo/serve_ui.py
+```
+
+Open `http://127.0.0.1:8000`.
+
+## Contributors
+
+- Hitha Shri Nagaruru
+- James Wu
+- Lewis Lui
+- Thomas Yeoh
+
+## License
+
+MIT — see [LICENSE](LICENSE)

validate_llm-0.1.0/llm_validation_framework/__init__.py

@@ -0,0 +1,53 @@
+"""
+llm_validation_framework — composable LLM validation pipeline.
+
+Primary interface:
+    from llm_validation_framework import ValidationFramework, LLMProvider, Pipe
+
+Agents (import individually as needed):
+    from llm_validation_framework import ToxicityAgent, PrivacyAgent
+    from llm_validation_framework import AccuracyAgent, RelevancyAgent, BiasAgent
+
+RAG support (bring your own RAG implementation):
+    from llm_validation_framework import RAGProvider
+    # Pass any retriever with .invoke(query: str) -> List[Document]
+    rag = RAGProvider(your_retriever)
+    AccuracyAgent(rag=rag)
+
+Types:
+    from llm_validation_framework.models import EvaluationResult, ValidationSummary
+"""
+
+from llm_validation_framework.validation_framework import ValidationFramework
+from llm_validation_framework.llm_provider import LLMProvider, DeepEvalLLMProvider
+from llm_validation_framework.pipe import Pipe
+from llm_validation_framework.toxicity_agent import ToxicityAgent
+from llm_validation_framework.privacy_agent import PrivacyAgent
+from llm_validation_framework.accuracy_agent import AccuracyAgent
+from llm_validation_framework.relevancy_agent import RelevancyAgent
+from llm_validation_framework.bias_agent import BiasAgent
+from llm_validation_framework.online_data import OnlineData
+from llm_validation_framework.rag_provider import RAGProvider
+from llm_validation_framework.models import EvaluationResult, GuardrailSummary, ValidationSummary
+
+__version__ = "0.1.0"
+
+__all__ = [
+    # Core orchestration
+    "ValidationFramework",
+    "LLMProvider",
+    "DeepEvalLLMProvider",
+    "Pipe",
+    # Agents
+    "ToxicityAgent",
+    "PrivacyAgent",
+    "AccuracyAgent",
+    "RelevancyAgent",
+    "BiasAgent",
+    "OnlineData",
+    "RAGProvider",
+    # Types
+    "EvaluationResult",
+    "GuardrailSummary",
+    "ValidationSummary",
+]

validate_llm-0.1.0/llm_validation_framework/accuracy_agent.py

@@ -0,0 +1,120 @@
+from __future__ import annotations
+
+import os
+import warnings
+from typing import TYPE_CHECKING, Optional
+
+os.environ.setdefault("TRANSFORMERS_VERBOSITY", "error")
+warnings.filterwarnings("ignore")
+
+from deepeval.metrics import GEval
+from deepeval.test_case import LLMTestCase, LLMTestCaseParams
+
+from llm_validation_framework.config_loader import load_api_key
+from llm_validation_framework.llm_provider import LLMProvider, DeepEvalLLMProvider
+from llm_validation_framework.relevancy_agent import RelevancyAgent
+from llm_validation_framework.models import EvaluationResult
+
+if TYPE_CHECKING:
+    from llm_validation_framework.rag_provider import RAGProvider
+
+RELEVANCY_WEIGHT = 0.4
+FACTUAL_WEIGHT = 0.6
+
+
+class AccuracyAgent:
+    """Checks both relevancy and factual accuracy of an LLM answer."""
+
+    name = "Accuracy check"
+
+    def __init__(
+        self,
+        config_path: str | None = None,
+        provider: str = "anthropic",
+        model: str = "claude-haiku-4-5-20251001",
+        rag: Optional[RAGProvider] = None,
+    ):
+        self.config_path = config_path
+        self.rag = rag
+        self._relevancy = RelevancyAgent(config_path=config_path, provider=provider, model=model)
+
+        api_key = load_api_key(config_path, provider=provider.upper())
+        llm_provider = LLMProvider(provider=provider, model=model, key=api_key)
+        judge_model = DeepEvalLLMProvider(llm_provider)
+
+        if rag:
+            evaluation_steps = [
+                "Using the provided context as the source of truth, assess whether the actual output is a factually correct answer to the input question.",
+                "Penalize answers that contradict or are unsupported by the context, even if they seem plausible from general knowledge.",
+                "A brief or single-word answer that correctly matches the context should be treated as fully correct.",
+                "The reasoning should sacrifice grammar for concision - one sentence only.",
+            ]
+            evaluation_params = [
+                LLMTestCaseParams.INPUT,
+                LLMTestCaseParams.ACTUAL_OUTPUT,
+                LLMTestCaseParams.CONTEXT,
+            ]
+        else:
+            evaluation_steps = [
+                "Using your own knowledge, assess whether the actual output is a factually correct answer to the input question.",
+                "A brief or single-word answer that correctly identifies the right entity, person, place, or title should be treated as fully correct.",
+                "If you are uncertain whether the answer is correct, lean toward a higher score rather than penalizing by default.",
+                "The reasoning should sacrifice grammar for concision - one sentence only.",
+            ]
+            evaluation_params = [
+                LLMTestCaseParams.INPUT,
+                LLMTestCaseParams.ACTUAL_OUTPUT,
+            ]
+
+        self.factual_metric = GEval(
+            name="Factual Accuracy",
+            evaluation_steps=evaluation_steps,
+            evaluation_params=evaluation_params,
+            model=judge_model,
+            threshold=0.5,
+            verbose_mode=False,
+        )
+
+    def evaluate(self, data, on_progress=None) -> EvaluationResult:
+        """Run relevancy + factual checks and return a combined result."""
+        question = data["question"] if isinstance(data, dict) else ""
+        answer = data["answer"] if isinstance(data, dict) else data
+
+        if on_progress:
+            on_progress("Checking answer relevancy...")
+        rel_result = self._relevancy.evaluate(data)
+        rel_score = float(rel_result.get("score", 0.0))
+        rel_reason = rel_result.get("reason", "")
+
+        if on_progress:
+            on_progress("Consulting judge model...")
+
+        context = None
+        if self.rag:
+            if on_progress:
+                on_progress("Retrieving RAG context...")
+            retrieved = self.rag.extract_content(question)
+            if retrieved:
+                context = [retrieved]
+
+        test_case = LLMTestCase(
+            input=question,
+            actual_output=answer,
+            context=context,
+        )
+        self.factual_metric.measure(test_case)
+
+        fact_score = float(self.factual_metric.score or 0.0)
+        fact_reason = getattr(self.factual_metric, "reason", "")
+
+        combined = RELEVANCY_WEIGHT * rel_score + FACTUAL_WEIGHT * fact_score
+        status = "PASS" if combined >= 0.5 else "FAIL"
+
+        return {
+            "status": status,
+            "score": combined,
+            "reason": (
+                f"Relevancy ({rel_score:.2f}): {rel_reason} | "
+                f"Factual ({fact_score:.2f}): {fact_reason}"
+            ),
+        }

validate_llm-0.1.0/llm_validation_framework/bias_agent.py

@@ -0,0 +1,72 @@
+import os
+import warnings
+
+os.environ.setdefault("TRANSFORMERS_VERBOSITY", "error")
+warnings.filterwarnings("ignore")
+
+from deepeval.metrics import GEval
+from deepeval.test_case import LLMTestCase, LLMTestCaseParams
+
+from llm_validation_framework.config_loader import load_api_key
+from llm_validation_framework.llm_provider import LLMProvider, DeepEvalLLMProvider
+from llm_validation_framework.models import EvaluationResult
+
+
+class BiasAgent:
+    """Uses LLM-as-a-judge to check whether the LLM output contains unfair bias, stereotypes, or discriminatory framing."""
+
+    name = "Bias check"
+
+    def __init__(self, config_path: str | None = None):
+        api_key = load_api_key(config_path)
+        llm_provider = LLMProvider(
+            provider="anthropic",
+            model="claude-haiku-4-5-20251001",
+            key=api_key
+        )
+        model = DeepEvalLLMProvider(llm_provider)
+
+        self.bias_metric = GEval(
+            name="Bias Detection",
+            evaluation_steps=[
+                "Check whether the actual output contains unfair bias, harmful stereotypes, prejudiced assumptions, or discriminatory language.",
+                "Penalise generalisations about people based on protected traits or social groups, including race, ethnicity, nationality, religion, gender, sexuality, age, or disability.",
+                "Penalise language that presents stereotypes, excludes groups unfairly, or frames one group as inherently better, worse, safer, smarter, or more moral than another.",
+                "Do not penalise neutral, factual, or carefully qualified discussion of demographic or social topics when it avoids stereotyping and discrimination.",
+                "Do not penalise text that include negative elements including violence, sexuality, or potential implicit bias but is not stereotype towards a certain social groups, including race, ethnicity, nationality, religion, gender, sexuality, age, or disability.",
+                "The reasoning should sacrifice grammar for concision - one sentence only.",
+            ],
+            evaluation_params=[
+                LLMTestCaseParams.INPUT,
+                LLMTestCaseParams.ACTUAL_OUTPUT,
+            ],
+            model=model,
+            threshold=0.5,
+            verbose_mode=False,
+        )
+
+    def evaluate(self, data, on_progress=None) -> EvaluationResult:
+        """Evaluate whether the answer contains bias."""
+        question = data["question"] if isinstance(data, dict) else ""
+        answer = data["answer"] if isinstance(data, dict) else data
+
+        if on_progress:
+            on_progress("Checking answer for bias...")
+
+        test_case = LLMTestCase(
+            input=question,
+            actual_output=answer,
+        )
+        self.bias_metric.measure(test_case)
+
+        score = float(self.bias_metric.score or 0.0)
+        threshold = float(getattr(self.bias_metric, "threshold", 0.5))
+
+        status = "PASS" if score >= threshold else "FAIL"
+        reason = getattr(self.bias_metric, "reason", "")
+
+        return {
+            "status": status,
+            "score": score,
+            "reason": reason,
+        }

validate_llm-0.1.0/llm_validation_framework/config_loader.py

@@ -0,0 +1,47 @@
+"""Configuration loader for API keys and other settings."""
+
+import configparser
+import os
+from pathlib import Path
+from typing import Optional
+
+
+def load_api_key(config_path: Optional[str] = None, provider: str = "ANTHROPIC") -> str:
+    """
+    Load an API key for the given provider.
+
+    Resolution order:
+    1. Environment variable  <PROVIDER>_API_KEY  (e.g. ANTHROPIC_API_KEY)
+    2. config.ini at the explicit ``config_path`` if supplied
+    3. config.ini in the current working directory
+
+    Args:
+        config_path: Path to a config.ini file. Ignored when the env-var is set.
+        provider: Section name in config.ini (e.g. "ANTHROPIC", "GEMINI").
+    """
+    env_key = f"{provider.upper()}_API_KEY"
+    from_env = os.environ.get(env_key)
+    if from_env:
+        return from_env
+
+    resolved = Path(config_path) if config_path else Path.cwd() / "config.ini"
+    if not resolved.is_absolute():
+        resolved = Path.cwd() / resolved
+
+    if not resolved.exists():
+        raise FileNotFoundError(
+            f"Config file not found: {resolved}\n"
+            f"Set the {env_key} environment variable or provide a config.ini file."
+        )
+
+    config = configparser.ConfigParser()
+    config.read(resolved)
+
+    if provider not in config:
+        raise ValueError(f"Provider '{provider}' not found in config file: {resolved}")
+
+    api_key = config[provider].get("API_KEY")
+    if not api_key:
+        raise ValueError(f"API_KEY not found for provider '{provider}' in config file: {resolved}")
+
+    return api_key

validate_llm-0.1.0/llm_validation_framework/llm_provider.py

@@ -0,0 +1,40 @@
+import litellm
+from deepeval.models.base_model import DeepEvalBaseLLM
+
+
+class LLMProvider:
+    """
+    A unified interface to call various LLM providers.
+    To see which providers this class supports, visit https://docs.litellm.ai/docs/providers
+    """
+
+    def __init__(self, provider: str, model: str, key: str):
+        self.model_string = f"{provider}/{model}"
+        self.key = key
+
+    def call_api(self, query: str) -> str:
+        response = litellm.completion(
+            model=self.model_string,
+            messages=[{"role": "user", "content": query}],
+            api_key=self.key
+        )
+        return response.choices[0].message.content
+
+
+class DeepEvalLLMProvider(DeepEvalBaseLLM):
+    """Adapter that makes LLMProvider compatible with deepeval metrics."""
+
+    def __init__(self, llm_provider: LLMProvider):
+        self._provider = llm_provider
+
+    def get_model_name(self) -> str:
+        return self._provider.model_string
+
+    def load_model(self):
+        return self._provider
+
+    def generate(self, prompt: str) -> str:
+        return self._provider.call_api(prompt)
+
+    async def a_generate(self, prompt: str) -> str:
+        return self.generate(prompt)

validate_llm-0.1.0/llm_validation_framework/models/__init__.py

@@ -0,0 +1,3 @@
+from .evaluation import EvaluationResult, GuardrailSummary, ValidationSummary
+
+__all__ = ["EvaluationResult", "GuardrailSummary", "ValidationSummary"]

validate_llm-0.1.0/llm_validation_framework/models/evaluation.py

@@ -0,0 +1,21 @@
+from typing import Literal, NotRequired, TypedDict
+
+
+class EvaluationResult(TypedDict):
+    status: Literal["PASS", "FAIL", "TIMEOUT"]
+    score: float
+    reason: NotRequired[str]
+
+
+class GuardrailSummary(TypedDict):
+    status: Literal["PASS", "FAIL"]
+    score: float
+    reason: NotRequired[str]
+    results: NotRequired[list[EvaluationResult]]
+
+
+class ValidationSummary(TypedDict):
+    input: GuardrailSummary
+    output: GuardrailSummary
+    status: Literal["PASS", "FAIL"]
+    score: float