ragverdict 0.2.1__py3-none-any.whl

ragverdict/__init__.py

@@ -0,0 +1,30 @@
+"""ragverdict — pytest for RAG agents."""
+
+__version__ = "0.2.1"
+
+from ragverdict.adapters.base import (
+    Citation,
+    ContextDoc,
+    Message,
+    RagAdapter,
+    RagResponse,
+    SourceDoc,
+    ToolCall,
+    ToolSpec,
+)
+from ragverdict.evaluators.base import Evaluator, TestResult, Verdict
+
+__all__ = [
+    "Citation",
+    "ContextDoc",
+    "Evaluator",
+    "Message",
+    "RagAdapter",
+    "RagResponse",
+    "SourceDoc",
+    "TestResult",
+    "ToolCall",
+    "ToolSpec",
+    "Verdict",
+    "__version__",
+]

ragverdict/adapters/__init__.py

@@ -0,0 +1,23 @@
+"""Adapters connect ragverdict to user-defined RAG systems."""
+
+from ragverdict.adapters.base import (
+    Citation,
+    ContextDoc,
+    Message,
+    RagAdapter,
+    RagResponse,
+    SourceDoc,
+    ToolCall,
+    ToolSpec,
+)
+
+__all__ = [
+    "Citation",
+    "ContextDoc",
+    "Message",
+    "RagAdapter",
+    "RagResponse",
+    "SourceDoc",
+    "ToolCall",
+    "ToolSpec",
+]

ragverdict/adapters/base.py

@@ -0,0 +1,92 @@
+"""Core protocol types every RAG adapter speaks."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from collections.abc import Iterable
+from dataclasses import dataclass, field
+from typing import Any, Literal
+
+
+@dataclass
+class Message:
+    role: Literal["user", "assistant", "system"]
+    content: str
+
+
+@dataclass
+class Citation:
+    """A reference from a response back to a source document."""
+
+    id: str
+    source_id: str
+    span: str = ""
+
+
+@dataclass
+class ToolCall:
+    """A tool invocation the agent made while answering."""
+
+    name: str
+    args: dict[str, Any] = field(default_factory=dict)
+    result: Any = None
+    latency_ms: int = 0
+    error: str | None = None
+
+
+@dataclass
+class ContextDoc:
+    """A retrieved chunk the agent considered while answering."""
+
+    source_id: str
+    chunk: str
+    score: float = 0.0
+
+
+@dataclass
+class ToolSpec:
+    """Describes a tool the adapter exposes for tool_coverage testing."""
+
+    name: str
+    description: str = ""
+    trigger_prompt: str | None = None
+
+
+@dataclass
+class SourceDoc:
+    """A document in the adapter's source corpus for citation_audit."""
+
+    source_id: str
+    content: str
+    title: str = ""
+
+
+@dataclass
+class RagResponse:
+    """What an adapter returns from `query`."""
+
+    text: str
+    citations: list[Citation] = field(default_factory=list)
+    tool_calls: list[ToolCall] = field(default_factory=list)
+    retrieved_context: list[ContextDoc] = field(default_factory=list)
+    raw: dict[str, Any] = field(default_factory=dict)
+
+
+class RagAdapter(ABC):
+    """Subclass and implement `query` to connect ragverdict to your RAG system."""
+
+    @abstractmethod
+    def query(
+        self,
+        prompt: str,
+        *,
+        conversation: list[Message] | None = None,
+    ) -> RagResponse: ...
+
+    def available_tools(self) -> list[ToolSpec]:
+        """Tools the adapter exposes. Used by tool_coverage. Default: none."""
+        return []
+
+    def corpus(self) -> Iterable[SourceDoc] | None:
+        """The source corpus citations resolve against. Default: not available."""
+        return None

ragverdict/adapters/http.py

@@ -0,0 +1,65 @@
+"""HTTP-endpoint adapter — posts {prompt, conversation} and parses the response."""
+
+from __future__ import annotations
+
+from typing import Any
+
+import httpx
+
+from ragverdict.adapters.base import (
+    Citation,
+    ContextDoc,
+    Message,
+    RagAdapter,
+    RagResponse,
+    ToolCall,
+)
+
+
+class HttpAdapter(RagAdapter):
+    """Generic adapter for a RAG agent exposed over HTTP JSON.
+
+    Expects POST {endpoint} with:
+        {"prompt": str, "conversation": [{"role": str, "content": str}, ...]}
+
+    Returns:
+        {
+          "text": str,
+          "citations": [{"id": str, "source_id": str, "span": str}, ...],
+          "tool_calls": [{"name": str, "args": {}, "result": ..., "latency_ms": int, "error": null}, ...],
+          "retrieved_context": [{"source_id": str, "chunk": str, "score": float}, ...]
+        }
+    """
+
+    def __init__(self, *, endpoint: str, headers: dict[str, str], timeout_s: float) -> None:
+        self.endpoint = endpoint
+        self.headers = headers
+        self.timeout_s = timeout_s
+
+    def query(
+        self,
+        prompt: str,
+        *,
+        conversation: list[Message] | None = None,
+    ) -> RagResponse:
+        payload: dict[str, Any] = {
+            "prompt": prompt,
+            "conversation": [
+                {"role": m.role, "content": m.content} for m in (conversation or [])
+            ],
+        }
+        resp = httpx.post(
+            self.endpoint,
+            json=payload,
+            headers=self.headers,
+            timeout=self.timeout_s,
+        )
+        resp.raise_for_status()
+        data: dict[str, Any] = resp.json()
+        return RagResponse(
+            text=data.get("text", ""),
+            citations=[Citation(**c) for c in data.get("citations", [])],
+            tool_calls=[ToolCall(**t) for t in data.get("tool_calls", [])],
+            retrieved_context=[ContextDoc(**c) for c in data.get("retrieved_context", [])],
+            raw=data,
+        )

ragverdict/adapters/loader.py

@@ -0,0 +1,53 @@
+"""Instantiate the user's RagAdapter from config."""
+
+from __future__ import annotations
+
+import importlib
+import os
+import sys
+from typing import Any
+
+from ragverdict.adapters.base import RagAdapter
+from ragverdict.adapters.http import HttpAdapter
+from ragverdict.config import AdapterConfig, HttpAdapterConfig, PythonAdapterConfig
+
+
+class AdapterLoadError(Exception):
+    """Raised when an adapter cannot be loaded or instantiated."""
+
+
+def load_adapter(config: AdapterConfig) -> RagAdapter:
+    if isinstance(config, PythonAdapterConfig):
+        return _load_python(config.module, config.cls)
+    if isinstance(config, HttpAdapterConfig):
+        return HttpAdapter(
+            endpoint=config.endpoint,
+            headers=config.headers,
+            timeout_s=config.timeout_s,
+        )
+    raise AdapterLoadError(f"unknown adapter type: {type(config).__name__}")
+
+
+def _load_python(module_name: str, class_name: str) -> RagAdapter:
+    # User adapters live in their project tree, not in this package — make cwd
+    # importable so `module: examples.demo_rag.adapter` resolves when the user
+    # runs `ragverdict` from their project root.
+    cwd = os.getcwd()
+    if cwd not in sys.path:
+        sys.path.insert(0, cwd)
+    try:
+        module = importlib.import_module(module_name)
+    except ImportError as exc:
+        raise AdapterLoadError(f"cannot import module {module_name!r}: {exc}") from exc
+    cls: Any = getattr(module, class_name, None)
+    if cls is None:
+        raise AdapterLoadError(f"module {module_name!r} has no attribute {class_name!r}")
+    try:
+        instance = cls()
+    except Exception as exc:
+        raise AdapterLoadError(f"failed to instantiate {module_name}.{class_name}: {exc}") from exc
+    if not isinstance(instance, RagAdapter):
+        raise AdapterLoadError(
+            f"{module_name}.{class_name} is not a RagAdapter subclass"
+        )
+    return instance

ragverdict/cli.py

@@ -0,0 +1,58 @@
+"""Click CLI entry point: `ragverdict run <config.yaml>`."""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+import click
+
+from ragverdict import __version__
+from ragverdict.adapters.loader import AdapterLoadError
+from ragverdict.config import ConfigError
+from ragverdict.runner import Runner, RunnerError
+
+
+@click.group(name="ragverdict")
+@click.version_option(__version__, prog_name="ragverdict")
+def cli() -> None:
+    """ragverdict — pytest for RAG agents."""
+
+
+@cli.command()
+@click.argument("config_path", type=click.Path(exists=True, path_type=Path))
+@click.option(
+    "--out-dir",
+    type=click.Path(path_type=Path),
+    default=Path("./report"),
+    show_default=True,
+    help="Directory to write report.json and report.md.",
+)
+@click.option(
+    "--no-judge",
+    is_flag=True,
+    default=False,
+    help="Skip LLM-as-judge entirely. Hard assertions still run; WEAK verdicts and "
+    "citation support scoring are disabled.",
+)
+def run(config_path: Path, out_dir: Path, no_judge: bool) -> None:
+    """Run the evaluation defined in CONFIG_PATH."""
+    try:
+        runner = Runner.from_config_path(
+            config_path,
+            out_dir=out_dir,
+            judge=None if no_judge else "auto",
+        )
+        _results, exit_code = runner.execute()
+    except (ConfigError, AdapterLoadError, RunnerError) as exc:
+        click.echo(f"error: {exc}", err=True)
+        sys.exit(2)
+    sys.exit(exit_code)
+
+
+def main() -> None:
+    cli()
+
+
+if __name__ == "__main__":
+    main()

ragverdict/config.py

@@ -0,0 +1,117 @@
+"""Pydantic schemas + YAML loader for ragverdict configuration."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any, Literal
+
+import yaml
+from pydantic import BaseModel, ConfigDict, Field, ValidationError
+
+
+class ConfigError(Exception):
+    """Raised when a config file is missing, malformed, or fails validation."""
+
+
+class PythonAdapterConfig(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    type: Literal["python"] = "python"
+    module: str
+    cls: str = Field(alias="class")
+
+
+class HttpAdapterConfig(BaseModel):
+    model_config = ConfigDict(extra="forbid", populate_by_name=True)
+
+    type: Literal["http"] = "http"
+    endpoint: str
+    headers: dict[str, str] = Field(default_factory=dict)
+    timeout_s: float = 30.0
+
+
+AdapterConfig = PythonAdapterConfig | HttpAdapterConfig
+
+
+class JudgeConfig(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    provider: Literal["anthropic"] = "anthropic"
+    model: str = "claude-sonnet-4-6"
+    max_concurrency: int = 4
+    fixtures_path: Path | None = None
+
+
+class Thresholds(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    faithfulness_pass: float = 0.85
+    faithfulness_weak: float = 0.7
+    relevance_pass: float = 0.85
+    relevance_weak: float = 0.7
+    citation_support_pass: float = 0.95
+    citation_support_weak: float = 0.8
+
+
+class RagQualityCase(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    query: str
+    expects_citations: bool = False
+    must_mention: list[str] = Field(default_factory=list)
+    must_not_cite: bool = False
+    must_refuse: bool = False
+
+
+class ToolCoverageSpec(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    require_all_tools: bool = True
+    trigger_prompts: dict[str, str] = Field(default_factory=dict)
+
+
+class CitationAuditSpec(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    sample_queries: list[str] = Field(default_factory=list)
+    sample_size: int = 10
+
+
+class TestSpec(BaseModel):
+    """One entry in the top-level `tests:` list.
+
+    `evaluator` selects which evaluator runs; the remaining fields are evaluator-specific
+    and validated by the evaluator at run time. We keep them as a free-form dict here so
+    evaluators can evolve their case shapes without churning Config.
+    """
+
+    __test__ = False  # tell pytest this is not a test class
+    model_config = ConfigDict(extra="allow")
+
+    name: str
+    evaluator: str
+
+
+class Config(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    adapter: AdapterConfig
+    judge: JudgeConfig = Field(default_factory=JudgeConfig)
+    thresholds: Thresholds = Field(default_factory=Thresholds)
+    tests: list[TestSpec] = Field(default_factory=list)
+
+
+def load_config(path: Path) -> Config:
+    """Read a YAML config file and validate it. Raises ConfigError on any failure."""
+    if not path.exists():
+        raise ConfigError(f"config file not found: {path}")
+    try:
+        raw: Any = yaml.safe_load(path.read_text())
+    except yaml.YAMLError as exc:
+        raise ConfigError(f"invalid YAML in {path}: {exc}") from exc
+    if not isinstance(raw, dict):
+        raise ConfigError(f"config root must be a mapping, got {type(raw).__name__}")
+    try:
+        return Config.model_validate(raw)
+    except ValidationError as exc:
+        raise ConfigError(f"config validation failed:\n{exc}") from exc

ragverdict/evaluators/__init__.py

@@ -0,0 +1,27 @@
+"""Evaluators run behavioral tests against a RagAdapter."""
+
+from ragverdict.evaluators.base import Evaluator, TestResult, Verdict
+
+EVALUATORS: dict[str, type[Evaluator]] = {}
+
+
+def register(cls: type[Evaluator]) -> type[Evaluator]:
+    """Register an Evaluator subclass by its `name` class attribute."""
+    EVALUATORS[cls.name] = cls
+    return cls
+
+
+def _autoload() -> None:
+    # Import the bundled evaluator modules so they self-register on the registry.
+    # Kept inside a function (not module-level) to avoid circular imports.
+    from ragverdict.evaluators import (  # noqa: F401
+        citation_audit,
+        edge_cases,
+        rag_quality,
+        tool_coverage,
+    )
+
+
+_autoload()
+
+__all__ = ["EVALUATORS", "Evaluator", "TestResult", "Verdict", "register"]

ragverdict/evaluators/base.py

@@ -0,0 +1,47 @@
+"""Evaluator ABC, verdict enum, and the TestResult dataclass."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import TYPE_CHECKING, Any, ClassVar
+
+if TYPE_CHECKING:
+    from ragverdict.adapters.base import RagAdapter
+    from ragverdict.config import TestSpec, Thresholds
+    from ragverdict.judges.llm_judge import LLMJudge
+
+
+class Verdict(str, Enum):
+    PASS = "PASS"
+    WEAK = "WEAK"
+    FAIL = "FAIL"
+    ERROR = "ERROR"
+
+
+@dataclass
+class TestResult:
+    name: str
+    evaluator: str
+    verdict: Verdict
+    detail: str = ""
+    metrics: dict[str, float] = field(default_factory=dict)
+    duration_ms: int = 0
+    artifacts: dict[str, Any] = field(default_factory=dict)
+
+
+class Evaluator(ABC):
+    """Subclass and implement `run`. Register with `@register` in evaluators/__init__.py."""
+
+    name: ClassVar[str]
+
+    @abstractmethod
+    def run(
+        self,
+        adapter: RagAdapter,
+        spec: TestSpec,
+        *,
+        judge: LLMJudge | None,
+        thresholds: Thresholds,
+    ) -> TestResult: ...