llm-evalgate 0.1.0__tar.gz

llm_evalgate-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,30 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: pip install -e ".[dev]"
+
+      - name: Lint
+        run: ruff check .
+
+      - name: Test
+        run: pytest

llm_evalgate-0.1.0/.github/workflows/publish.yml

@@ -0,0 +1,33 @@
+name: Publish to PyPI
+
+on:
+  push:
+    tags:
+      - "v*"
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    environment: release
+    permissions:
+      contents: read
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Build
+        run: |
+          pip install build
+          python -m build
+
+      - name: Publish
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
+        run: |
+          pip install twine
+          twine upload --verbose dist/*

llm_evalgate-0.1.0/.gitignore

@@ -0,0 +1,15 @@
+__pycache__/
+*.py[cod]
+*.egg-info/
+dist/
+build/
+.eggs/
+.venv/
+venv/
+.env
+*.env
+.pytest_cache/
+.ruff_cache/
+.coverage
+htmlcov/
+*.log

llm_evalgate-0.1.0/PKG-INFO

@@ -0,0 +1,188 @@
+Metadata-Version: 2.4
+Name: llm-evalgate
+Version: 0.1.0
+Summary: Deterministic eval gates and reliability primitives for LLM pipelines
+Project-URL: Homepage, https://github.com/LesterALeong/llm-evalkit
+Project-URL: Repository, https://github.com/LesterALeong/llm-evalkit
+Project-URL: Issues, https://github.com/LesterALeong/llm-evalkit/issues
+Author-email: Lester Leong <lester.leong89@gmail.com>
+License: MIT
+Keywords: agents,ai,eval,evaluation,llm,pipeline,reliability
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.9
+Requires-Dist: textstat>=0.7
+Provides-Extra: dev
+Requires-Dist: hatch>=1.12; extra == 'dev'
+Requires-Dist: pytest-cov>=5.0; extra == 'dev'
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Requires-Dist: ruff>=0.4; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# llm-evalkit
+
+Deterministic eval gates and reliability primitives for LLM pipelines.
+
+[![CI](https://github.com/LesterALeong/llm-evalkit/actions/workflows/ci.yml/badge.svg)](https://github.com/LesterALeong/llm-evalkit/actions/workflows/ci.yml)
+[![PyPI](https://img.shields.io/pypi/v/llm-evalgate)](https://pypi.org/project/llm-evalgate/)
+[![Python](https://img.shields.io/pypi/pyversions/llm-evalgate)](https://pypi.org/project/llm-evalgate/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](LICENSE)
+
+---
+
+Most LLM eval tooling is either LLM-as-judge (non-deterministic, expensive, not CI-friendly) or a heavy enterprise suite. `llm-evalkit` is neither.
+
+It gives you two things:
+
+- **Eval gates**: code-only quality dimensions that run the same way every time. Drop them into any pipeline, run them in CI, get a pass/fail with a reason.
+- **Reliability primitives**: retry with backoff, model fallback chains, and a circuit breaker. The building blocks for LLM pipelines that hold up in production.
+
+## Install
+
+```bash
+pip install llm-evalgate
+```
+
+## Quickstart
+
+### Eval gates
+
+```python
+from llm_evalkit import EvalHarness
+from llm_evalkit.eval.dimensions import BlocklistDimension, ReadabilityDimension, SchemaComplianceDimension
+
+harness = EvalHarness([
+    BlocklistDimension(terms=["confidential", "internal use only"]),
+    ReadabilityDimension(threshold=0.3),
+    SchemaComplianceDimension(required_fields=["title:", "summary:"]),
+])
+
+report = harness.run(llm_output)
+
+if not report.passed:
+    print(report)
+    # EvalReport: FAIL
+    #   FAIL [blocklist] score=0.000 — prohibited terms found: ['confidential']
+    #   PASS [readability] score=0.612 — Flesch ease=61.2, FK grade=8.4
+    #   PASS [schema_compliance] score=1.000 — all 2 required fields present
+```
+
+### Custom dimension
+
+```python
+from llm_evalkit import Dimension
+
+class JsonDimension(Dimension):
+    def evaluate(self, text: str) -> tuple[float, str]:
+        import json
+        try:
+            json.loads(text)
+            return 1.0, "valid JSON"
+        except json.JSONDecodeError as e:
+            return 0.0, f"invalid JSON: {e}"
+
+harness = EvalHarness([JsonDimension(threshold=1.0)])
+report = harness.run('{"key": "value"}')
+assert report.passed
+```
+
+### Retry
+
+```python
+from llm_evalkit.reliable import retry
+
+@retry(max_attempts=3, backoff=2.0)
+def call_llm(prompt: str) -> str:
+    return client.messages.create(...)
+```
+
+### Fallback chain
+
+```python
+from llm_evalkit.reliable import with_fallback, with_fallback_chain
+
+# two-model fallback
+result = with_fallback(
+    primary=lambda: call_model("claude-opus-4-8", prompt),
+    fallback=lambda: call_model("claude-sonnet-4-6", prompt),
+)
+
+# ordered chain — first success wins
+result = with_fallback_chain([
+    lambda: call_model("claude-opus-4-8", prompt),
+    lambda: call_model("claude-sonnet-4-6", prompt),
+    lambda: call_model("claude-haiku-4-5", prompt),
+])
+```
+
+### Circuit breaker
+
+```python
+from llm_evalkit.reliable import CircuitBreaker, CircuitOpenError
+
+breaker = CircuitBreaker(failure_threshold=5, recovery_timeout=60)
+
+try:
+    with breaker:
+        result = call_llm(prompt)
+except CircuitOpenError:
+    result = cached_response  # serve from cache while circuit is open
+```
+
+## Built-in dimensions
+
+| Dimension | What it checks | Default threshold |
+|---|---|---|
+| `BlocklistDimension` | No prohibited terms in output | 1.0 (zero tolerance) |
+| `ReadabilityDimension` | Flesch Reading Ease score | 0.3 (college-level prose) |
+| `SchemaComplianceDimension` | Required fields are present | 1.0 (all fields) |
+| `FactualGroundingDimension` | Numeric claims traceable to evidence | 0.85 |
+
+All dimensions follow the same interface: `evaluate(text) -> (score, detail)`. Writing a new one is ten lines.
+
+## Why deterministic?
+
+LLM-as-judge eval is useful for research. In production pipelines, you need:
+
+- The same input to produce the same pass/fail result every run
+- CI to catch regressions without burning tokens on every commit
+- An audit trail that doesn't depend on a model that may drift
+
+`llm-evalkit` eval dimensions are pure functions. No model calls, no network, no randomness.
+
+## Composing with a pipeline
+
+```python
+from llm_evalkit import EvalHarness
+from llm_evalkit.eval.dimensions import BlocklistDimension, ReadabilityDimension
+from llm_evalkit.reliable import retry, with_fallback
+
+harness = EvalHarness([
+    BlocklistDimension(terms=["[REDACTED]", "TODO"]),
+    ReadabilityDimension(threshold=0.2),
+])
+
+@retry(max_attempts=3, backoff=2.0)
+def generate(prompt: str) -> str:
+    return with_fallback(
+        primary=lambda: call_model("claude-opus-4-8", prompt),
+        fallback=lambda: call_model("claude-sonnet-4-6", prompt),
+    )
+
+output = generate(prompt)
+report = harness.run(output)
+if not report.passed:
+    raise ValueError(f"Output failed eval gate:\n{report}")
+```
+
+## License
+
+MIT

llm_evalgate-0.1.0/README.md

@@ -0,0 +1,159 @@
+# llm-evalkit
+
+Deterministic eval gates and reliability primitives for LLM pipelines.
+
+[![CI](https://github.com/LesterALeong/llm-evalkit/actions/workflows/ci.yml/badge.svg)](https://github.com/LesterALeong/llm-evalkit/actions/workflows/ci.yml)
+[![PyPI](https://img.shields.io/pypi/v/llm-evalgate)](https://pypi.org/project/llm-evalgate/)
+[![Python](https://img.shields.io/pypi/pyversions/llm-evalgate)](https://pypi.org/project/llm-evalgate/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](LICENSE)
+
+---
+
+Most LLM eval tooling is either LLM-as-judge (non-deterministic, expensive, not CI-friendly) or a heavy enterprise suite. `llm-evalkit` is neither.
+
+It gives you two things:
+
+- **Eval gates**: code-only quality dimensions that run the same way every time. Drop them into any pipeline, run them in CI, get a pass/fail with a reason.
+- **Reliability primitives**: retry with backoff, model fallback chains, and a circuit breaker. The building blocks for LLM pipelines that hold up in production.
+
+## Install
+
+```bash
+pip install llm-evalgate
+```
+
+## Quickstart
+
+### Eval gates
+
+```python
+from llm_evalkit import EvalHarness
+from llm_evalkit.eval.dimensions import BlocklistDimension, ReadabilityDimension, SchemaComplianceDimension
+
+harness = EvalHarness([
+    BlocklistDimension(terms=["confidential", "internal use only"]),
+    ReadabilityDimension(threshold=0.3),
+    SchemaComplianceDimension(required_fields=["title:", "summary:"]),
+])
+
+report = harness.run(llm_output)
+
+if not report.passed:
+    print(report)
+    # EvalReport: FAIL
+    #   FAIL [blocklist] score=0.000 — prohibited terms found: ['confidential']
+    #   PASS [readability] score=0.612 — Flesch ease=61.2, FK grade=8.4
+    #   PASS [schema_compliance] score=1.000 — all 2 required fields present
+```
+
+### Custom dimension
+
+```python
+from llm_evalkit import Dimension
+
+class JsonDimension(Dimension):
+    def evaluate(self, text: str) -> tuple[float, str]:
+        import json
+        try:
+            json.loads(text)
+            return 1.0, "valid JSON"
+        except json.JSONDecodeError as e:
+            return 0.0, f"invalid JSON: {e}"
+
+harness = EvalHarness([JsonDimension(threshold=1.0)])
+report = harness.run('{"key": "value"}')
+assert report.passed
+```
+
+### Retry
+
+```python
+from llm_evalkit.reliable import retry
+
+@retry(max_attempts=3, backoff=2.0)
+def call_llm(prompt: str) -> str:
+    return client.messages.create(...)
+```
+
+### Fallback chain
+
+```python
+from llm_evalkit.reliable import with_fallback, with_fallback_chain
+
+# two-model fallback
+result = with_fallback(
+    primary=lambda: call_model("claude-opus-4-8", prompt),
+    fallback=lambda: call_model("claude-sonnet-4-6", prompt),
+)
+
+# ordered chain — first success wins
+result = with_fallback_chain([
+    lambda: call_model("claude-opus-4-8", prompt),
+    lambda: call_model("claude-sonnet-4-6", prompt),
+    lambda: call_model("claude-haiku-4-5", prompt),
+])
+```
+
+### Circuit breaker
+
+```python
+from llm_evalkit.reliable import CircuitBreaker, CircuitOpenError
+
+breaker = CircuitBreaker(failure_threshold=5, recovery_timeout=60)
+
+try:
+    with breaker:
+        result = call_llm(prompt)
+except CircuitOpenError:
+    result = cached_response  # serve from cache while circuit is open
+```
+
+## Built-in dimensions
+
+| Dimension | What it checks | Default threshold |
+|---|---|---|
+| `BlocklistDimension` | No prohibited terms in output | 1.0 (zero tolerance) |
+| `ReadabilityDimension` | Flesch Reading Ease score | 0.3 (college-level prose) |
+| `SchemaComplianceDimension` | Required fields are present | 1.0 (all fields) |
+| `FactualGroundingDimension` | Numeric claims traceable to evidence | 0.85 |
+
+All dimensions follow the same interface: `evaluate(text) -> (score, detail)`. Writing a new one is ten lines.
+
+## Why deterministic?
+
+LLM-as-judge eval is useful for research. In production pipelines, you need:
+
+- The same input to produce the same pass/fail result every run
+- CI to catch regressions without burning tokens on every commit
+- An audit trail that doesn't depend on a model that may drift
+
+`llm-evalkit` eval dimensions are pure functions. No model calls, no network, no randomness.
+
+## Composing with a pipeline
+
+```python
+from llm_evalkit import EvalHarness
+from llm_evalkit.eval.dimensions import BlocklistDimension, ReadabilityDimension
+from llm_evalkit.reliable import retry, with_fallback
+
+harness = EvalHarness([
+    BlocklistDimension(terms=["[REDACTED]", "TODO"]),
+    ReadabilityDimension(threshold=0.2),
+])
+
+@retry(max_attempts=3, backoff=2.0)
+def generate(prompt: str) -> str:
+    return with_fallback(
+        primary=lambda: call_model("claude-opus-4-8", prompt),
+        fallback=lambda: call_model("claude-sonnet-4-6", prompt),
+    )
+
+output = generate(prompt)
+report = harness.run(output)
+if not report.passed:
+    raise ValueError(f"Output failed eval gate:\n{report}")
+```
+
+## License
+
+MIT

llm_evalgate-0.1.0/pyproject.toml

@@ -0,0 +1,58 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "llm-evalgate"
+version = "0.1.0"
+description = "Deterministic eval gates and reliability primitives for LLM pipelines"
+readme = "README.md"
+license = { text = "MIT" }
+requires-python = ">=3.9"
+authors = [{ name = "Lester Leong", email = "lester.leong89@gmail.com" }]
+keywords = ["llm", "eval", "evaluation", "reliability", "ai", "agents", "pipeline"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+]
+dependencies = [
+    "textstat>=0.7",
+]
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=8.0",
+    "pytest-cov>=5.0",
+    "ruff>=0.4",
+    "hatch>=1.12",
+]
+
+[project.urls]
+Homepage = "https://github.com/LesterALeong/llm-evalkit"
+Repository = "https://github.com/LesterALeong/llm-evalkit"
+Issues = "https://github.com/LesterALeong/llm-evalkit/issues"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/llm_evalkit"]
+
+[tool.ruff]
+line-length = 100
+target-version = "py39"
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "UP"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+addopts = "--cov=llm_evalkit --cov-report=term-missing"
+
+[tool.coverage.run]
+source = ["src"]

llm_evalgate-0.1.0/src/llm_evalkit/__init__.py

@@ -0,0 +1,4 @@
+from .eval.dimension import Dimension, DimensionResult
+from .eval.harness import EvalHarness, EvalReport
+
+__all__ = ["Dimension", "DimensionResult", "EvalHarness", "EvalReport"]

llm_evalgate-0.1.0/src/llm_evalkit/eval/__init__.py

@@ -0,0 +1,4 @@
+from .dimension import Dimension, DimensionResult
+from .harness import EvalHarness, EvalReport
+
+__all__ = ["Dimension", "DimensionResult", "EvalHarness", "EvalReport"]

llm_evalgate-0.1.0/src/llm_evalkit/eval/dimension.py

@@ -0,0 +1,31 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class DimensionResult:
+    score: float
+    passed: bool
+    detail: str
+
+
+class Dimension(ABC):
+    """Base class for a single eval dimension.
+
+    Subclass this and implement ``evaluate``. The harness calls ``run``,
+    which applies the threshold and returns a ``DimensionResult``.
+    """
+
+    def __init__(self, threshold: float = 1.0, name: str | None = None) -> None:
+        self.threshold = threshold
+        self.name = name or self.__class__.__name__
+
+    @abstractmethod
+    def evaluate(self, text: str) -> tuple[float, str]:
+        """Return (score, detail). Score is in [0.0, 1.0]."""
+
+    def run(self, text: str) -> DimensionResult:
+        score, detail = self.evaluate(text)
+        return DimensionResult(score=score, passed=score >= self.threshold, detail=detail)

llm_evalgate-0.1.0/src/llm_evalkit/eval/dimensions/__init__.py

@@ -0,0 +1,11 @@
+from .blocklist import BlocklistDimension
+from .factual import FactualGroundingDimension
+from .readability import ReadabilityDimension
+from .schema import SchemaComplianceDimension
+
+__all__ = [
+    "BlocklistDimension",
+    "FactualGroundingDimension",
+    "ReadabilityDimension",
+    "SchemaComplianceDimension",
+]

llm_evalgate-0.1.0/src/llm_evalkit/eval/dimensions/blocklist.py

@@ -0,0 +1,32 @@
+from __future__ import annotations
+
+import re
+
+from ..dimension import Dimension
+
+
+class BlocklistDimension(Dimension):
+    """Fail if any prohibited term appears in the text (case-insensitive).
+
+    Score is 1.0 when clean, 0.0 when any term is found.  Useful for
+    preventing confidential identifiers, brand names, or internal jargon
+    from leaking into LLM output.
+    """
+
+    def __init__(
+        self,
+        terms: list[str],
+        threshold: float = 1.0,
+        name: str = "blocklist",
+        case_sensitive: bool = False,
+    ) -> None:
+        super().__init__(threshold=threshold, name=name)
+        flags = 0 if case_sensitive else re.IGNORECASE
+        self._patterns = [re.compile(re.escape(t), flags) for t in terms]
+        self._terms = terms
+
+    def evaluate(self, text: str) -> tuple[float, str]:
+        found = [t for t, p in zip(self._terms, self._patterns) if p.search(text)]
+        if found:
+            return 0.0, f"prohibited terms found: {found}"
+        return 1.0, "clean"

llm_evalgate-0.1.0/src/llm_evalkit/eval/dimensions/factual.py

@@ -0,0 +1,56 @@
+from __future__ import annotations
+
+import re
+
+from ..dimension import Dimension
+
+
+class FactualGroundingDimension(Dimension):
+    """Check that numeric claims in LLM output are traceable to evidence.
+
+    For each number extracted from the text, we check whether a value
+    within ``rel_tolerance`` of it appears in the evidence list.  Score
+    is the fraction of numeric claims that are grounded.
+
+    If no evidence is supplied the dimension is skipped (returns 1.0).
+    If the text contains no numbers, it also passes.
+    """
+
+    def __init__(
+        self,
+        evidence: list[float] | None = None,
+        rel_tolerance: float = 0.02,
+        threshold: float = 0.85,
+        name: str = "factual_grounding",
+    ) -> None:
+        super().__init__(threshold=threshold, name=name)
+        self._evidence = evidence or []
+        self._rel_tolerance = rel_tolerance
+
+    def _numbers_in_text(self, text: str) -> list[float]:
+        raw = re.findall(r"[\d,]+(?:\.\d+)?", text)
+        results = []
+        for r in raw:
+            try:
+                results.append(float(r.replace(",", "")))
+            except ValueError:
+                pass
+        return results
+
+    def _is_grounded(self, value: float) -> bool:
+        for ev in self._evidence:
+            if ev == 0:
+                continue
+            if abs(value - ev) / abs(ev) <= self._rel_tolerance:
+                return True
+        return False
+
+    def evaluate(self, text: str) -> tuple[float, str]:
+        if not self._evidence:
+            return 1.0, "skipped (no evidence supplied)"
+        numbers = self._numbers_in_text(text)
+        if not numbers:
+            return 1.0, "no numeric claims found"
+        grounded = [n for n in numbers if self._is_grounded(n)]
+        score = len(grounded) / len(numbers)
+        return score, f"{len(grounded)}/{len(numbers)} numeric claims grounded"

llm_evalgate-0.1.0/src/llm_evalkit/eval/dimensions/readability.py

@@ -0,0 +1,26 @@
+from __future__ import annotations
+
+import textstat
+
+from ..dimension import Dimension
+
+
+class ReadabilityDimension(Dimension):
+    """Pass when Flesch Reading Ease score maps to a grade <= max_grade.
+
+    ``threshold`` is a normalised [0, 1] score derived from Flesch Reading
+    Ease, where 1.0 = very easy and 0.0 = very difficult.  The default
+    threshold of 0.3 accepts most professional prose up to ~college level.
+    """
+
+    def __init__(self, threshold: float = 0.3, name: str = "readability") -> None:
+        super().__init__(threshold=threshold, name=name)
+
+    def evaluate(self, text: str) -> tuple[float, str]:
+        if not text.strip():
+            return 0.0, "empty text"
+        ease = textstat.flesch_reading_ease(text)
+        # Flesch ease: 100=very easy, 0=very hard. Normalise to [0, 1].
+        score = max(0.0, min(1.0, ease / 100.0))
+        grade = textstat.flesch_kincaid_grade(text)
+        return score, f"Flesch ease={ease:.1f}, FK grade={grade:.1f}"

llm_evalgate-0.1.0/src/llm_evalkit/eval/dimensions/schema.py

@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+from ..dimension import Dimension
+
+
+class SchemaComplianceDimension(Dimension):
+    """Check that required fields are present in the text.
+
+    Useful for structured LLM outputs (JSON, YAML, markdown with
+    required sections) where missing fields are a hard failure.
+
+    ``required_fields`` is a list of strings that must each appear
+    verbatim somewhere in the text.
+    """
+
+    def __init__(
+        self,
+        required_fields: list[str],
+        threshold: float = 1.0,
+        name: str = "schema_compliance",
+    ) -> None:
+        super().__init__(threshold=threshold, name=name)
+        self._required = required_fields
+
+    def evaluate(self, text: str) -> tuple[float, str]:
+        missing = [f for f in self._required if f not in text]
+        if missing:
+            score = 1.0 - len(missing) / len(self._required)
+            return score, f"missing fields: {missing}"
+        return 1.0, f"all {len(self._required)} required fields present"

llm_evalgate-0.1.0/src/llm_evalkit/eval/harness.py

@@ -0,0 +1,50 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from .dimension import Dimension, DimensionResult
+
+
+@dataclass
+class EvalReport:
+    passed: bool
+    results: dict[str, DimensionResult]
+    text: str
+
+    @property
+    def failures(self) -> dict[str, DimensionResult]:
+        return {name: r for name, r in self.results.items() if not r.passed}
+
+    def __str__(self) -> str:
+        lines = [f"EvalReport: {'PASS' if self.passed else 'FAIL'}"]
+        for name, result in self.results.items():
+            status = "PASS" if result.passed else "FAIL"
+            lines.append(f"  {status} [{name}] score={result.score:.3f} — {result.detail}")
+        return "\n".join(lines)
+
+
+class EvalHarness:
+    """Run a list of dimensions against text and produce an EvalReport.
+
+    Usage::
+
+        harness = EvalHarness([
+            ReadabilityDimension(threshold=0.7),
+            BlocklistDimension(terms=["secret", "internal"]),
+        ])
+        report = harness.run(text)
+        if not report.passed:
+            raise ValueError(str(report))
+    """
+
+    def __init__(self, dimensions: list[Dimension]) -> None:
+        if not dimensions:
+            raise ValueError("EvalHarness requires at least one dimension.")
+        self._dimensions = dimensions
+
+    def run(self, text: str) -> EvalReport:
+        results: dict[str, DimensionResult] = {}
+        for dim in self._dimensions:
+            results[dim.name] = dim.run(text)
+        passed = all(r.passed for r in results.values())
+        return EvalReport(passed=passed, results=results, text=text)

llm_evalgate-0.1.0/src/llm_evalkit/reliable/__init__.py

@@ -0,0 +1,12 @@
+from .circuit import CircuitBreaker, CircuitOpenError, CircuitState
+from .fallback import with_fallback, with_fallback_chain
+from .retry import retry
+
+__all__ = [
+    "retry",
+    "with_fallback",
+    "with_fallback_chain",
+    "CircuitBreaker",
+    "CircuitOpenError",
+    "CircuitState",
+]

llm_evalgate-0.1.0/src/llm_evalkit/reliable/circuit.py

@@ -0,0 +1,97 @@
+from __future__ import annotations
+
+import time
+from enum import Enum
+
+
+class CircuitState(Enum):
+    CLOSED = "closed"      # normal operation
+    OPEN = "open"          # failing, rejecting calls
+    HALF_OPEN = "half_open"  # probing for recovery
+
+
+class CircuitOpenError(Exception):
+    """Raised when a call is attempted while the circuit is open."""
+
+
+class CircuitBreaker:
+    """Prevent cascading failures by stopping calls to a failing service.
+
+    Usage::
+
+        breaker = CircuitBreaker(failure_threshold=5, recovery_timeout=60)
+
+        # context manager form
+        with breaker:
+            result = call_llm(prompt)
+
+        # or call form
+        result = breaker.call(lambda: call_llm(prompt))
+
+    State transitions:
+    - CLOSED -> OPEN: after ``failure_threshold`` consecutive failures
+    - OPEN -> HALF_OPEN: after ``recovery_timeout`` seconds
+    - HALF_OPEN -> CLOSED: on first success
+    - HALF_OPEN -> OPEN: on first failure
+    """
+
+    def __init__(
+        self,
+        failure_threshold: int = 5,
+        recovery_timeout: float = 60.0,
+        exceptions: tuple[type[Exception], ...] = (Exception,),
+    ) -> None:
+        self.failure_threshold = failure_threshold
+        self.recovery_timeout = recovery_timeout
+        self.exceptions = exceptions
+
+        self._state = CircuitState.CLOSED
+        self._failure_count = 0
+        self._opened_at: float | None = None
+
+    @property
+    def state(self) -> CircuitState:
+        if self._state is CircuitState.OPEN:
+            assert self._opened_at is not None
+            if time.monotonic() - self._opened_at >= self.recovery_timeout:
+                self._state = CircuitState.HALF_OPEN
+        return self._state
+
+    def _on_success(self) -> None:
+        self._failure_count = 0
+        self._state = CircuitState.CLOSED
+        self._opened_at = None
+
+    def _on_failure(self) -> None:
+        self._failure_count += 1
+        if self._failure_count >= self.failure_threshold:
+            self._state = CircuitState.OPEN
+            self._opened_at = time.monotonic()
+
+    def call(self, fn):
+        if self.state is CircuitState.OPEN:
+            raise CircuitOpenError(
+                f"Circuit is open. Retry after {self.recovery_timeout}s."
+            )
+        try:
+            result = fn()
+            self._on_success()
+            return result
+        except self.exceptions as exc:
+            self._on_failure()
+            raise exc
+
+    def __enter__(self):
+        if self.state is CircuitState.OPEN:
+            raise CircuitOpenError(
+                f"Circuit is open. Retry after {self.recovery_timeout}s."
+            )
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if exc_type is not None and issubclass(exc_type, self.exceptions):
+            self._on_failure()
+            return False
+        if exc_type is None:
+            self._on_success()
+        return False

llm_evalgate-0.1.0/src/llm_evalkit/reliable/fallback.py

@@ -0,0 +1,53 @@
+from __future__ import annotations
+
+from collections.abc import Callable
+from typing import TypeVar
+
+T = TypeVar("T")
+
+
+def with_fallback(
+    primary: Callable[[], T],
+    fallback: Callable[[], T],
+    exceptions: tuple[type[Exception], ...] = (Exception,),
+) -> T:
+    """Call ``primary``; on failure call ``fallback``.
+
+    Usage::
+
+        result = with_fallback(
+            primary=lambda: call_opus(prompt),
+            fallback=lambda: call_sonnet(prompt),
+        )
+    """
+    try:
+        return primary()
+    except exceptions:
+        return fallback()
+
+
+def with_fallback_chain(
+    callables: list[Callable[[], T]],
+    exceptions: tuple[type[Exception], ...] = (Exception,),
+) -> T:
+    """Try each callable in order, returning the first success.
+
+    Raises the last exception if all callables fail.
+
+    Usage::
+
+        result = with_fallback_chain([
+            lambda: call_opus(prompt),
+            lambda: call_sonnet(prompt),
+            lambda: call_haiku(prompt),
+        ])
+    """
+    if not callables:
+        raise ValueError("callables list is empty")
+    last_exc: Exception | None = None
+    for fn in callables:
+        try:
+            return fn()
+        except exceptions as exc:
+            last_exc = exc
+    raise last_exc  # type: ignore[misc]

llm_evalgate-0.1.0/src/llm_evalkit/reliable/retry.py

@@ -0,0 +1,45 @@
+from __future__ import annotations
+
+import functools
+import time
+from collections.abc import Callable
+from typing import Any, TypeVar
+
+F = TypeVar("F", bound=Callable[..., Any])
+
+
+def retry(
+    max_attempts: int = 3,
+    backoff: float = 2.0,
+    exceptions: tuple[type[Exception], ...] = (Exception,),
+) -> Callable[[F], F]:
+    """Retry a callable on failure with exponential backoff.
+
+    Usage::
+
+        @retry(max_attempts=3, backoff=2.0)
+        def call_llm(prompt: str) -> str:
+            ...
+
+        # or without decorator syntax:
+        result = retry(max_attempts=3)(call_llm)(prompt)
+    """
+
+    def decorator(fn: F) -> F:
+        @functools.wraps(fn)
+        def wrapper(*args: Any, **kwargs: Any) -> Any:
+            delay = backoff
+            last_exc: Exception | None = None
+            for attempt in range(1, max_attempts + 1):
+                try:
+                    return fn(*args, **kwargs)
+                except exceptions as exc:
+                    last_exc = exc
+                    if attempt < max_attempts:
+                        time.sleep(delay)
+                        delay *= backoff
+            raise last_exc  # type: ignore[misc]
+
+        return wrapper  # type: ignore[return-value]
+
+    return decorator

llm_evalgate-0.1.0/tests/test_dimensions.py

@@ -0,0 +1,97 @@
+from llm_evalkit.eval.dimensions import (
+    BlocklistDimension,
+    FactualGroundingDimension,
+    ReadabilityDimension,
+    SchemaComplianceDimension,
+)
+
+
+# --- BlocklistDimension ---
+
+def test_blocklist_clean():
+    dim = BlocklistDimension(terms=["secret", "internal"])
+    result = dim.run("This is a public document.")
+    assert result.passed
+    assert result.score == 1.0
+
+
+def test_blocklist_hit():
+    dim = BlocklistDimension(terms=["secret", "internal"])
+    result = dim.run("This is an internal document.")
+    assert not result.passed
+    assert result.score == 0.0
+    assert "internal" in result.detail
+
+
+def test_blocklist_case_insensitive_default():
+    dim = BlocklistDimension(terms=["SECRET"])
+    result = dim.run("This contains secret info.")
+    assert not result.passed
+
+
+def test_blocklist_case_sensitive():
+    dim = BlocklistDimension(terms=["SECRET"], case_sensitive=True)
+    result = dim.run("This contains secret info.")
+    assert result.passed
+
+
+# --- SchemaComplianceDimension ---
+
+def test_schema_all_present():
+    dim = SchemaComplianceDimension(required_fields=["title:", "summary:", "date:"])
+    text = "title: Foo\nsummary: Bar\ndate: 2024-01-01"
+    result = dim.run(text)
+    assert result.passed
+    assert result.score == 1.0
+
+
+def test_schema_missing_fields():
+    dim = SchemaComplianceDimension(required_fields=["title:", "summary:", "date:"])
+    text = "title: Foo"
+    result = dim.run(text)
+    assert not result.passed
+    assert result.score < 1.0
+    assert "summary:" in result.detail
+
+
+# --- ReadabilityDimension ---
+
+def test_readability_simple_text_passes():
+    dim = ReadabilityDimension(threshold=0.1)
+    text = "The cat sat on the mat. It was a fat cat."
+    result = dim.run(text)
+    assert result.passed
+
+
+def test_readability_empty_text_fails():
+    dim = ReadabilityDimension(threshold=0.1)
+    result = dim.run("   ")
+    assert not result.passed
+
+
+# --- FactualGroundingDimension ---
+
+def test_factual_no_evidence_skips():
+    dim = FactualGroundingDimension(evidence=None)
+    result = dim.run("Revenue was $1.2 billion.")
+    assert result.passed
+    assert "skipped" in result.detail
+
+
+def test_factual_grounded():
+    dim = FactualGroundingDimension(evidence=[1200000000.0], threshold=0.85)
+    result = dim.run("Revenue was 1200000000 dollars.")
+    assert result.passed
+
+
+def test_factual_ungrounded():
+    dim = FactualGroundingDimension(evidence=[999.0], threshold=0.85)
+    result = dim.run("Revenue was 1200000000 dollars.")
+    assert not result.passed
+
+
+def test_factual_no_numbers_passes():
+    dim = FactualGroundingDimension(evidence=[1000.0], threshold=0.85)
+    result = dim.run("Revenue grew significantly.")
+    assert result.passed
+    assert "no numeric" in result.detail

llm_evalgate-0.1.0/tests/test_harness.py

@@ -0,0 +1,58 @@
+import pytest
+
+from llm_evalkit import Dimension, EvalHarness
+
+
+class AlwaysPassDimension(Dimension):
+    def evaluate(self, text: str) -> tuple[float, str]:
+        return 1.0, "always passes"
+
+
+class AlwaysFailDimension(Dimension):
+    def evaluate(self, text: str) -> tuple[float, str]:
+        return 0.0, "always fails"
+
+
+class HalfScoreDimension(Dimension):
+    def evaluate(self, text: str) -> tuple[float, str]:
+        return 0.5, "half score"
+
+
+def test_harness_all_pass():
+    harness = EvalHarness([AlwaysPassDimension(name="d1"), AlwaysPassDimension(name="d2")])
+    report = harness.run("some text")
+    assert report.passed
+    assert report.failures == {}
+
+
+def test_harness_one_fail():
+    harness = EvalHarness([AlwaysPassDimension(name="pass"), AlwaysFailDimension(name="fail")])
+    report = harness.run("some text")
+    assert not report.passed
+    assert "fail" in report.failures
+    assert "pass" not in report.failures
+
+
+def test_harness_threshold():
+    dim = HalfScoreDimension(threshold=0.4, name="half")
+    harness = EvalHarness([dim])
+    report = harness.run("text")
+    assert report.passed
+
+    dim_strict = HalfScoreDimension(threshold=0.6, name="half_strict")
+    harness2 = EvalHarness([dim_strict])
+    report2 = harness2.run("text")
+    assert not report2.passed
+
+
+def test_harness_empty_dimensions_raises():
+    with pytest.raises(ValueError):
+        EvalHarness([])
+
+
+def test_report_str_contains_pass_fail():
+    harness = EvalHarness([AlwaysPassDimension(name="p"), AlwaysFailDimension(name="f")])
+    report = harness.run("x")
+    s = str(report)
+    assert "FAIL" in s
+    assert "PASS" in s

llm_evalgate-0.1.0/tests/test_reliable.py

@@ -0,0 +1,151 @@
+import pytest
+
+from llm_evalkit.reliable import (
+    CircuitBreaker,
+    CircuitOpenError,
+    CircuitState,
+    retry,
+    with_fallback,
+    with_fallback_chain,
+)
+
+
+# --- retry ---
+
+def test_retry_succeeds_first_try():
+    calls = []
+
+    @retry(max_attempts=3, backoff=0.0)
+    def fn():
+        calls.append(1)
+        return "ok"
+
+    assert fn() == "ok"
+    assert len(calls) == 1
+
+
+def test_retry_succeeds_on_second_attempt():
+    calls = []
+
+    @retry(max_attempts=3, backoff=0.0)
+    def fn():
+        calls.append(1)
+        if len(calls) < 2:
+            raise ValueError("not yet")
+        return "ok"
+
+    assert fn() == "ok"
+    assert len(calls) == 2
+
+
+def test_retry_exhausted_raises():
+    @retry(max_attempts=3, backoff=0.0)
+    def fn():
+        raise RuntimeError("always fails")
+
+    with pytest.raises(RuntimeError):
+        fn()
+
+
+def test_retry_only_catches_specified_exceptions():
+    @retry(max_attempts=3, backoff=0.0, exceptions=(ValueError,))
+    def fn():
+        raise TypeError("wrong type")
+
+    with pytest.raises(TypeError):
+        fn()
+
+
+# --- with_fallback ---
+
+def test_fallback_primary_succeeds():
+    result = with_fallback(primary=lambda: "primary", fallback=lambda: "fallback")
+    assert result == "primary"
+
+
+def test_fallback_primary_fails_uses_fallback():
+    def fail():
+        raise RuntimeError("primary down")
+
+    result = with_fallback(primary=fail, fallback=lambda: "fallback")
+    assert result == "fallback"
+
+
+def test_fallback_chain_first_succeeds():
+    result = with_fallback_chain([lambda: "a", lambda: "b", lambda: "c"])
+    assert result == "a"
+
+
+def test_fallback_chain_first_two_fail():
+    calls = []
+
+    def fail():
+        calls.append(1)
+        raise RuntimeError("down")
+
+    result = with_fallback_chain([fail, fail, lambda: "c"])
+    assert result == "c"
+    assert len(calls) == 2
+
+
+def test_fallback_chain_all_fail():
+    def fail():
+        raise RuntimeError("always")
+
+    with pytest.raises(RuntimeError):
+        with_fallback_chain([fail, fail])
+
+
+def test_fallback_chain_empty_raises():
+    with pytest.raises(ValueError):
+        with_fallback_chain([])
+
+
+# --- CircuitBreaker ---
+
+def test_circuit_starts_closed():
+    cb = CircuitBreaker()
+    assert cb.state is CircuitState.CLOSED
+
+
+def test_circuit_opens_after_threshold():
+    cb = CircuitBreaker(failure_threshold=3)
+    for _ in range(3):
+        try:
+            with cb:
+                raise RuntimeError("fail")
+        except RuntimeError:
+            pass
+    assert cb.state is CircuitState.OPEN
+
+
+def test_circuit_open_raises_circuit_error():
+    cb = CircuitBreaker(failure_threshold=1)
+    try:
+        with cb:
+            raise RuntimeError("fail")
+    except RuntimeError:
+        pass
+    with pytest.raises(CircuitOpenError):
+        with cb:
+            pass
+
+
+def test_circuit_resets_on_success():
+    cb = CircuitBreaker(failure_threshold=3)
+    try:
+        with cb:
+            raise RuntimeError("fail")
+    except RuntimeError:
+        pass
+    assert cb._failure_count == 1
+    with cb:
+        pass
+    assert cb.state is CircuitState.CLOSED
+    assert cb._failure_count == 0
+
+
+def test_circuit_call_form():
+    cb = CircuitBreaker()
+    result = cb.call(lambda: "ok")
+    assert result == "ok"