checkllm 0.1.0__tar.gz

checkllm-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,49 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  test:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
+      fail-fast: false
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: pip install -e ".[dev]"
+
+      - name: Run tests (deterministic only, no API key needed)
+        run: pytest tests/ -v -m "not llm" --tb=short
+
+      - name: Check CLI works
+        run: checkllm --version
+
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install dependencies
+        run: pip install -e ".[dev]"
+
+      - name: Type check
+        run: python -m py_compile src/checkllm/__init__.py

checkllm-0.1.0/.github/workflows/publish.yml

@@ -0,0 +1,30 @@
+name: Publish to PyPI
+
+on:
+  release:
+    types: [published]
+
+permissions:
+  contents: read
+  id-token: write
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    environment: pypi
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install build tools
+        run: pip install build
+
+      - name: Build package
+        run: python -m build
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

checkllm-0.1.0/.gitignore

@@ -0,0 +1,9 @@
+.worktrees/
+__pycache__/
+*.pyc
+*.egg-info/
+dist/
+build/
+.checkllm/
+*.egg
+.venv/

checkllm-0.1.0/CHANGELOG.md

@@ -0,0 +1,21 @@
+# Changelog
+
+## 0.1.0 (2026-03-28)
+
+Initial release.
+
+### Features
+
+- **pytest plugin** with `check` fixture for LLM testing in pytest
+- **Deterministic checks**: `contains`, `not_contains`, `regex`, `json_schema`, `max_tokens`, `latency`, `cost`
+- **LLM-as-judge metrics**: `hallucination`, `relevance`, `toxicity`, `rubric`
+- **Custom metrics** via `@metric` decorator and plugin entry points
+- **Dataset system**: YAML loading, generator functions, `@dataset` decorator for parametrized tests
+- **Regression detection**: Welch's t-test with configurable p-value threshold
+- **Snapshot system**: save/load/compare test result baselines
+- **Reporting**: Rich terminal output, HTML reports, JUnit XML
+- **CLI**: `checkllm run`, `snapshot`, `report`, `eval`, `diff`, `init`
+- **Multiple judge backends**: OpenAI and Anthropic
+- **Retry logic** with exponential backoff for transient API failures
+- **Cost tracking** from OpenAI/Anthropic token usage
+- **Configuration** via `pyproject.toml [tool.checkllm]` and environment variables

checkllm-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 checkllm contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

checkllm-0.1.0/PKG-INFO

@@ -0,0 +1,404 @@
+Metadata-Version: 2.4
+Name: checkllm
+Version: 0.1.0
+Summary: Test LLM-powered applications with the same rigor as traditional software.
+Author: checkllm contributors
+License-Expression: MIT
+License-File: LICENSE
+Classifier: Development Status :: 3 - Alpha
+Classifier: Framework :: Pytest
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development :: Testing
+Requires-Python: >=3.10
+Requires-Dist: jinja2>=3.1.0
+Requires-Dist: openai>=1.0.0
+Requires-Dist: pydantic>=2.0.0
+Requires-Dist: pyyaml>=6.0.0
+Requires-Dist: rich>=13.0.0
+Requires-Dist: scipy>=1.11.0
+Requires-Dist: tenacity>=8.0.0
+Requires-Dist: tiktoken>=0.5.0
+Requires-Dist: typer>=0.9.0
+Provides-Extra: all
+Requires-Dist: anthropic>=0.20.0; extra == 'all'
+Provides-Extra: anthropic
+Requires-Dist: anthropic>=0.20.0; extra == 'anthropic'
+Provides-Extra: dev
+Requires-Dist: coverage>=7.0.0; extra == 'dev'
+Requires-Dist: pytest-asyncio>=0.21.0; extra == 'dev'
+Requires-Dist: pytest>=7.0.0; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# checkllm
+
+Test LLM-powered applications with the same rigor as traditional software.
+
+checkllm is a pytest plugin and CLI that lets you write assertions for LLM outputs using deterministic checks, LLM-as-judge evaluation, and statistical regression detection.
+
+## Why checkllm?
+
+- **Works with pytest** - no new test runner to learn, just add a `check` fixture
+- **Free deterministic checks** run instantly with zero API calls
+- **LLM-as-judge** for subjective quality (hallucination, relevance, toxicity, custom rubrics)
+- **Statistical regression detection** using Welch's t-test, not just "did it change?"
+- **Multiple judge backends** - OpenAI and Anthropic, or bring your own
+- **One command** to snapshot, report, or diff your test results
+
+## Installation
+
+```bash
+pip install checkllm
+```
+
+For Anthropic Claude support:
+
+```bash
+pip install checkllm[anthropic]
+```
+
+## Quick Start
+
+### 1. Write a test
+
+```python
+# tests/test_my_agent.py
+
+def test_output_quality(check):
+    output = my_agent("What is Python?")
+
+    # Deterministic checks (free, instant)
+    check.contains(output, "programming language")
+    check.not_contains(output, "JavaScript")
+    check.max_tokens(output, limit=200)
+
+    # LLM-as-judge checks (requires OPENAI_API_KEY)
+    check.hallucination(output, context="Python is a high-level programming language.")
+    check.relevance(output, query="What is Python?")
+    check.toxicity(output)
+```
+
+### 2. Run it
+
+```bash
+export OPENAI_API_KEY=sk-...
+
+pytest tests/test_my_agent.py -v
+
+# Or use the CLI
+checkllm run tests/test_my_agent.py
+```
+
+### 3. Track regressions
+
+```bash
+checkllm snapshot tests/ --output .checkllm/snapshots/baseline.json
+
+# After changes, compare
+checkllm snapshot tests/ --output .checkllm/snapshots/current.json
+checkllm diff --baseline .checkllm/snapshots/baseline.json \
+              --current .checkllm/snapshots/current.json
+```
+
+## Deterministic Checks
+
+Zero-cost, zero-latency checks that run locally:
+
+```python
+def test_deterministic(check):
+    output = my_agent("...")
+
+    check.contains(output, "expected substring")
+    check.not_contains(output, "forbidden text")
+    check.exact_match(output, "exact expected output")
+    check.exact_match(output, "EXPECTED", ignore_case=True)
+    check.starts_with(output, "Python")
+    check.ends_with(output, "language.")
+    check.regex(output, pattern=r"\d{3}-\d{4}")
+    check.max_tokens(output, limit=500)
+    check.latency(response_time_ms, max_ms=2000)
+    check.cost(api_cost_usd, max_usd=0.05)
+
+    # Validate JSON structure
+    from pydantic import BaseModel
+
+    class Response(BaseModel):
+        answer: str
+        confidence: float
+
+    check.json_schema(output, schema=Response)
+```
+
+## LLM-as-Judge Metrics
+
+Use GPT-4o (or Claude) as an automated judge:
+
+```python
+def test_llm_quality(check):
+    output = my_agent("Summarize this article about climate change.")
+    article = "..."
+
+    check.hallucination(output, context=article)
+    check.relevance(output, query="Summarize the article")
+    check.toxicity(output)
+    check.rubric(output, criteria="concise, under 3 sentences, mentions key findings")
+```
+
+Each check records a score (0.0-1.0), pass/fail status, reasoning, cost, and latency.
+
+### Custom Thresholds
+
+```python
+check.hallucination(output, context=ctx, threshold=0.9)  # stricter
+check.relevance(output, query=q, threshold=0.6)           # more lenient
+```
+
+### Multiple Runs
+
+```python
+check.hallucination(output, context=ctx, runs=5)
+```
+
+Or set globally:
+
+```toml
+[tool.checkllm]
+runs_per_test = 3
+```
+
+## Dataset-Driven Testing
+
+```yaml
+# tests/fixtures/cases.yaml
+- input: "What is Python?"
+  expected: "Python is a programming language"
+  query: "Explain Python"
+  context: "Python was created by Guido van Rossum in 1991."
+  criteria: "accurate, mentions creator"
+
+- input: "What is 2+2?"
+  expected: "4"
+  criteria: "correct, concise"
+```
+
+```python
+from checkllm import dataset
+
+@dataset("tests/fixtures/cases.yaml")
+def test_across_cases(check, case):
+    output = my_agent(case.input)
+    check.contains(output, case.expected)
+    if case.context:
+        check.hallucination(output, context=case.context)
+```
+
+Or use a Python generator:
+
+```python
+from checkllm import Case, dataset
+
+def my_cases():
+    yield Case(input="Hello", expected="greeting", criteria="friendly")
+    yield Case(input="Goodbye", expected="farewell", criteria="polite")
+
+@dataset(my_cases)
+def test_generated(check, case):
+    output = my_agent(case.input)
+    check.rubric(output, criteria=case.criteria)
+```
+
+## Custom Metrics
+
+```python
+import checkllm
+from checkllm import CheckResult
+
+@checkllm.metric("brevity")
+def brevity_check(output: str, max_words: int = 50, **kwargs) -> CheckResult:
+    word_count = len(output.split())
+    return CheckResult(
+        passed=word_count <= max_words,
+        score=min(1.0, max_words / max(word_count, 1)),
+        reasoning=f"{word_count} words (limit: {max_words})",
+        cost=0.0,
+        latency_ms=0,
+        metric_name="brevity",
+    )
+
+def test_brevity(check):
+    output = my_agent("Explain quantum physics")
+    check.run_metric("brevity", output=output, max_words=100)
+```
+
+## Async Tests
+
+```python
+import pytest
+
+@pytest.mark.asyncio
+async def test_async_quality(check):
+    output = await my_async_agent("What is Python?")
+
+    await check.ahallucination(output, context="...")
+    await check.arelevance(output, query="What is Python?")
+    await check.atoxicity(output)
+    await check.arubric(output, criteria="concise and accurate")
+
+    # Deterministic checks are always sync (instant, no I/O)
+    check.contains(output, "Python")
+```
+
+## Separating Fast and Slow Tests
+
+Mark LLM tests so you can skip them in fast CI runs:
+
+```python
+import pytest
+
+@pytest.mark.llm
+def test_with_llm(check):
+    check.hallucination(output, context=ctx)
+
+def test_fast(check):
+    check.contains(output, "Python")
+```
+
+```bash
+# Run only fast deterministic tests
+pytest -m "not llm"
+
+# Run only LLM tests
+pytest -m llm
+```
+
+If `OPENAI_API_KEY` is not set, LLM checks automatically skip instead of crashing.
+
+## Regression Detection
+
+checkllm uses Welch's t-test to detect statistically significant score regressions.
+
+```bash
+checkllm snapshot tests/ --output .checkllm/snapshots/v1.json
+# ... make changes ...
+checkllm snapshot tests/ --output .checkllm/snapshots/v2.json
+checkllm diff -b .checkllm/snapshots/v1.json -c .checkllm/snapshots/v2.json
+
+# Fail CI on regression
+checkllm diff -b v1.json -c v2.json --fail-on-regression
+```
+
+## Reporting
+
+```bash
+# HTML report
+checkllm report tests/ --output report.html
+
+# JUnit XML for CI/CD
+checkllm run tests/ --junit-xml results.xml
+
+# pytest flags work directly
+pytest tests/ --checkllm-snapshot=snap.json --checkllm-report=report.html
+```
+
+## CLI Reference
+
+| Command | Description |
+|---------|-------------|
+| `checkllm run <path>` | Run tests with `--snapshot`, `--html-report`, `--junit-xml`, `--compare`, `--fail-on-regression` |
+| `checkllm snapshot <path>` | Save test results as baseline (`--output PATH`) |
+| `checkllm report <path>` | Generate HTML report (`--output PATH`, `--junit-xml PATH`) |
+| `checkllm diff` | Compare snapshots (`--baseline`, `--current`, `--fail-on-regression`) |
+| `checkllm eval` | Evaluate prompt template (`--prompt`, `--dataset`, `--metric`, `--threshold`) |
+| `checkllm init [path]` | Scaffold a new project |
+| `checkllm list-metrics` | List available metrics |
+| `checkllm --version` | Show version |
+
+## Configuration
+
+```toml
+[tool.checkllm]
+judge_backend = "openai"           # "openai" or "anthropic"
+judge_model = "gpt-4o"             # Model for LLM-as-judge
+default_threshold = 0.8            # Pass/fail threshold (0.0-1.0)
+runs_per_test = 1                  # Repeat LLM checks N times
+snapshot_dir = ".checkllm/snapshots"
+confidence_level = 0.95
+p_value_threshold = 0.05
+```
+
+Environment variable overrides: `CHECKLLM_JUDGE_BACKEND`, `CHECKLLM_JUDGE_MODEL`, `CHECKLLM_DEFAULT_THRESHOLD`, `CHECKLLM_RUNS_PER_TEST`.
+
+## Custom Judge Backends
+
+### Anthropic Claude
+
+```toml
+[tool.checkllm]
+judge_backend = "anthropic"
+judge_model = "claude-sonnet-4-6"
+```
+
+### Your Own Backend
+
+Implement the `JudgeBackend` protocol:
+
+```python
+from checkllm import JudgeBackend, JudgeResponse
+from checkllm.check import CheckCollector
+from checkllm.config import CheckllmConfig
+
+class MyJudge:
+    async def evaluate(self, prompt: str, system_prompt: str | None = None) -> JudgeResponse:
+        return JudgeResponse(score=0.9, reasoning="Looks good", cost=0.0)
+
+config = CheckllmConfig()
+collector = CheckCollector(config=config, judge=MyJudge())
+```
+
+## Configuring the Judge in conftest.py
+
+To use a cheaper model or a custom backend for all tests:
+
+```python
+# tests/conftest.py
+import pytest
+from checkllm.check import CheckCollector
+from checkllm.config import load_config
+from checkllm.judge import OpenAIJudge
+from checkllm.pytest_plugin import _CHECKLLM_KEY
+
+@pytest.fixture
+def check(request):
+    config = load_config()
+    judge = OpenAIJudge(model="gpt-4o-mini")  # cheaper model for dev
+    collector = CheckCollector(config=config, judge=judge)
+    request.node.stash[_CHECKLLM_KEY] = collector
+    return collector
+```
+
+## Project Setup
+
+```bash
+checkllm init
+```
+
+Creates `pyproject.toml`, `tests/conftest.py`, sample test file, sample dataset, and `.checkllm/snapshots/` directory.
+
+## Examples
+
+See the [examples/](examples/) directory for working code:
+
+- [test_basic.py](examples/test_basic.py) - Deterministic checks (no API key needed)
+- [test_dataset_driven.py](examples/test_dataset_driven.py) - YAML and generator datasets
+- [test_custom_metrics.py](examples/test_custom_metrics.py) - Register domain-specific metrics
+- [test_llm_judge.py](examples/test_llm_judge.py) - LLM-as-judge evaluation
+- [test_regression_workflow.py](examples/test_regression_workflow.py) - Snapshot and regression detection
+
+## License
+
+MIT