evaldiff 0.0.1__tar.gz

evaldiff-0.0.1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Peter Hedman
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

evaldiff-0.0.1/PKG-INFO

@@ -0,0 +1,100 @@
+Metadata-Version: 2.5
+Name: evaldiff
+Version: 0.0.1
+Summary: CI for LLM prompts: datasets, eval runs, and regression diffs as a pass/fail gate.
+Project-URL: Homepage, https://evaldiff.io
+Project-URL: Repository, https://github.com/evaldiff/evaldiff
+Author-email: Peter Hedman <peter.mejl@gmail.com>
+License: MIT
+License-File: LICENSE
+Keywords: ci,eval,evals,llm,prompts,regression,testing
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Software Development :: Quality Assurance
+Requires-Python: >=3.10
+Requires-Dist: httpx>=0.27
+Requires-Dist: rich>=13.7
+Requires-Dist: typer>=0.12
+Description-Content-Type: text/markdown
+
+# Evaldiff
+
+> **CI for LLM prompts.** Datasets, eval runs, and regression diffs as a
+> pass/fail gate for your CI — the `npm test` of prompts.
+
+Evaldiff scores your LLM output against a dataset of expected answers and
+rubric criteria, then gives you a green/red gate you can wire straight into
+GitHub Actions. When a model change or prompt edit regresses a case, the diff
+tells you exactly which one.
+
+## Why
+
+You already know how to test normal software. LLM output is non-deterministic,
+so "it worked yesterday, why not today?" is a daily problem. Evaldiff sells the
+**green checkmark** — not "AI testing" as an abstract service.
+
+- **Dataset** — a JSON file of `{input, expected, rubric?}` cases.
+- **Run** — score a (dataset, model, prompt) combination; per-case score,
+  pass/fail vs. a threshold, latency, tokens, cost.
+- **Diff** — `run A vs B`: regressions highlighted, side by side.
+- **Gate** — `evaldiff run --threshold 0.85` → exit code `0`/`1`, drops straight
+  into GitHub Actions.
+
+## Install
+
+```bash
+pip install evaldiff
+evaldiff --version
+```
+
+## Quickstart
+
+```bash
+# Scaffold a dataset with two example cases
+evaldiff init my-dataset
+
+# Edit my-dataset.json, then run an eval against your model endpoint
+evaldiff run --dataset my-dataset.json \
+             --endpoint https://your-openai-compatible/v1 \
+             --model your-model \
+             --threshold 0.85
+
+# Compare two runs and see regressions
+evaldiff diff <run-a-id> <run-b-id>
+```
+
+## Dataset format
+
+```json
+[
+  {
+    "input": "Refund policy for a returned item",
+    "expected": "You can return any item within 30 days for a full refund.",
+    "rubric": ["mentions the 30-day window", "mentions full refund"],
+    "tags": ["support"]
+  }
+]
+```
+
+`rubric` is optional — a list of criteria a judge model scores 0/1 against.
+Without it, evaldiff falls back to semantic / exact similarity vs. `expected`.
+
+## Supported endpoints (v0)
+
+- Any **OpenAI-compatible** `/chat/completions` endpoint (BYO key)
+- OpenAI
+- Anthropic
+
+Everything else: bring your own adapter. Scope is the moat.
+
+## Status
+
+**v0.0.1 — name reservation.** The hosted API, judge, and runner land in the
+coming days. This release exists to lock the `evaldiff` name on PyPI.
+
+MIT licensed.

evaldiff-0.0.1/README.md

@@ -0,0 +1,76 @@
+# Evaldiff
+
+> **CI for LLM prompts.** Datasets, eval runs, and regression diffs as a
+> pass/fail gate for your CI — the `npm test` of prompts.
+
+Evaldiff scores your LLM output against a dataset of expected answers and
+rubric criteria, then gives you a green/red gate you can wire straight into
+GitHub Actions. When a model change or prompt edit regresses a case, the diff
+tells you exactly which one.
+
+## Why
+
+You already know how to test normal software. LLM output is non-deterministic,
+so "it worked yesterday, why not today?" is a daily problem. Evaldiff sells the
+**green checkmark** — not "AI testing" as an abstract service.
+
+- **Dataset** — a JSON file of `{input, expected, rubric?}` cases.
+- **Run** — score a (dataset, model, prompt) combination; per-case score,
+  pass/fail vs. a threshold, latency, tokens, cost.
+- **Diff** — `run A vs B`: regressions highlighted, side by side.
+- **Gate** — `evaldiff run --threshold 0.85` → exit code `0`/`1`, drops straight
+  into GitHub Actions.
+
+## Install
+
+```bash
+pip install evaldiff
+evaldiff --version
+```
+
+## Quickstart
+
+```bash
+# Scaffold a dataset with two example cases
+evaldiff init my-dataset
+
+# Edit my-dataset.json, then run an eval against your model endpoint
+evaldiff run --dataset my-dataset.json \
+             --endpoint https://your-openai-compatible/v1 \
+             --model your-model \
+             --threshold 0.85
+
+# Compare two runs and see regressions
+evaldiff diff <run-a-id> <run-b-id>
+```
+
+## Dataset format
+
+```json
+[
+  {
+    "input": "Refund policy for a returned item",
+    "expected": "You can return any item within 30 days for a full refund.",
+    "rubric": ["mentions the 30-day window", "mentions full refund"],
+    "tags": ["support"]
+  }
+]
+```
+
+`rubric` is optional — a list of criteria a judge model scores 0/1 against.
+Without it, evaldiff falls back to semantic / exact similarity vs. `expected`.
+
+## Supported endpoints (v0)
+
+- Any **OpenAI-compatible** `/chat/completions` endpoint (BYO key)
+- OpenAI
+- Anthropic
+
+Everything else: bring your own adapter. Scope is the moat.
+
+## Status
+
+**v0.0.1 — name reservation.** The hosted API, judge, and runner land in the
+coming days. This release exists to lock the `evaldiff` name on PyPI.
+
+MIT licensed.

evaldiff-0.0.1/evaldiff_cli/__init__.py

@@ -0,0 +1 @@
+"""evaldiff_cli package."""

evaldiff-0.0.1/evaldiff_cli/main.py

@@ -0,0 +1,63 @@
+"""evaldiff — CI for LLM prompts.
+
+Stubs a small, self-serve evaluation gate: datasets, runs, and
+regression diffs exposed as a pass/fail gate for CI.
+"""
+from __future__ import annotations
+
+import typer
+from rich.console import Console
+
+__version__ = "0.0.1"
+
+app = typer.Typer(
+    name="evaldiff",
+    help="CI for LLM prompts: datasets, eval runs, regression diffs.",
+    no_args_is_help=True,
+)
+console = Console()
+
+
+@app.command()
+def version() -> None:
+    """Print the evaldiff version."""
+    console.print(f"evaldiff {__version__}")
+
+
+@app.command()
+def init(
+    name: str = typer.Option("my-dataset", help="Dataset name"),
+) -> None:
+    """Scaffold a dataset JSON file in the current directory."""
+    import json
+    import pathlib
+
+    path = pathlib.Path(f"{name}.json")
+    if path.exists():
+        console.print(f"[yellow]{path} already exists, leaving untouched.[/yellow]")
+        raise typer.Exit(0)
+    sample = [
+        {
+            "input": "Refund policy for a returned item",
+            "expected": "You can return any item within 30 days for a full refund.",
+            "rubric": ["mentions the 30-day window", "mentions full refund"],
+            "tags": ["support"],
+        },
+        {
+            "input": "How do I reset my password?",
+            "expected": "Go to Settings → Security → Reset password and follow the email link.",
+            "rubric": ["mentions Settings", "mentions Security"],
+            "tags": ["support"],
+        },
+    ]
+    path.write_text(json.dumps(sample, indent=2) + "\n")
+    console.print(f"[green]Wrote[/green] {path} with {len(sample)} example cases.")
+    console.print("Edit it, then run [bold]evaldiff run[/bold].")
+
+
+def main() -> None:
+    app()
+
+
+if __name__ == "__main__":
+    main()

evaldiff-0.0.1/pyproject.toml

@@ -0,0 +1,40 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "evaldiff"
+version = "0.0.1"
+description = "CI for LLM prompts: datasets, eval runs, and regression diffs as a pass/fail gate."
+readme = "README.md"
+requires-python = ">=3.10"
+license = { text = "MIT" }
+authors = [
+    { name = "Peter Hedman", email = "peter.mejl@gmail.com" }
+]
+keywords = ["llm", "evals", "eval", "testing", "ci", "prompts", "regression"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Software Development :: Quality Assurance",
+]
+dependencies = [
+    "httpx>=0.27",
+    "typer>=0.12",
+    "rich>=13.7",
+]
+
+[project.scripts]
+evaldiff = "evaldiff_cli.main:app"
+
+[project.urls]
+Homepage = "https://evaldiff.io"
+Repository = "https://github.com/evaldiff/evaldiff"
+
+[tool.hatch.build.targets.wheel]
+packages = ["evaldiff_cli"]

evaldiff-0.0.1/tests/test_cli.py

@@ -0,0 +1,38 @@
+"""Tests for the evaldiff CLI stub."""
+from __future__ import annotations
+
+import json
+import pathlib
+
+from typer.testing import CliRunner
+
+from evaldiff_cli.main import app
+
+runner = CliRunner()
+
+
+def test_version_command() -> None:
+    result = runner.invoke(app, ["version"])
+    assert result.exit_code == 0
+    assert "evaldiff" in result.output
+
+
+def test_init_writes_valid_json(tmp_path: pathlib.Path, monkeypatch) -> None:
+    monkeypatch.chdir(tmp_path)
+    result = runner.invoke(app, ["init", "--name", "my-dataset"])
+    assert result.exit_code == 0
+    path = tmp_path / "my-dataset.json"
+    assert path.exists()
+    data = json.loads(path.read_text())
+    assert isinstance(data, list) and len(data) == 2
+    assert "input" in data[0] and "expected" in data[0] and "rubric" in data[0]
+
+
+def test_init_is_idempotent(tmp_path: pathlib.Path, monkeypatch) -> None:
+    monkeypatch.chdir(tmp_path)
+    first = runner.invoke(app, ["init", "--name", "my-dataset"])
+    second = runner.invoke(app, ["init", "--name", "my-dataset"])
+    assert first.exit_code == 0
+    assert second.exit_code == 0
+    data = json.loads((tmp_path / "my-dataset.json").read_text())
+    assert len(data) == 2