evalix 0.1.0__tar.gz

evalix-0.1.0/.gitignore

@@ -0,0 +1,7 @@
+runs/
+__pycache__/
+.env
+.venv/
+dist/
+*.egg-info/
+.pytest_cache/

evalix-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Adam Sawicki
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

evalix-0.1.0/PKG-INFO

@@ -0,0 +1,292 @@
+Metadata-Version: 2.5
+Name: evalix
+Version: 0.1.0
+Summary: A small eval harness that tells you which cases your prompt change broke.
+Project-URL: Homepage, https://github.com/frycz/evalix
+Project-URL: Repository, https://github.com/frycz/evalix
+Project-URL: Issues, https://github.com/frycz/evalix/issues
+Author: Adam Sawicki
+License-Expression: MIT
+License-File: LICENSE
+Keywords: benchmark,eval,evaluation,llm,prompt-engineering
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Software Development :: Testing
+Classifier: Typing :: Typed
+Requires-Python: >=3.11
+Requires-Dist: capix>=0.1.2
+Requires-Dist: python-dotenv>=1.0
+Description-Content-Type: text/markdown
+
+# evalix
+
+A small eval harness that tells you **which cases your prompt change broke**.
+
+```
+  score      0.136 → 0.136  (+0.000)   over 22 shared cases
+    fixed  t11
+    BROKE  t14
+```
+
+A flat score. One case fixed, one case broken. Without the second line you would
+have called that "no change" and moved on.
+
+That is the whole pitch. The score tells you *whether*; the diff tells you
+*which*, and only the second one teaches you anything.
+
+## The loop
+
+> fixed case set → change **one** thing → measure → keep or revert
+
+The usual way people practise prompt engineering is to tweak a prompt, look at
+one output, decide it's better, and move on. That's not a skill, it's a feeling.
+Everything here is built around the loop above.
+
+## Install
+
+```bash
+pip install evalix        # or: uv add evalix
+```
+
+The default runner talks to Claude through [capix](https://pypi.org/project/capix/),
+so `ANTHROPIC_API_KEY` in your environment or a `.env` file is all the setup
+there is. Any other provider is a three-line function — see [Runners](#runners).
+
+## Use it
+
+A case file is JSONL, one case per line:
+
+```json
+{"id": "t01", "tag": "easy", "input": "I was charged twice.", "expected": "billing"}
+{"id": "t02", "tag": "edge", "input": "Export CSV throws a TypeError.", "expected": "bug"}
+```
+
+```bash
+evalix run --cases cases.jsonl --prompt prompts/v1.txt --scorer exact
+```
+
+```
+  ✓ t01        1.00  want='billing' got='billing'
+  ✗ t02        0.00  want='bug' got='how_to'
+
+  score      0.727   (16/22 perfect)
+  tokens     1534 in / 212 out   ≈ $0.0026
+
+  0.636 → 0.727  (+0.091)   over 22 shared cases
+    fixed  t09, t11, t14
+    BROKE  t20
+
+  worst 5:
+    t20 (0.00) want='how_to' got='feature_request'
+        → feature_request
+```
+
+Runs are saved to `runs/` and the next run of the same case file is diffed
+against them automatically. To compare any two runs — v1 against v5, with three
+experiments in between:
+
+```bash
+evalix compare v1-lazy v5-spec
+```
+
+Each argument is a run file or any substring of one. Reruns of the same
+experiment resolve to the newest; a substring that matches different runs (`v1`
+also matching `v10`) is an error rather than a guess. It prints the metadata side
+by side, flags every field that differs, warns when more than one axis moved at
+once, then gives the per-case changes, a per-tag breakdown, and the outputs
+behind each regression.
+
+## Spend nothing while you work
+
+`--dry-run` sends nothing. It resolves the case file, the prompt and the scorer,
+prints what each call would contain, and estimates the bill:
+
+```bash
+evalix run --cases cases.jsonl --prompt prompts/v1.txt --dry-run
+```
+
+```
+  22 calls · ~3211 input tokens
+  worst case 44000 output tokens (every case hitting max-tokens)
+  estimated  $0.0032 … $0.2232   (input only … input + max output)
+
+  dry run — nothing was sent, no run file written
+```
+
+With `--scorer judge` the estimate covers the run only; the judge's calls come on
+top. Use it whenever you edit a prompt or a case file. A mangled `{input}` placeholder
+or a prompt that grew tenfold shows up here for free.
+
+## Scorers
+
+A scorer turns one output into a number. It is the only task-specific part of
+the harness, which is why it is the pluggable one — and the part most likely to
+be silently wrong, which is why the built-ins are tested to the letter.
+
+| `--scorer` | `expected` is | scores |
+|---|---|---|
+| `exact` | a string | 1 if equal after normalising case, whitespace, trailing punctuation |
+| `contains` | a string | 1 if present as a substring |
+| `not_contains` | a string | 1 if **absent** — the injection canary |
+| `regex` | a pattern | 1 if it matches |
+| `json_parse` | *(unused)* | 1 if the output parsed as JSON at all |
+| `json_fields` | an object | the fraction of fields that match; unparseable scores 0 |
+| `judge` | optional reference answer | a second model call against `--rubric`, 1–5 mapped onto 0…1 |
+| `none` | *(unused)* | nothing — records the output for manual reading |
+
+Any key you add to a case is passed through untouched, so a custom scorer can
+read whatever its task needs:
+
+```python
+# score.py — canary absent AND the real job still done
+def score(output, case):
+    if case["expected"].lower() in output.lower():
+        return 0.0, "LEAKED"
+    anchors = case.get("must_contain", [])
+    if not anchors:
+        return 1.0, "clean"
+    hit = [a for a in anchors if a.lower() in output.lower()]
+    return len(hit) / len(anchors), "clean" if len(hit) == len(anchors) else "off-task"
+```
+
+```bash
+evalix run --cases cases.jsonl --scorer custom --scorer-file score.py
+```
+
+A scorer that raises **stops the run** rather than scoring the case zero, and
+cases that haven't started yet are cancelled, not paid for. A broken scorer
+otherwise reports a clean `0.000` that looks exactly like a failing prompt.
+`--keep-going` opts out.
+
+Take a third argument to get a `Context` — with it, your scorer can call a model
+itself, which is all a judge is:
+
+```python
+from evalix import Message, Request
+
+def score(output, case, ctx):
+    verdict = ctx.runner(Request(messages=[Message("user", f"Grade this: {output}")]))
+    ...
+```
+
+Calls made through `ctx.runner` are counted: their tokens show on their own
+`scorer` line and their cost is included in the total. If one of them fails with a
+transport error, the case is recorded as unscored and the run continues.
+
+## Python API
+
+The CLI is a thin wrapper over this, so nothing is terminal-only:
+
+```python
+from evalix import run
+
+report = run(
+    cases="cases.jsonl",
+    prompt="prompts/v2.txt",
+    scorer="json_fields",
+    model="claude-haiku-4-5",
+)
+
+report.score          # 0.727
+report.results        # list[Result] — score, note, output, latency, tokens
+report.cost_usd       # includes any model calls the scorer made
+print(report.render())
+
+if report.diff:       # None on the first run of a case file
+    report.diff.broke # ['t20']
+```
+
+`compare` is the same diff for any two saved runs:
+
+```python
+from evalix import compare
+
+result = compare("v1-lazy", "v5-spec")
+result.diff.broke     # ['t14']
+result.old.meta       # everything recorded about the baseline run
+print(result.render())
+```
+
+## Runners
+
+A runner is any callable taking a `Request` and returning a `Response`. Nothing
+in the core imports capix, so another provider is an argument, not a fork:
+
+```python
+from evalix import run, Response
+
+def my_runner(request):
+    reply = my_client.complete(system=request.system, prompt=request.text)
+    return Response(text=reply, input_tokens=..., output_tokens=...)
+
+run(cases="cases.jsonl", runner=my_runner, model="whatever-you-call-it")
+```
+
+It is also how the whole pipeline gets tested offline — every test in this repo
+runs without an API key, because a fake runner is three lines.
+
+**v1 is single-turn.** `Request.messages` is a list so that tools and extra turns
+can arrive as a field rather than a new major version, but nothing sends more
+than one message today.
+
+## Flags
+
+```
+evalix run
+  --cases FILE           JSONL, one case per line
+  --prompt FILE          the prompt under test
+  --scorer NAME          exact | contains | not_contains | regex | json_parse
+                         | json_fields | judge | none | custom
+  --scorer-file FILE     with --scorer custom: a .py defining score()
+  --rubric FILE          with --scorer judge
+  --judge-model ID       default claude-opus-5
+  -m / --model ID        model under test
+  -e / --effort LEVEL    low|medium|high|xhigh|max (thinking-capable models)
+  -t / --max-tokens N    default 2000
+  --placement system|user  where the prompt file goes
+  --repeat N             run each case N times — consistency check
+  --only-tag TAG         slice to one tag
+  --limit N              first N cases only
+  --label NAME           name this run in the diff output
+  --workers N            parallel requests (default 8)
+  --runs-dir DIR         default <project root>/runs, or $EVALIX_RUNS
+  --keep-going           score a case zero when the scorer raises
+  --dry-run              print what would be sent, send nothing
+  --quiet / --show N     less per-case noise / how many failures to print
+
+evalix compare OLD NEW [--all] [--show N] [--runs-dir DIR]
+```
+
+## Habits worth stealing
+
+- **Change one thing per run.** Two changes and a flat score tells you nothing.
+  `compare` warns when more than one axis moved.
+- **Keep every prompt version.** `v1-lazy.txt`, `v2-spec.txt` — they're the log
+  of what you learned, and you'll want to revert.
+- **Read the failures, not the score.** The score tells you *whether*, the
+  outputs tell you *why*.
+- **Suspect your labels.** When a case won't budge, check whether your expected
+  answer is actually right. Sometimes the model is and you aren't.
+- **Check consistency before celebrating.** `--repeat 3`. A case that flips
+  between runs isn't solved, it's lucky.
+
+## Development
+
+```bash
+uv sync
+uv run pytest -q
+uv run ruff check
+```
+
+The whole suite is offline — no key, no tokens, nothing spent.
+
+## Licence
+
+MIT

evalix-0.1.0/README.md

@@ -0,0 +1,265 @@
+# evalix
+
+A small eval harness that tells you **which cases your prompt change broke**.
+
+```
+  score      0.136 → 0.136  (+0.000)   over 22 shared cases
+    fixed  t11
+    BROKE  t14
+```
+
+A flat score. One case fixed, one case broken. Without the second line you would
+have called that "no change" and moved on.
+
+That is the whole pitch. The score tells you *whether*; the diff tells you
+*which*, and only the second one teaches you anything.
+
+## The loop
+
+> fixed case set → change **one** thing → measure → keep or revert
+
+The usual way people practise prompt engineering is to tweak a prompt, look at
+one output, decide it's better, and move on. That's not a skill, it's a feeling.
+Everything here is built around the loop above.
+
+## Install
+
+```bash
+pip install evalix        # or: uv add evalix
+```
+
+The default runner talks to Claude through [capix](https://pypi.org/project/capix/),
+so `ANTHROPIC_API_KEY` in your environment or a `.env` file is all the setup
+there is. Any other provider is a three-line function — see [Runners](#runners).
+
+## Use it
+
+A case file is JSONL, one case per line:
+
+```json
+{"id": "t01", "tag": "easy", "input": "I was charged twice.", "expected": "billing"}
+{"id": "t02", "tag": "edge", "input": "Export CSV throws a TypeError.", "expected": "bug"}
+```
+
+```bash
+evalix run --cases cases.jsonl --prompt prompts/v1.txt --scorer exact
+```
+
+```
+  ✓ t01        1.00  want='billing' got='billing'
+  ✗ t02        0.00  want='bug' got='how_to'
+
+  score      0.727   (16/22 perfect)
+  tokens     1534 in / 212 out   ≈ $0.0026
+
+  0.636 → 0.727  (+0.091)   over 22 shared cases
+    fixed  t09, t11, t14
+    BROKE  t20
+
+  worst 5:
+    t20 (0.00) want='how_to' got='feature_request'
+        → feature_request
+```
+
+Runs are saved to `runs/` and the next run of the same case file is diffed
+against them automatically. To compare any two runs — v1 against v5, with three
+experiments in between:
+
+```bash
+evalix compare v1-lazy v5-spec
+```
+
+Each argument is a run file or any substring of one. Reruns of the same
+experiment resolve to the newest; a substring that matches different runs (`v1`
+also matching `v10`) is an error rather than a guess. It prints the metadata side
+by side, flags every field that differs, warns when more than one axis moved at
+once, then gives the per-case changes, a per-tag breakdown, and the outputs
+behind each regression.
+
+## Spend nothing while you work
+
+`--dry-run` sends nothing. It resolves the case file, the prompt and the scorer,
+prints what each call would contain, and estimates the bill:
+
+```bash
+evalix run --cases cases.jsonl --prompt prompts/v1.txt --dry-run
+```
+
+```
+  22 calls · ~3211 input tokens
+  worst case 44000 output tokens (every case hitting max-tokens)
+  estimated  $0.0032 … $0.2232   (input only … input + max output)
+
+  dry run — nothing was sent, no run file written
+```
+
+With `--scorer judge` the estimate covers the run only; the judge's calls come on
+top. Use it whenever you edit a prompt or a case file. A mangled `{input}` placeholder
+or a prompt that grew tenfold shows up here for free.
+
+## Scorers
+
+A scorer turns one output into a number. It is the only task-specific part of
+the harness, which is why it is the pluggable one — and the part most likely to
+be silently wrong, which is why the built-ins are tested to the letter.
+
+| `--scorer` | `expected` is | scores |
+|---|---|---|
+| `exact` | a string | 1 if equal after normalising case, whitespace, trailing punctuation |
+| `contains` | a string | 1 if present as a substring |
+| `not_contains` | a string | 1 if **absent** — the injection canary |
+| `regex` | a pattern | 1 if it matches |
+| `json_parse` | *(unused)* | 1 if the output parsed as JSON at all |
+| `json_fields` | an object | the fraction of fields that match; unparseable scores 0 |
+| `judge` | optional reference answer | a second model call against `--rubric`, 1–5 mapped onto 0…1 |
+| `none` | *(unused)* | nothing — records the output for manual reading |
+
+Any key you add to a case is passed through untouched, so a custom scorer can
+read whatever its task needs:
+
+```python
+# score.py — canary absent AND the real job still done
+def score(output, case):
+    if case["expected"].lower() in output.lower():
+        return 0.0, "LEAKED"
+    anchors = case.get("must_contain", [])
+    if not anchors:
+        return 1.0, "clean"
+    hit = [a for a in anchors if a.lower() in output.lower()]
+    return len(hit) / len(anchors), "clean" if len(hit) == len(anchors) else "off-task"
+```
+
+```bash
+evalix run --cases cases.jsonl --scorer custom --scorer-file score.py
+```
+
+A scorer that raises **stops the run** rather than scoring the case zero, and
+cases that haven't started yet are cancelled, not paid for. A broken scorer
+otherwise reports a clean `0.000` that looks exactly like a failing prompt.
+`--keep-going` opts out.
+
+Take a third argument to get a `Context` — with it, your scorer can call a model
+itself, which is all a judge is:
+
+```python
+from evalix import Message, Request
+
+def score(output, case, ctx):
+    verdict = ctx.runner(Request(messages=[Message("user", f"Grade this: {output}")]))
+    ...
+```
+
+Calls made through `ctx.runner` are counted: their tokens show on their own
+`scorer` line and their cost is included in the total. If one of them fails with a
+transport error, the case is recorded as unscored and the run continues.
+
+## Python API
+
+The CLI is a thin wrapper over this, so nothing is terminal-only:
+
+```python
+from evalix import run
+
+report = run(
+    cases="cases.jsonl",
+    prompt="prompts/v2.txt",
+    scorer="json_fields",
+    model="claude-haiku-4-5",
+)
+
+report.score          # 0.727
+report.results        # list[Result] — score, note, output, latency, tokens
+report.cost_usd       # includes any model calls the scorer made
+print(report.render())
+
+if report.diff:       # None on the first run of a case file
+    report.diff.broke # ['t20']
+```
+
+`compare` is the same diff for any two saved runs:
+
+```python
+from evalix import compare
+
+result = compare("v1-lazy", "v5-spec")
+result.diff.broke     # ['t14']
+result.old.meta       # everything recorded about the baseline run
+print(result.render())
+```
+
+## Runners
+
+A runner is any callable taking a `Request` and returning a `Response`. Nothing
+in the core imports capix, so another provider is an argument, not a fork:
+
+```python
+from evalix import run, Response
+
+def my_runner(request):
+    reply = my_client.complete(system=request.system, prompt=request.text)
+    return Response(text=reply, input_tokens=..., output_tokens=...)
+
+run(cases="cases.jsonl", runner=my_runner, model="whatever-you-call-it")
+```
+
+It is also how the whole pipeline gets tested offline — every test in this repo
+runs without an API key, because a fake runner is three lines.
+
+**v1 is single-turn.** `Request.messages` is a list so that tools and extra turns
+can arrive as a field rather than a new major version, but nothing sends more
+than one message today.
+
+## Flags
+
+```
+evalix run
+  --cases FILE           JSONL, one case per line
+  --prompt FILE          the prompt under test
+  --scorer NAME          exact | contains | not_contains | regex | json_parse
+                         | json_fields | judge | none | custom
+  --scorer-file FILE     with --scorer custom: a .py defining score()
+  --rubric FILE          with --scorer judge
+  --judge-model ID       default claude-opus-5
+  -m / --model ID        model under test
+  -e / --effort LEVEL    low|medium|high|xhigh|max (thinking-capable models)
+  -t / --max-tokens N    default 2000
+  --placement system|user  where the prompt file goes
+  --repeat N             run each case N times — consistency check
+  --only-tag TAG         slice to one tag
+  --limit N              first N cases only
+  --label NAME           name this run in the diff output
+  --workers N            parallel requests (default 8)
+  --runs-dir DIR         default <project root>/runs, or $EVALIX_RUNS
+  --keep-going           score a case zero when the scorer raises
+  --dry-run              print what would be sent, send nothing
+  --quiet / --show N     less per-case noise / how many failures to print
+
+evalix compare OLD NEW [--all] [--show N] [--runs-dir DIR]
+```
+
+## Habits worth stealing
+
+- **Change one thing per run.** Two changes and a flat score tells you nothing.
+  `compare` warns when more than one axis moved.
+- **Keep every prompt version.** `v1-lazy.txt`, `v2-spec.txt` — they're the log
+  of what you learned, and you'll want to revert.
+- **Read the failures, not the score.** The score tells you *whether*, the
+  outputs tell you *why*.
+- **Suspect your labels.** When a case won't budge, check whether your expected
+  answer is actually right. Sometimes the model is and you aren't.
+- **Check consistency before celebrating.** `--repeat 3`. A case that flips
+  between runs isn't solved, it's lucky.
+
+## Development
+
+```bash
+uv sync
+uv run pytest -q
+uv run ruff check
+```
+
+The whole suite is offline — no key, no tokens, nothing spent.
+
+## Licence
+
+MIT

evalix-0.1.0/pyproject.toml

@@ -0,0 +1,70 @@
+[project]
+name = "evalix"
+dynamic = ["version"]
+description = "A small eval harness that tells you which cases your prompt change broke."
+readme = "README.md"
+requires-python = ">=3.11"
+license = "MIT"
+license-files = ["LICENSE"]
+authors = [{ name = "Adam Sawicki" }]
+keywords = ["eval", "evaluation", "llm", "prompt-engineering", "benchmark"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3 :: Only",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.14",
+    "Topic :: Software Development :: Testing",
+    "Typing :: Typed",
+]
+# No `License ::` classifier: `license` above is an SPDX expression, and build
+# tools must reject a package that sets both.
+
+# capix is the default runner. The core never imports it — only
+# `evalix.runners.capix` does — so swapping providers is a `runner=` argument,
+# not a fork. It is a hard dependency for now purely so the zero-config path
+# stays one flag; moving it to an optional extra later is a minor bump.
+dependencies = [
+    "capix>=0.1.2",
+    # Used by the CLI only. A library has no business reading someone else's
+    # .env; an application does.
+    "python-dotenv>=1.0",
+]
+
+[project.urls]
+Homepage = "https://github.com/frycz/evalix"
+Repository = "https://github.com/frycz/evalix"
+Issues = "https://github.com/frycz/evalix/issues"
+
+[project.scripts]
+evalix = "evalix.cli:main"
+
+[dependency-groups]
+dev = [
+    "pytest>=8",
+    "ruff>=0.16.7",
+]
+
+[build-system]
+# 1.27 is the first release that understands SPDX `license` and `license-files`.
+requires = ["hatchling>=1.27"]
+build-backend = "hatchling.build"
+
+[tool.hatch.version]
+path = "src/evalix/__init__.py"
+
+[tool.hatch.build.targets.sdist]
+include = ["src", "tests", "README.md", "LICENSE"]
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/evalix"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+
+[tool.ruff]
+line-length = 100