jevkit-bench 0.1.0__tar.gz

jevkit_bench-0.1.0/.gitignore

@@ -0,0 +1,12 @@
+__pycache__/
+*.py[cod]
+.venv/
+venv/
+dist/
+build/
+*.egg-info/
+.pytest_cache/
+.coverage
+htmlcov/
+.ruff_cache/
+uv.lock

jevkit_bench-0.1.0/PKG-INFO

@@ -0,0 +1,84 @@
+Metadata-Version: 2.5
+Name: jevkit-bench
+Version: 0.1.0
+Summary: Score a labeled TypeSafe Jev suite for accuracy and cost, and compare two runs. Catches regressions aggregate accuracy hides.
+Project-URL: Homepage, https://github.com/pjdurden/jevkit-py
+Project-URL: Issues, https://github.com/pjdurden/jevkit-py/issues
+Author: Prajjwal Chittori
+License-Expression: MIT
+Keywords: accuracy,benchmark,cost,evaluation,jev,system-one,typesafe
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Software Development :: Testing
+Requires-Python: >=3.10
+Requires-Dist: jevkit-core>=0.2.0
+Description-Content-Type: text/markdown
+
+# jevkit-bench
+
+Score a labeled Jev suite for accuracy **and** cost, and compare two runs.
+
+Any one number alone is easy to win. Accuracy without cost hides that you spent
+ten times the tokens; cost without accuracy hides that you broke the task. This
+reports them together.
+
+> Unofficial and unaffiliated with TypeSafe.
+
+```bash
+pip install jevkit-bench
+```
+
+## Use
+
+```python
+from jevkit_core import read_records
+from jevkit_bench import compare_suites, score_records
+
+suite = score_records(read_records("suite.jevl"))
+print(suite.summary())
+
+for failure in suite.failures()[:5]:
+    print(failure.question_id, failure.predicted, "should be", failure.label)
+```
+
+`failures()` sorts by probability descending, so the most confident wrong answers
+come first. Those are the interesting bugs: a wrong answer at 0.35 is the model
+telling you it was unsure, while a wrong answer at 0.98 is a question that needs
+rewriting.
+
+## Comparing runs
+
+```python
+comparison = compare_suites(baseline, candidate)
+print(comparison.summary())
+print(comparison.regressions)   # right before, wrong now
+```
+
+Aggregate accuracy can hold perfectly steady while the set of things you get
+right churns underneath. That matters when a specific case is the one you
+promised someone would work, so regressions and fixes are tracked individually
+rather than netted off.
+
+## CLI
+
+```bash
+jevkit-bench suite.jevl
+jevkit-bench suite.jevl --baseline last-week.jevl
+jevkit-bench suite.jevl --min-accuracy 0.90              # CI gate
+jevkit-bench suite.jevl --baseline last-week.jevl --max-regressions 0
+jevkit-bench suite.jevl --show-failures 10
+```
+
+## Cost
+
+Computed from input tokens at $0.042 per million, Jev's published price. Output
+tokens are free on Jev, so they are reported but never billed. Override with
+`--price-per-mtok` if your plan differs.
+
+## License
+
+MIT

jevkit_bench-0.1.0/README.md

@@ -0,0 +1,64 @@
+# jevkit-bench
+
+Score a labeled Jev suite for accuracy **and** cost, and compare two runs.
+
+Any one number alone is easy to win. Accuracy without cost hides that you spent
+ten times the tokens; cost without accuracy hides that you broke the task. This
+reports them together.
+
+> Unofficial and unaffiliated with TypeSafe.
+
+```bash
+pip install jevkit-bench
+```
+
+## Use
+
+```python
+from jevkit_core import read_records
+from jevkit_bench import compare_suites, score_records
+
+suite = score_records(read_records("suite.jevl"))
+print(suite.summary())
+
+for failure in suite.failures()[:5]:
+    print(failure.question_id, failure.predicted, "should be", failure.label)
+```
+
+`failures()` sorts by probability descending, so the most confident wrong answers
+come first. Those are the interesting bugs: a wrong answer at 0.35 is the model
+telling you it was unsure, while a wrong answer at 0.98 is a question that needs
+rewriting.
+
+## Comparing runs
+
+```python
+comparison = compare_suites(baseline, candidate)
+print(comparison.summary())
+print(comparison.regressions)   # right before, wrong now
+```
+
+Aggregate accuracy can hold perfectly steady while the set of things you get
+right churns underneath. That matters when a specific case is the one you
+promised someone would work, so regressions and fixes are tracked individually
+rather than netted off.
+
+## CLI
+
+```bash
+jevkit-bench suite.jevl
+jevkit-bench suite.jevl --baseline last-week.jevl
+jevkit-bench suite.jevl --min-accuracy 0.90              # CI gate
+jevkit-bench suite.jevl --baseline last-week.jevl --max-regressions 0
+jevkit-bench suite.jevl --show-failures 10
+```
+
+## Cost
+
+Computed from input tokens at $0.042 per million, Jev's published price. Output
+tokens are free on Jev, so they are reported but never billed. Override with
+`--price-per-mtok` if your plan differs.
+
+## License
+
+MIT

jevkit_bench-0.1.0/pyproject.toml

@@ -0,0 +1,33 @@
+[project]
+name = "jevkit-bench"
+version = "0.1.0"
+description = "Score a labeled TypeSafe Jev suite for accuracy and cost, and compare two runs. Catches regressions aggregate accuracy hides."
+readme = "README.md"
+requires-python = ">=3.10"
+license = "MIT"
+authors = [{ name = "Prajjwal Chittori" }]
+keywords = ["jev", "typesafe", "system-one", "benchmark", "evaluation", "accuracy", "cost"]
+classifiers = [
+  "Development Status :: 3 - Alpha",
+  "Intended Audience :: Developers",
+  "License :: OSI Approved :: MIT License",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
+  "Topic :: Software Development :: Testing",
+]
+dependencies = ["jevkit-core>=0.2.0"]
+
+[project.scripts]
+jevkit-bench = "jevkit_bench.cli:main"
+
+[project.urls]
+Homepage = "https://github.com/pjdurden/jevkit-py"
+Issues = "https://github.com/pjdurden/jevkit-py/issues"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/jevkit_bench"]

jevkit_bench-0.1.0/src/jevkit_bench/__init__.py

@@ -0,0 +1,14 @@
+"""Score a labeled Jev suite for accuracy and cost, and compare runs.
+
+Answers the question you have to defend: for this task, on my data, is Jev good
+enough and what does it cost? Accuracy, tokens and dollars together, because any
+one of them alone is easy to win.
+"""
+
+from .score import (PRICE_PER_MTOK, QuestionResult, SuiteComparison, SuiteResult,
+                    compare_suites, score_records)
+
+__version__ = "0.1.0"
+
+__all__ = ["score_records", "compare_suites", "SuiteResult", "SuiteComparison",
+           "QuestionResult", "PRICE_PER_MTOK", "__version__"]

jevkit_bench-0.1.0/src/jevkit_bench/cli.py

@@ -0,0 +1,105 @@
+"""``jevkit-bench`` command line interface."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+
+from jevkit_core import RecordFormatError, read_records
+
+from .score import PRICE_PER_MTOK, compare_suites, score_records
+
+EXIT_OK = 0
+EXIT_FAILED_GATE = 1
+EXIT_USAGE = 2
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(
+        prog="jevkit-bench",
+        description="Score a labeled .jevl suite for accuracy and cost, and compare two "
+                    "runs of it. Never calls the API.",
+    )
+    parser.add_argument("suite", help="scored .jevl suite")
+    parser.add_argument("--baseline", help="a previous run, to compare against")
+    parser.add_argument("--question", action="append", dest="questions",
+                        help="restrict to this question id (repeatable)")
+    parser.add_argument("--price-per-mtok", type=float, default=PRICE_PER_MTOK,
+                        help=f"input price per million tokens (default {PRICE_PER_MTOK})")
+    parser.add_argument("--min-accuracy", type=float, help="exit non-zero below this")
+    parser.add_argument("--max-regressions", type=int,
+                        help="exit non-zero above this many regressions (needs --baseline)")
+    parser.add_argument("--show-failures", type=int, default=0, metavar="N",
+                        help="print the N most confident wrong answers")
+    parser.add_argument("--format", choices=("text", "json"), default="text")
+    args = parser.parse_args(argv)
+
+    if args.max_regressions is not None and not args.baseline:
+        print("jevkit-bench: --max-regressions needs --baseline", file=sys.stderr)
+        return EXIT_USAGE
+
+    try:
+        suite = score_records(read_records(args.suite), question_ids=args.questions,
+                              price_per_mtok=args.price_per_mtok)
+        baseline = (
+            score_records(read_records(args.baseline), question_ids=args.questions,
+                          price_per_mtok=args.price_per_mtok)
+            if args.baseline else None
+        )
+    except (OSError, RecordFormatError) as exc:
+        print(f"jevkit-bench: {exc}", file=sys.stderr)
+        return EXIT_USAGE
+
+    if not suite.count:
+        print("jevkit-bench: nothing scored. Records need a 'label' object keyed by "
+              "question id.", file=sys.stderr)
+        return EXIT_USAGE
+
+    comparison = compare_suites(baseline, suite) if baseline else None
+
+    if args.format == "json":
+        payload = suite.to_dict()
+        if comparison:
+            payload["comparison"] = {
+                "baseline_accuracy": comparison.baseline.accuracy,
+                "accuracy_delta": comparison.accuracy_delta,
+                "cost_delta": comparison.cost_delta,
+                "regressions": [list(k) for k in comparison.regressions],
+                "fixes": [list(k) for k in comparison.fixes],
+            }
+        print(json.dumps(payload, indent=2))
+    else:
+        print(suite.summary())
+        if comparison:
+            print()
+            print(comparison.summary())
+            if comparison.regressions:
+                print("\nregressions:")
+                for rid, qid in comparison.regressions[:20]:
+                    print(f"  {rid[7:19]}  {qid}")
+        if args.show_failures:
+            failures = suite.failures()[: args.show_failures]
+            if failures:
+                print(f"\nmost confident wrong answers ({len(failures)}):")
+                for f in failures:
+                    # JSON rendering keeps this byte-identical to the
+                    # JavaScript CLI; repr() would quote with '.
+                    print(f"  {f.request_id[7:19]}  {f.question_id}: "
+                          f"said {json.dumps(f.predicted)}, "
+                          f"label {json.dumps(f.label)}, p={f.probability:.3f}")
+
+    if args.min_accuracy is not None and suite.accuracy < args.min_accuracy:
+        print(f"jevkit-bench: accuracy {suite.accuracy:.4f} below {args.min_accuracy}",
+              file=sys.stderr)
+        return EXIT_FAILED_GATE
+    if args.max_regressions is not None and comparison is not None:
+        if len(comparison.regressions) > args.max_regressions:
+            print(f"jevkit-bench: {len(comparison.regressions)} regressions exceed "
+                  f"{args.max_regressions}", file=sys.stderr)
+            return EXIT_FAILED_GATE
+    return EXIT_OK
+
+
+if __name__ == "__main__":  # pragma: no cover
+    raise SystemExit(main())

jevkit_bench-0.1.0/src/jevkit_bench/score.py

@@ -0,0 +1,233 @@
+"""Scoring a labeled `.jevl` suite, and comparing two runs of it.
+
+The question this answers is the one you actually have to defend: for this task,
+on my data, is Jev good enough, and what does it cost? That needs accuracy and
+cost and latency together, because any one of them alone is easy to win.
+
+Costs are computed from a price per million input tokens, defaulting to Jev's
+published $0.042. Output tokens are free on Jev and are reported but not billed.
+"""
+
+from __future__ import annotations
+
+from collections import defaultdict
+from dataclasses import dataclass, field
+from typing import Any, Iterable, Sequence
+
+from jevkit_core import Record, parse_answers
+
+__all__ = ["QuestionResult", "SuiteResult", "score_records", "compare_suites",
+           "SuiteComparison", "PRICE_PER_MTOK"]
+
+PRICE_PER_MTOK = 0.042
+
+
+@dataclass(frozen=True)
+class QuestionResult:
+    """One scored question from one record."""
+
+    request_id: str
+    question_id: str
+    type: str
+    predicted: Any
+    label: Any
+    correct: bool
+    probability: float
+    confidence: float | None
+    tags: tuple[str, ...] = ()
+
+
+@dataclass
+class SuiteResult:
+    """Everything scored from one run of a suite."""
+
+    results: list[QuestionResult] = field(default_factory=list)
+    input_tokens: int = 0
+    output_tokens: int = 0
+    records: int = 0
+    unlabeled: int = 0
+    models: set[str] = field(default_factory=set)
+    price_per_mtok: float = PRICE_PER_MTOK
+
+    @property
+    def count(self) -> int:
+        return len(self.results)
+
+    @property
+    def correct(self) -> int:
+        return sum(1 for r in self.results if r.correct)
+
+    @property
+    def accuracy(self) -> float:
+        return self.correct / self.count if self.count else 0.0
+
+    @property
+    def cost(self) -> float:
+        """Input-token cost in dollars. Jev bills input only."""
+        return self.input_tokens / 1_000_000 * self.price_per_mtok
+
+    @property
+    def cost_per_question(self) -> float:
+        return self.cost / self.count if self.count else 0.0
+
+    def by_tag(self) -> dict[str, tuple[int, float]]:
+        """tag -> (n, accuracy). A result with several tags counts under each."""
+        buckets: dict[str, list[QuestionResult]] = defaultdict(list)
+        for r in self.results:
+            for tag in r.tags:
+                buckets[tag].append(r)
+        return {
+            tag: (len(rs), sum(1 for r in rs if r.correct) / len(rs))
+            for tag, rs in sorted(buckets.items())
+        }
+
+    def by_question(self) -> dict[str, tuple[int, float]]:
+        """question id -> (n, accuracy). Finds the one question dragging the suite."""
+        buckets: dict[str, list[QuestionResult]] = defaultdict(list)
+        for r in self.results:
+            buckets[r.question_id].append(r)
+        return {
+            qid: (len(rs), sum(1 for r in rs if r.correct) / len(rs))
+            for qid, rs in sorted(buckets.items())
+        }
+
+    def failures(self) -> list[QuestionResult]:
+        """Wrong answers, most confident first: the most interesting bugs."""
+        return sorted(
+            (r for r in self.results if not r.correct),
+            key=lambda r: -r.probability,
+        )
+
+    def summary(self) -> str:
+        lines = [
+            f"records:   {self.records}"
+            + (f" ({self.unlabeled} unlabeled, skipped)" if self.unlabeled else ""),
+            f"scored:    {self.count} question(s)",
+            f"model(s):  {', '.join(sorted(self.models)) or '?'}",
+            f"accuracy:  {self.accuracy:.4f}  ({self.correct}/{self.count})",
+            f"tokens:    {self.input_tokens:,} in, {self.output_tokens:,} out (out is free)",
+            f"cost:      ${self.cost:.6f}  (${self.cost_per_question:.8f}/question "
+            f"at ${self.price_per_mtok}/Mtok)",
+        ]
+        tags = self.by_tag()
+        if tags:
+            lines.append("by tag:")
+            for tag, (n, acc) in tags.items():
+                lines.append(f"  {tag:<24} {acc:.4f}  (n={n})")
+        questions = self.by_question()
+        if len(questions) > 1:
+            lines.append("by question:")
+            for qid, (n, acc) in questions.items():
+                lines.append(f"  {qid:<24} {acc:.4f}  (n={n})")
+        return "\n".join(lines)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "records": self.records,
+            "unlabeled": self.unlabeled,
+            "scored": self.count,
+            "models": sorted(self.models),
+            "accuracy": self.accuracy,
+            "correct": self.correct,
+            "input_tokens": self.input_tokens,
+            "output_tokens": self.output_tokens,
+            "cost": self.cost,
+            "by_tag": {k: {"n": n, "accuracy": a} for k, (n, a) in self.by_tag().items()},
+            "by_question": {k: {"n": n, "accuracy": a} for k, (n, a) in self.by_question().items()},
+        }
+
+
+def score_records(
+    records: Iterable[Record],
+    *,
+    question_ids: Iterable[str] | None = None,
+    price_per_mtok: float = PRICE_PER_MTOK,
+) -> SuiteResult:
+    """Score every labeled answer in a suite.
+
+    Records with no ``label`` are counted and skipped rather than silently
+    dropped, so a suite that quietly lost its labels is visible in the summary
+    instead of showing a suspiciously perfect score over three records.
+    """
+    wanted = set(question_ids) if question_ids is not None else None
+    suite = SuiteResult(price_per_mtok=price_per_mtok)
+
+    for record in records:
+        suite.records += 1
+        suite.models.add(record.model)
+        usage = record.usage or {}
+        suite.input_tokens += int(usage.get("input_tokens") or 0)
+        suite.output_tokens += int(usage.get("output_tokens") or 0)
+
+        if not record.label:
+            suite.unlabeled += 1
+            continue
+
+        answers = parse_answers(record.answers)
+        for qid, label in record.label.items():
+            if wanted is not None and qid not in wanted:
+                continue
+            answer = answers.get(qid)
+            if answer is None:
+                continue
+            suite.results.append(QuestionResult(
+                request_id=record.request_id,
+                question_id=qid,
+                type=answer.type,
+                predicted=answer.predicted(),
+                label=label,
+                correct=answer.is_correct(label),
+                probability=answer.top_probability,
+                confidence=answer.confidence,
+                tags=tuple(record.tags),
+            ))
+    return suite
+
+
+@dataclass
+class SuiteComparison:
+    """Two runs of the same suite, side by side."""
+
+    baseline: SuiteResult
+    candidate: SuiteResult
+    regressions: list[tuple[str, str]] = field(default_factory=list)
+    fixes: list[tuple[str, str]] = field(default_factory=list)
+
+    @property
+    def accuracy_delta(self) -> float:
+        return self.candidate.accuracy - self.baseline.accuracy
+
+    @property
+    def cost_delta(self) -> float:
+        return self.candidate.cost - self.baseline.cost
+
+    def summary(self) -> str:
+        return "\n".join([
+            f"accuracy:  {self.baseline.accuracy:.4f} -> {self.candidate.accuracy:.4f} "
+            f"({self.accuracy_delta:+.4f})",
+            f"cost:      ${self.baseline.cost:.6f} -> ${self.candidate.cost:.6f} "
+            f"({self.cost_delta:+.6f})",
+            f"regressed: {len(self.regressions)} (right before, wrong now)",
+            f"fixed:     {len(self.fixes)} (wrong before, right now)",
+        ])
+
+
+def compare_suites(baseline: SuiteResult, candidate: SuiteResult) -> SuiteComparison:
+    """Compare two scored runs by (request, question).
+
+    Aggregate accuracy can hold steady while the *set* of things you get right
+    churns underneath, which matters when a specific case is the one you promised
+    someone would work. Regressions and fixes are tracked individually for that
+    reason.
+    """
+    def index(suite: SuiteResult) -> dict[tuple[str, str], QuestionResult]:
+        return {(r.request_id, r.question_id): r for r in suite.results}
+
+    before, after = index(baseline), index(candidate)
+    shared = set(before) & set(after)
+    return SuiteComparison(
+        baseline=baseline,
+        candidate=candidate,
+        regressions=sorted(k for k in shared if before[k].correct and not after[k].correct),
+        fixes=sorted(k for k in shared if not before[k].correct and after[k].correct),
+    )

jevkit_bench-0.1.0/tests/test_bench.py

@@ -0,0 +1,79 @@
+import pytest
+from jevkit_core import Record
+from jevkit_bench import compare_suites, score_records
+
+
+def rec(choice, label, probs=None, state="s", usage=None, tags=()):
+    return Record(
+        model="jev-1.13.0", state=state,
+        questions={"team": {"type": "choice", "instructions": "Which team"}},
+        answers={"team": {"type": "choice", "choice": choice,
+                          "probabilities": probs or {choice: 1.0}, "confidence": 0.9}},
+        label={"team": label}, usage=usage, tags=list(tags),
+    )
+
+
+def test_accuracy_counts_matches():
+    suite = score_records([rec("a", "a"), rec("b", "a", state="t")])
+    assert suite.count == 2 and suite.correct == 1
+    assert suite.accuracy == 0.5
+
+
+def test_unlabeled_records_are_counted_and_skipped():
+    unlabeled = Record(model="m", state="u", questions={"q": {"type": "noul"}},
+                       answers={"q": {"type": "noul", "noul": 0.9}})
+    suite = score_records([rec("a", "a"), unlabeled])
+    assert suite.records == 2 and suite.unlabeled == 1 and suite.count == 1
+
+
+def test_cost_uses_input_tokens_only():
+    suite = score_records([rec("a", "a", usage={"input_tokens": 1_000_000,
+                                                "output_tokens": 500_000})])
+    assert suite.input_tokens == 1_000_000
+    assert suite.output_tokens == 500_000
+    assert suite.cost == pytest.approx(0.042)
+
+
+def test_price_override_is_respected():
+    suite = score_records([rec("a", "a", usage={"input_tokens": 1_000_000})],
+                          price_per_mtok=1.0)
+    assert suite.cost == pytest.approx(1.0)
+
+
+def test_failures_are_sorted_most_confident_first():
+    suite = score_records([
+        rec("b", "a", probs={"b": 0.6, "a": 0.4}, state="one"),
+        rec("b", "a", probs={"b": 0.99, "a": 0.01}, state="two"),
+    ])
+    assert [round(f.probability, 2) for f in suite.failures()] == [0.99, 0.6]
+
+
+def test_by_tag_buckets_results():
+    suite = score_records([rec("a", "a", tags=["routing"]),
+                           rec("b", "a", state="t", tags=["routing"])])
+    assert suite.by_tag()["routing"] == (2, 0.5)
+
+
+def test_by_question_finds_the_weak_question():
+    good = Record(model="m", state="s",
+                  questions={"a": {"type": "noul"}, "b": {"type": "noul"}},
+                  answers={"a": {"type": "noul", "noul": 0.9},
+                           "b": {"type": "noul", "noul": 0.9}},
+                  label={"a": True, "b": False})
+    by_q = score_records([good]).by_question()
+    assert by_q["a"] == (1, 1.0) and by_q["b"] == (1, 0.0)
+
+
+def test_comparison_tracks_regressions_and_fixes_separately():
+    baseline = score_records([rec("a", "a", state="one"), rec("b", "a", state="two")])
+    candidate = score_records([rec("b", "a", state="one"), rec("a", "a", state="two")])
+    comparison = compare_suites(baseline, candidate)
+    # Accuracy is unchanged at 0.5, but the set of correct answers swapped entirely.
+    assert comparison.accuracy_delta == 0.0
+    assert len(comparison.regressions) == 1
+    assert len(comparison.fixes) == 1
+
+
+def test_empty_suite_reports_zero_rather_than_dividing_by_zero():
+    suite = score_records([])
+    assert suite.accuracy == 0.0 and suite.cost_per_question == 0.0