judgetap 0.2.1.dev42001__tar.gz → 0.2.1.dev43001__tar.gz

{judgetap-0.2.1.dev42001 → judgetap-0.2.1.dev43001}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: judgetap
-Version: 0.2.1.dev42001
+Version: 0.2.1.dev43001
 Summary: Fast typed decisions (choice, score, yes/no) across Jev-style engines, plus a guard for coding agents. Early development.
 Project-URL: Homepage, https://github.com/mergesafe-ai/judgetap
 Author-email: Omer Bar-Ness <omer@zsquared.io>
@@ -44,7 +44,7 @@ judgetap guard test "git push --force origin main"
 
 - **Guard.** A hook that checks every shell command (and, in Claude Code, every file write and edit) before it runs. Rules catch common destructive forms: recursive deletes outside the workspace, force-pushes and pushes to protected branches, `DROP`/`DELETE` without `WHERE`, `terraform destroy`, and secrets written to files. With an engine configured, a model judges the rest: is it irreversible? off-task? against a rule in `AGENTS.md`? Without an engine, the rules fail closed for shell commands (anything they can't vouch for asks you); file writes and edits are checked for secrets, your own rules, and writes to the guard's own configuration (which ask).
 - **Library.** One API (`choice`, `score`, `yesno`, `batch`) over every Jev-style decision engine, with a cascade that escalates low-confidence answers to a stronger engine.
-- **Eval.** `judgetap eval cases.jsonl --engines jev,laya` compares engines on your labelled cases: accuracy, calibration (ECE), latency and cost.
+- **Eval.** `judgetap eval cases.jsonl --engines jev,laya` compares engines on your labelled cases: accuracy, calibration (ECE), latency and cost. `judgetap eval --suite banking77 --limit 200 --engines jev` runs a public suite instead (`ag_news`, `banking77`): it is downloaded from Hugging Face on first use, converted to judgetap's case format and cached in `~/.judgetap/suites`; nothing is bundled. `--limit N` takes the same fixed-seed sample every run. Each suite's licence and source URL are listed in `src/judgetap/suites.py`.
 - **Dashboard.** `judgetap dashboard` is a local page with recent decisions, holds, asks, latency and cost per engine. You can mark a hold as a false alarm.
 
 ## Library

{judgetap-0.2.1.dev42001 → judgetap-0.2.1.dev43001}/README.md

@@ -27,7 +27,7 @@ judgetap guard test "git push --force origin main"
 
 - **Guard.** A hook that checks every shell command (and, in Claude Code, every file write and edit) before it runs. Rules catch common destructive forms: recursive deletes outside the workspace, force-pushes and pushes to protected branches, `DROP`/`DELETE` without `WHERE`, `terraform destroy`, and secrets written to files. With an engine configured, a model judges the rest: is it irreversible? off-task? against a rule in `AGENTS.md`? Without an engine, the rules fail closed for shell commands (anything they can't vouch for asks you); file writes and edits are checked for secrets, your own rules, and writes to the guard's own configuration (which ask).
 - **Library.** One API (`choice`, `score`, `yesno`, `batch`) over every Jev-style decision engine, with a cascade that escalates low-confidence answers to a stronger engine.
-- **Eval.** `judgetap eval cases.jsonl --engines jev,laya` compares engines on your labelled cases: accuracy, calibration (ECE), latency and cost.
+- **Eval.** `judgetap eval cases.jsonl --engines jev,laya` compares engines on your labelled cases: accuracy, calibration (ECE), latency and cost. `judgetap eval --suite banking77 --limit 200 --engines jev` runs a public suite instead (`ag_news`, `banking77`): it is downloaded from Hugging Face on first use, converted to judgetap's case format and cached in `~/.judgetap/suites`; nothing is bundled. `--limit N` takes the same fixed-seed sample every run. Each suite's licence and source URL are listed in `src/judgetap/suites.py`.
 - **Dashboard.** `judgetap dashboard` is a local page with recent decisions, holds, asks, latency and cost per engine. You can mark a hold as a false alarm.
 
 ## Library

{judgetap-0.2.1.dev42001 → judgetap-0.2.1.dev43001}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "judgetap"
-version = "0.2.1.dev42001"
+version = "0.2.1.dev43001"
 description = "Fast typed decisions (choice, score, yes/no) across Jev-style engines, plus a guard for coding agents. Early development."
 readme = "README.md"
 license = "Apache-2.0"

{judgetap-0.2.1.dev42001 → judgetap-0.2.1.dev43001}/src/judgetap/__init__.py

@@ -23,7 +23,7 @@ from judgetap.errors import (
 )
 from judgetap.types import Decision, Question
 
-__version__ = "0.2.1.dev42001"  # x-release-please-version
+__version__ = "0.2.1.dev43001"  # x-release-please-version
 
 __all__ = [
     "Cascade",

{judgetap-0.2.1.dev42001 → judgetap-0.2.1.dev43001}/src/judgetap/evaluate.py

@@ -13,6 +13,7 @@ from __future__ import annotations
 
 import json
 import math
+import sys
 from collections.abc import Iterable, Sequence
 from dataclasses import asdict, dataclass, field
 from pathlib import Path
@@ -194,13 +195,35 @@ def main(argv: Sequence[str] | None = None) -> int:
     parser = argparse.ArgumentParser(
         prog="judgetap eval", description=__doc__.split("\n")[0]
     )
-    parser.add_argument("cases", help="JSONL file of labelled cases")
+    parser.add_argument("cases", nargs="?", help="JSONL file of labelled cases")
+    parser.add_argument(
+        "--suite", help="public suite instead of a file (ag_news, banking77)"
+    )
+    parser.add_argument(
+        "--limit", type=int, help="deterministic sample of N cases (fixed seed)"
+    )
     parser.add_argument(
         "--engines", required=True, help="comma-separated specs, e.g. jev,laya"
     )
     parser.add_argument("--json", action="store_true", help="JSON instead of Markdown")
     args = parser.parse_args(argv)
-    cases = load_cases(args.cases)
+    if (args.cases is None) == (args.suite is None):
+        parser.error("give exactly one of a cases file or --suite")
+    if args.suite is not None:
+        from judgetap.suites import suite_path
+
+        if not args.suite.strip():
+            parser.error("--suite needs a suite name (ag_news, banking77)")
+        try:
+            path: str | Path = suite_path(args.suite)
+        except JudgetapError as err:
+            print(f"judgetap eval: {err}", file=sys.stderr)
+            return 1
+    else:
+        path = args.cases
+    from judgetap.suites import sample
+
+    cases = sample(load_cases(path), args.limit)
     reports = [evaluate(cases, load(spec)) for spec in args.engines.split(",")]
     print(to_json(reports) if args.json else to_markdown(reports), end="")
     return 0

judgetap-0.2.1.dev43001/src/judgetap/suites.py

@@ -0,0 +1,177 @@
+"""Public eval suites, converted to judgetap's case format on demand.
+
+Nothing here is vendored: the first ``judgetap eval --suite <name>`` pages the
+split out of the Hugging Face datasets-server JSON API (plain HTTPS, stdlib
+only), converts each row to a ``choice`` case and caches the JSONL under
+``~/.judgetap/suites`` (override with ``JUDGETAP_SUITES_DIR``). Later runs read
+the cache. ``--limit N`` takes a deterministic sample: a fixed seed, so the same
+N cases every run and on every machine.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import random
+import urllib.parse
+import urllib.request
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from judgetap.errors import JudgetapError
+
+ROWS_API = "https://datasets-server.huggingface.co/rows"
+PAGE = 100  # the rows API's maximum page size
+SEED = 88
+TIMEOUT = 30
+
+
+@dataclass(frozen=True)
+class Suite:
+    name: str
+    dataset: str  # Hugging Face dataset id
+    split: str
+    question: str
+    licence: str
+    source: str  # human-readable page for the dataset
+    config: str = "default"
+    text_field: str = "text"
+    label_field: str = "label"
+
+
+REGISTRY: dict[str, Suite] = {
+    s.name: s
+    for s in (
+        Suite(
+            name="ag_news",
+            dataset="fancyzhx/ag_news",
+            split="test",
+            question="Which topic is this news article about?",
+            licence="unknown; AG's corpus is provided for non-commercial research use",
+            source="https://huggingface.co/datasets/fancyzhx/ag_news",
+        ),
+        Suite(
+            name="banking77",
+            dataset="legacy-datasets/banking77",
+            split="test",
+            question="Which intent does this banking customer message express?",
+            licence="CC-BY-4.0",
+            source="https://huggingface.co/datasets/legacy-datasets/banking77",
+        ),
+    )
+}
+
+
+def cache_dir() -> Path:
+    override = os.environ.get("JUDGETAP_SUITES_DIR")
+    return Path(override) if override else Path.home() / ".judgetap" / "suites"
+
+
+def _get_json(url: str) -> dict[str, Any]:
+    req = urllib.request.Request(url, headers={"User-Agent": "judgetap-eval"})
+    with urllib.request.urlopen(req, timeout=TIMEOUT) as resp:
+        return json.load(resp)
+
+
+def _page_url(suite: Suite, offset: int) -> str:
+    query = urllib.parse.urlencode(
+        {
+            "dataset": suite.dataset,
+            "config": suite.config,
+            "split": suite.split,
+            "offset": offset,
+            "length": PAGE,
+        }
+    )
+    return f"{ROWS_API}?{query}"
+
+
+def _label_names(features: list[dict[str, Any]], field: str) -> list[str]:
+    for feature in features:
+        if feature.get("name") == field:
+            # The live rows API puts the ClassLabel under "type" (checked
+            # 2026-09-27); "feature" is accepted too, the key older docs name.
+            for key in ("type", "feature"):
+                spec = feature.get(key)
+                names = spec.get("names") if isinstance(spec, dict) else None
+                if isinstance(names, list) and names:
+                    return [str(n) for n in names]
+    raise JudgetapError(f"no class-label names for {field!r} in the dataset")
+
+
+def _to_case(suite: Suite, names: list[str], item: Any, index: int) -> dict[str, Any]:
+    try:
+        row = item["row"]
+        context = row[suite.text_field]
+        label = int(row[suite.label_field])
+        if not isinstance(context, str) or not 0 <= label < len(names):
+            raise ValueError(f"label {label} or text is out of range")
+    except (KeyError, TypeError, ValueError) as err:
+        raise JudgetapError(
+            f"suite {suite.name!r}: malformed row {index}: {err!r}"
+        ) from err
+    return {
+        "kind": "choice",
+        "question": suite.question,
+        "options": names,
+        "context": context,
+        "label": names[label],
+    }
+
+
+def download(suite: Suite) -> list[dict[str, Any]]:
+    """Every row of the split, as judgetap case dicts."""
+    cases: list[dict[str, Any]] = []
+    names: list[str] | None = None
+    offset, total = 0, None
+    while total is None or offset < total:
+        try:
+            page = _get_json(_page_url(suite, offset))
+        except (OSError, ValueError) as err:
+            raise JudgetapError(f"downloading suite {suite.name!r}: {err}") from err
+        if names is None:
+            names = _label_names(page.get("features", []), suite.label_field)
+        total = int(page.get("num_rows_total", 0))
+        rows = page.get("rows", [])
+        if not rows:
+            if offset < total:
+                raise JudgetapError(
+                    f"suite {suite.name!r}: empty page at offset {offset} "
+                    f"of {total} rows; refusing to cache a partial suite"
+                )
+            break
+        for item in rows:
+            cases.append(_to_case(suite, names, item, offset + len(cases)))
+        offset += len(rows)
+    if not cases:
+        raise JudgetapError(f"suite {suite.name!r} downloaded no rows")
+    return cases
+
+
+def suite_path(name: str) -> Path:
+    """The cached JSONL for ``name``, downloading it the first time."""
+    suite = REGISTRY.get(name)
+    if suite is None:
+        raise JudgetapError(
+            f"unknown suite {name!r}; available: {', '.join(sorted(REGISTRY))}"
+        )
+    path = cache_dir() / f"{suite.name}-{suite.split}.jsonl"
+    if path.exists():
+        return path
+    cases = download(suite)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    tmp = path.with_suffix(".jsonl.part")
+    tmp.write_text("".join(json.dumps(c) + "\n" for c in cases))
+    tmp.replace(path)  # a half-written download never looks cached
+    return path
+
+
+def sample(items: list[Any], limit: int | None) -> list[Any]:
+    """A fixed-seed sample of ``limit`` items, kept in their original order."""
+    if limit is None or limit >= len(items):
+        return items
+    if limit < 1:
+        raise JudgetapError("--limit must be at least 1")
+    picked = sorted(random.Random(SEED).sample(range(len(items)), limit))
+    return [items[i] for i in picked]

judgetap-0.2.1.dev43001/tests/test_suites.py

@@ -0,0 +1,169 @@
+import json
+
+import pytest
+
+import judgetap as sj
+from judgetap import suites
+from judgetap.evaluate import load_cases, main
+
+NAMES = ["World", "Sports", "Business", "Sci/Tech"]
+
+
+def fake_api(total, calls):
+    def get(url):
+        calls.append(url)
+        offset = int(url.split("offset=")[1].split("&")[0])
+        rows = [
+            {"row_idx": i, "row": {"text": f"article {i}", "label": i % 4}}
+            for i in range(offset, min(offset + suites.PAGE, total))
+        ]
+        return {
+            "features": [
+                {"name": "text", "type": {"dtype": "string"}},
+                {"name": "label", "type": {"names": NAMES, "_type": "ClassLabel"}},
+            ],
+            "rows": rows,
+            "num_rows_total": total,
+        }
+
+    return get
+
+
+@pytest.fixture
+def cache(tmp_path, monkeypatch):
+    monkeypatch.setenv("JUDGETAP_SUITES_DIR", str(tmp_path))
+    return tmp_path
+
+
+def test_registry_records_licence_and_source():
+    assert set(suites.REGISTRY) >= {"ag_news", "banking77"}
+    for s in suites.REGISTRY.values():
+        assert s.licence and s.source.startswith("https://huggingface.co/")
+
+
+def test_download_pages_converts_and_caches(cache, monkeypatch):
+    calls = []
+    monkeypatch.setattr(suites, "_get_json", fake_api(250, calls))
+    path = suites.suite_path("ag_news")
+    assert len(calls) == 3 and "dataset=fancyzhx%2Fag_news" in calls[0]
+    assert path.parent == cache
+    cases = load_cases(path)
+    assert len(cases) == 250
+    assert cases[1].label == "Sports" and cases[1].context == "article 1"
+    assert list(cases[0].question.options) == NAMES
+    suites.suite_path("ag_news")  # cached: no further requests
+    assert len(calls) == 3
+
+
+def test_failed_download_leaves_no_cache(cache, monkeypatch):
+    def boom(url):
+        raise OSError("offline")
+
+    monkeypatch.setattr(suites, "_get_json", boom)
+    with pytest.raises(sj.JudgetapError, match="offline"):
+        suites.suite_path("banking77")
+    assert list(cache.iterdir()) == []
+
+
+def test_unknown_suite_lists_available():
+    with pytest.raises(sj.JudgetapError, match="ag_news, banking77"):
+        suites.suite_path("nope")
+
+
+def test_sample_is_deterministic_and_ordered():
+    items = list(range(1000))
+    a, b = suites.sample(items, 10), suites.sample(items, 10)
+    assert a == b == sorted(a) and len(set(a)) == 10
+    assert suites.sample(items, None) is items
+    assert suites.sample(items, 5000) is items
+    with pytest.raises(sj.JudgetapError):
+        suites.sample(items, 0)
+
+
+def test_cli_suite_with_limit(cache, monkeypatch, capsys):
+    from judgetap import engines
+    from judgetap.testing import StaticEngine
+
+    monkeypatch.setattr(suites, "_get_json", fake_api(40, []))
+    monkeypatch.setattr(
+        engines,
+        "load",
+        lambda spec: StaticEngine(lambda q, c: {"World": 1.0}, name=spec),
+    )
+    assert main(["--suite", "ag_news", "--limit", "7", "--engines", "e", "--json"]) == 0
+    report = json.loads(capsys.readouterr().out)[0]
+    assert report["cases"] == 7
+
+
+def test_cli_needs_exactly_one_source(tmp_path):
+    with pytest.raises(SystemExit):
+        main(["--engines", "x"])
+    with pytest.raises(SystemExit):
+        main([str(tmp_path / "c.jsonl"), "--suite", "ag_news", "--engines", "x"])
+
+
+def _page(features, rows, total):
+    return {"features": features, "rows": rows, "num_rows_total": total}
+
+
+LABEL_ONLY = [{"name": "label", "type": {"names": NAMES, "_type": "ClassLabel"}}]
+
+
+def test_label_names_live_type_key_and_feature_key():
+    # The live rows API shape, captured 2026-09-27, uses "type".
+    assert suites._label_names(LABEL_ONLY, "label") == NAMES
+    alt = [{"name": "label", "feature": {"names": NAMES}}]
+    assert suites._label_names(alt, "label") == NAMES
+    with pytest.raises(sj.JudgetapError, match="class-label"):
+        suites._label_names([{"name": "label", "type": {"dtype": "int64"}}], "label")
+
+
+def test_empty_intermediate_page_raises_and_caches_nothing(cache, monkeypatch):
+    real = fake_api(250, [])
+
+    def get(url):
+        page = real(url)
+        if "offset=100" in url:
+            page["rows"] = []
+        return page
+
+    monkeypatch.setattr(suites, "_get_json", get)
+    with pytest.raises(sj.JudgetapError, match="empty page at offset 100 of 250"):
+        suites.suite_path("ag_news")
+    assert list(cache.iterdir()) == []
+
+
+@pytest.mark.parametrize(
+    "row",
+    [
+        {"text": "x"},
+        {"label": 1},
+        {"text": "x", "label": 9},
+        {"text": "x", "label": -1},
+        {"text": "x", "label": "nan"},
+        {"text": None, "label": 1},
+    ],
+)
+def test_malformed_row_raises_judgetap_error(cache, monkeypatch, row):
+    page = _page(LABEL_ONLY, [{"row_idx": 0, "row": row}], 1)
+    monkeypatch.setattr(suites, "_get_json", lambda url: page)
+    with pytest.raises(sj.JudgetapError, match="malformed row 0"):
+        suites.suite_path("ag_news")
+    assert list(cache.iterdir()) == []
+
+
+def test_cli_empty_suite_is_a_usage_error(capsys):
+    with pytest.raises(SystemExit) as exc:
+        main(["--suite", "", "--engines", "x"])
+    assert exc.value.code == 2
+    assert "needs a suite name" in capsys.readouterr().err
+
+
+def test_cli_suite_failure_is_a_clean_error(cache, monkeypatch, capsys):
+    def boom(url):
+        raise OSError("offline")
+
+    monkeypatch.setattr(suites, "_get_json", boom)
+    assert main(["--suite", "ag_news", "--engines", "x"]) == 1
+    err = capsys.readouterr().err
+    assert "offline" in err and "Traceback" not in err