benchkit-cli 0.2.0__tar.gz → 0.2.1__tar.gz

{benchkit_cli-0.2.0/src/benchkit_cli.egg-info → benchkit_cli-0.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: benchkit-cli
-Version: 0.2.0
+Version: 0.2.1
 Summary: Benchmark your local LLMs with real eval suites
 License-Expression: Apache-2.0
 Requires-Python: >=3.11

{benchkit_cli-0.2.0 → benchkit_cli-0.2.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "benchkit-cli"
-version = "0.2.0"
+version = "0.2.1"
 description = "Benchmark your local LLMs with real eval suites"
 readme = "README.md"
 requires-python = ">=3.11"

{benchkit_cli-0.2.0 → benchkit_cli-0.2.1}/src/benchkit/benchmarks/patcheval.py

@@ -39,8 +39,10 @@ _RELEASE_FILE = "dataset.json"
 # files tasks.jsonl names are fetched (not the miner that built it), and every
 # task pins its source archive and hidden tests by sha256, which load_tasks
 # verifies, so a bad or moved upstream file fails loudly instead of scoring.
+# The revision is a pinned commit: a cache fetched from any other revision is
+# fetched again, so an errata release never grades with stale hidden tests.
 _HF_DATASET = "DogukanUrker/PatchEval"
-_HF_REVISION = "main"
+_HF_REVISION = "13c6964b497f6f118bf5af51a2b1b39cc2d612e6"
 _RELEASE = "pilot-20"
 _COMPLETE = ".complete"
 _FETCH_LOCK = threading.Lock()
@@ -178,11 +180,19 @@ def _dataset_dir() -> Path:
     return _configured_root() or _cache_dir()
 
 
+def _cached_revision_matches(root: Path) -> bool:
+    try:
+        marker = (root / _COMPLETE).read_text(encoding="utf-8")
+    except OSError:
+        return False
+    return marker.strip() == _HF_REVISION
+
+
 def _dataset_installed() -> bool:
     configured = _configured_root()
     if configured is not None:
         return (configured / _RELEASE_FILE).is_file()
-    return (_cache_dir() / _COMPLETE).is_file()
+    return _cached_revision_matches(_cache_dir())
 
 
 def _fetch_release(progress: Progress | None = None) -> Path:
@@ -192,8 +202,9 @@ def _fetch_release(progress: Progress | None = None) -> Path:
         return configured
     root = _cache_dir()
     with _FETCH_LOCK:
-        if (root / _COMPLETE).is_file():
+        if _cached_revision_matches(root):
             return root
+        (root / _COMPLETE).unlink(missing_ok=True)
         root.mkdir(parents=True, exist_ok=True)
         done = 0
         with vision._http_client() as client:
@@ -724,6 +735,7 @@ class PatchEval:
     def __init__(self, dataset_root: Path | None = None) -> None:
         self.dataset_root = dataset_root
         self.release = "unloaded"
+        self.revision: int | None = None
 
     def _root(self) -> Path:
         return (self.dataset_root or _fetch_release()).resolve()
@@ -753,6 +765,8 @@ class PatchEval:
                 f"PatchEval dataset must use schema_version {_SCHEMA_VERSION}"
             )
         self.release = str(release.get("release") or "unknown")
+        revision = release.get("revision")
+        self.revision = revision if isinstance(revision, int) else None
         try:
             records = [
                 json.loads(line)
@@ -774,7 +788,10 @@ class PatchEval:
         return [Task(spec.id, "", {"spec": spec}) for spec in specs]
 
     def result_metadata(self, _variant: str | None) -> dict:
-        return {"dataset_release": self.release}
+        metadata: dict = {"dataset_release": self.release}
+        if self.revision is not None:
+            metadata["dataset_revision"] = self.revision
+        return metadata
 
     def build_prompt(self, task: Task) -> str:
         spec = self._spec(task)

{benchkit_cli-0.2.0 → benchkit_cli-0.2.1/src/benchkit_cli.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: benchkit-cli
-Version: 0.2.0
+Version: 0.2.1
 Summary: Benchmark your local LLMs with real eval suites
 License-Expression: Apache-2.0
 Requires-Python: >=3.11

{benchkit_cli-0.2.0 → benchkit_cli-0.2.1}/tests/test_patcheval.py

@@ -20,7 +20,9 @@ import pytest
 from benchkit import catalog
 from benchkit.benchmarks import vision
 from benchkit.benchmarks.patcheval import (
+    _COMPLETE,
     _GENERIC_REPAIR,
+    _HF_REVISION,
     PatchEval,
     _grade_once,
     _trusted_patch,
@@ -139,6 +141,7 @@ class PatchEvalTests(unittest.TestCase):
         self.assertEqual(
             benchmark.result_metadata(None), {"dataset_release": "pilot-20"}
         )
+
         prompt = benchmark.build_prompt(tasks[0])
         self.assertEqual(
             prompt,
@@ -150,6 +153,20 @@ class PatchEvalTests(unittest.TestCase):
         self.assertNotIn("benchmark", prompt.lower())
         self.assertNotIn("hidden", prompt.lower())
 
+    def test_records_the_dataset_revision_when_the_release_has_one(self) -> None:
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            _dataset(root)
+            release = json.loads((root / "dataset.json").read_text("utf-8"))
+            release["revision"] = 2
+            (root / "dataset.json").write_text(json.dumps(release), "utf-8")
+            benchmark = PatchEval(root)
+            benchmark.load_tasks()
+        self.assertEqual(
+            benchmark.result_metadata(None),
+            {"dataset_release": "pilot-20", "dataset_revision": 2},
+        )
+
     def test_rejects_unvalidated_tasks(self) -> None:
         with tempfile.TemporaryDirectory() as directory:
             root = Path(directory)
@@ -716,7 +733,7 @@ def test_patcheval_fetches_its_release_and_verifies_it(cache, tmp_path, monkeypa
     seen: list[str] = []
 
     def serve(request: httpx.Request) -> httpx.Response:
-        prefix = "/datasets/DogukanUrker/PatchEval/resolve/main/"
+        prefix = f"/datasets/DogukanUrker/PatchEval/resolve/{_HF_REVISION}/"
         name = request.url.path.removeprefix(prefix)
         seen.append(name)
         path = release / name
@@ -744,6 +761,14 @@ def test_patcheval_fetches_its_release_and_verifies_it(cache, tmp_path, monkeypa
     catalog.install("patcheval")
     assert len(seen) == 4  # cached
 
+    # A cache from another revision (an older errata) is fetched again.
+    (cache / "patcheval" / "pilot-20" / _COMPLETE).write_text("main\n")
+    assert catalog.status("patcheval").installable
+    catalog.install("patcheval")
+    assert len(seen) == 8
+    marker = (cache / "patcheval" / "pilot-20" / _COMPLETE).read_text()
+    assert marker == _HF_REVISION + "\n"
+
 
 def test_patcheval_rejects_paths_outside_its_cache(cache, monkeypatch):
     monkeypatch.delenv("BENCHKIT_PATCHEVAL_DATASET", raising=False)