benchkit-cli 0.1.0__py3-none-any.whl

benchkit/controls.py

@@ -0,0 +1,87 @@
+"""Run controls the front-ends flip while the engine works."""
+
+from __future__ import annotations
+
+import threading
+
+
+class RunControls:
+    """Thread-safe pause / skip / stop / force-unload switches for the UI."""
+
+    def __init__(self, force_unload: bool = False) -> None:
+        self._stop = threading.Event()
+        self._skip = threading.Event()
+        self._cancel_request = threading.Event()
+        self._running = threading.Event()
+        self._running.set()
+        self._force_unload = force_unload
+
+    @property
+    def stopped(self) -> bool:
+        return self._stop.is_set()
+
+    @property
+    def cancel_event(self) -> threading.Event:
+        """Event passed to clients to kill every active model request."""
+        return self._cancel_request
+
+    @property
+    def cancel_requested(self) -> bool:
+        return self._cancel_request.is_set()
+
+    @property
+    def paused(self) -> bool:
+        return not self._running.is_set()
+
+    @property
+    def force_unload(self) -> bool:
+        return self._force_unload
+
+    def toggle_force_unload(self) -> bool:
+        """Flip force-unload and return the new state."""
+        self._force_unload = not self._force_unload
+        return self._force_unload
+
+    def stop(self) -> None:
+        self._stop.set()
+        self._cancel_request.set()
+        self._running.set()
+
+    def skip_job(self) -> None:
+        self._skip.set()
+        self._cancel_request.set()
+        self._running.set()
+
+    @property
+    def skip_requested(self) -> bool:
+        return self._skip.is_set()
+
+    def pause(self) -> None:
+        self._running.clear()
+
+    def resume(self) -> None:
+        self._running.set()
+
+    def toggle_pause(self) -> bool:
+        """Flip the paused state and return ``True`` when now paused."""
+        if self.paused:
+            self.resume()
+            return False
+        self.pause()
+        return True
+
+    def take_skip(self) -> bool:
+        """Consume a pending skip request."""
+        if self._skip.is_set():
+            self._skip.clear()
+            self._cancel_request.clear()
+            return True
+        return False
+
+    def cancel_active(self) -> None:
+        """Cancel in-flight requests while unwinding an internal job failure."""
+        self._cancel_request.set()
+        self._running.set()
+
+    def wait_while_paused(self) -> None:
+        self._running.wait()

benchkit/datasets/MMLU_LICENSE.txt

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2020 Dan Hendrycks
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

benchkit/datasets/README.md

@@ -0,0 +1,142 @@
+# Dataset attribution
+
+## GPQA Diamond
+
+`gpqa_diamond.jsonl` is a transformed copy of the 198-question GPQA Diamond
+split by David Rein et al. The source answer choices were shuffled with the
+official evaluator's default seed (`0`) and converted from CSV to BenchKit's
+JSONL schema.
+
+- Source: <https://github.com/idavidrein/gpqa>
+- Paper: <https://arxiv.org/abs/2311.12022>
+- Dataset license: [Creative Commons Attribution 4.0 International][cc-by-4.0]
+
+[cc-by-4.0]: https://creativecommons.org/licenses/by/4.0/
+
+## XSTest
+
+`xstest.csv` is the complete 450-prompt XSTest suite: 250 safe prompts across
+ten prompt types and 200 unsafe contrast prompts. BenchKit uses the authors'
+offline string-matching evaluator, which is less reliable than manual or
+LLM-based classification and cannot identify partial refusals.
+
+- Source: <https://github.com/paul-rottger/xstest>
+- Paper: <https://aclanthology.org/2024.naacl-long.301/>
+- Dataset license: [Creative Commons Attribution 4.0 International][cc-by-4.0]
+
+## WinoGrande
+
+`winogrande.jsonl` is a transformed copy of the 1,267-example labeled
+development split from WinoGrande 1.1.
+
+- Source: <https://github.com/allenai/winogrande>
+- Dataset license: [Creative Commons Attribution 2.0][cc-by-2.0]
+
+## PIQA
+
+`piqa.jsonl` is a transformed copy of the 1,838-example labeled validation
+split from Physical Interaction: Question Answering.
+
+- Source: <https://yonatanbisk.com/piqa/data/>
+- Dataset license: [Academic Free License 3.0][afl-3.0]
+
+## MMLU
+
+`mmlu.jsonl` is a transformed copy of the 14,042-question test split covering
+all 57 MMLU subjects. BenchKit evaluates it zero-shot. It is not bundled: it is
+downloaded on first use from a pinned commit of this repository and verified
+against the sha256 in `benchmarks/downloads.py`.
+
+- Source: <https://github.com/hendrycks/test>
+- Dataset license: MIT; see `MMLU_LICENSE.txt`
+
+## MMLU-Pro
+
+`mmlu_pro.jsonl` is a transformed copy of the MMLU-Pro test split: ten-option
+questions across 14 categories, filtered by the authors for ones that require
+reasoning. Only the schema changed - `question_id` became a namespaced
+`task_id` and `options` became `choices`, with the `N/A` padding left by the
+authors' pruning pass dropped, which is why some questions carry fewer than ten
+choices. Regenerate it with `uv run --with datasets python
+scripts/build_mmlu_pro.py`. BenchKit evaluates it zero-shot. Like MMLU it is
+downloaded on first use and checksum-verified rather than bundled.
+
+- Source: <https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro>
+- Code: <https://github.com/TIGER-AI-Lab/MMLU-Pro>
+- Paper: <https://arxiv.org/abs/2406.01574>
+- Dataset license: MIT
+
+## BANKING77
+
+`banking77.jsonl` is a transformed copy of the 3,080-message BANKING77 test
+split, and `banking77_intents.json` is the upstream list of its 77 intents.
+Only the schema changed - each row keeps its upstream position in `task_id` -
+and rows are interleaved round-robin across intents so that any slice covers
+them evenly. Regenerate both with `uv run python scripts/build_banking77.py`.
+
+- Source: <https://github.com/PolyAI-LDN/task-specific-datasets>
+- Paper: Casanueva et al., "Efficient Intent Detection with Dual Sentence
+  Encoders" (2020), <https://arxiv.org/abs/2003.04807>
+- Dataset license: [Creative Commons Attribution 4.0 International][cc-by-4.0]
+
+## OpenBookQA
+
+`openbookqa.jsonl` is a transformed copy of the 500-question OpenBookQA test
+split.
+
+- Source: <https://github.com/allenai/OpenBookQA>
+- Dataset license: Apache 2.0
+
+## BoolQ
+
+`boolq.jsonl` is a transformed copy of the 3,270-example labeled validation
+split distributed with SuperGLUE.
+
+- Source: <https://github.com/google-research-datasets/boolean-questions>
+- Dataset license: [Creative Commons Attribution-ShareAlike 3.0][cc-by-sa-3.0]
+
+## IFEval
+
+`ifeval.jsonl` is a transformed copy of the 541 prompts distributed with the
+IFEval paper. Only the schema changed: `key` became `task_id`, and the
+`instruction_id_list` / `kwargs` pairs are carried over unmodified.
+
+- Source: <https://github.com/google-research/google-research/tree/master/instruction_following_eval>
+- Paper: <https://arxiv.org/abs/2311.07911>
+- Dataset license: Apache 2.0
+
+## RULER
+
+RULER deterministically generates its NIAH, variable-tracking, CWE, and FWE
+tasks. Essay-backed NIAH uses upstream `PaulGrahamEssays.json`; QA-1 and QA-2
+use SQuAD 2.0 development and HotpotQA distractor validation data cached under
+`~/.cache/benchkit/ruler`. BenchKit downloads the QA files on first use. The
+essay corpus is generated with NVIDIA/RULER's
+`download_paulgraham_essay.py`.
+retrieval and variable-tracking prompts at runtime for each context bucket.
+
+The `ruler` benchmark is the practical preset: three samples from each of the
+13 task families in every supported context bucket (234 generations at
+4K–128K). Use `ruler-full` to opt into the research-scale configuration of 500
+samples per family (39,000 generations), or slice it explicitly (for example,
+`ruler-full:13`) for round-robin coverage of all task families.
+
+- Source design: <https://github.com/NVIDIA/RULER>
+- Paper: <https://arxiv.org/abs/2404.06654>
+- Reference implementation license: Apache 2.0
+
+[afl-3.0]: https://opensource.org/license/afl-3-0-php
+[cc-by-2.0]: https://creativecommons.org/licenses/by/2.0/
+[cc-by-sa-3.0]: https://creativecommons.org/licenses/by-sa/3.0/
+
+## Minecraft block ids (mc-arena)
+
+`mc_blocks_1_20_1.jsonl` is the block table for Minecraft Java Edition 1.20.1,
+extracted from `pc/1.20/blocks.json` and reduced to the block id, whether it is
+an air variant, and whether it is transparent. `mc-arena` validates generated
+block ids against it, and the same version is what the bundled
+prismarine-viewer renders (see `../mc_viewer/build/README.md`).
+
+- Source: <https://github.com/mc-bench/mc-data-files>, a packaging of
+  <https://github.com/PrismarineJS/minecraft-data>
+- Dataset license: MIT