tuieval 0.2.0.dev77__tar.gz → 0.2.0.dev79__tar.gz

{tuieval-0.2.0.dev77 → tuieval-0.2.0.dev79}/CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## Unreleased
 
+- Early stop: `tuieval run --stop-early`, **Stop a pack once it fails** in Setup, or `stop_early = true` in a pack.toml ends a pack once its FAIL is certain (a critical failure, or an accuracy or truncation bar out of reach), asking first the questions models fail most. Verdicts also say FAIL when a partial run can no longer pass.
 - `tuieval compare --reference <model>`: every other model paired against one on the questions both answered, per pack and overall: the difference with a 95% CI, better/tie/worse, critical failures and tokens vs the reference. `--packs` limits any compare to some packs.
 - Fix: a question description or grader reason containing `[` no longer crashes the run screen.
 - Retest asks per pack before it starts: from zero, continue or leave out. It now really starts from zero, and replaced smoke results go to history/.

{tuieval-0.2.0.dev77 → tuieval-0.2.0.dev79}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: tuieval
-Version: 0.2.0.dev77
+Version: 0.2.0.dev79
 Summary: Evaluate local and frontier LLMs on your own questions: accuracy, speed, tokens and PASS/FAIL verdicts, in the terminal.
 Project-URL: Homepage, https://github.com/ashe-wb/tuieval
 Project-URL: Issues, https://github.com/ashe-wb/tuieval/issues

{tuieval-0.2.0.dev77 → tuieval-0.2.0.dev79}/docs/writing-packs.md

@@ -34,6 +34,7 @@ needs = ["vision"]               # vision | tools | long_context | <python modul
 order = 30                       # position in the list
 tests = ["tests.yaml"]           # optional; default: tests.* first, then every other .yaml/.csv file
 screen = 20                      # tests in a Screen run (spread across categories, one variant per group)
+stop_early = true                # stop a run of this pack once its FAIL is certain (default false; see below)
 
 [certify]
 repeat = 3                       # repeats in a Certify run (default: models.toml [defaults] repeat)
@@ -48,6 +49,8 @@ min_consistency = 0.95           # share of variant groups where every variant p
 
 `group` doubles as the **use case** in the readiness verdicts: all packs in a group must PASS for the use case to PASS. For example, a "Coding" use case could be a `coding` pack plus a `coding-sql` pack.
 
+`stop_early` suits packs a weak model fails quickly, like a regression set of routine cases: a run stops the pack at the first critical failure, or once the accuracy or truncation bar is out of reach even if every remaining answer passes, saves what it has (the verdict is FAIL) and moves on. It asks first the questions models have failed most often. `tuieval run --stop-early` (or **Stop a pack once it fails** in Setup) does this for every pack in the run. A later Certify fills in the rest.
+
 `needs` entries other than `vision`, `tools` and `long_context` name Python modules the pack's grading needs (for example `pandas`, when your hidden tests use it). The pack is skipped, with a note, where that module isn't installed in tuieval's Python environment.
 
 ## Tests (YAML)

{tuieval-0.2.0.dev77 → tuieval-0.2.0.dev79}/src/tuieval/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.2.0.dev77'
-__version_tuple__ = version_tuple = (0, 2, 0, 'dev77')
+__version__ = version = '0.2.0.dev79'
+__version_tuple__ = version_tuple = (0, 2, 0, 'dev79')
 
 __commit_id__ = commit_id = None

{tuieval-0.2.0.dev77 → tuieval-0.2.0.dev79}/src/tuieval/engine.py

@@ -28,6 +28,7 @@ interrupted run resumes from its .partial.jsonl, request by request.
 Events are delivered as on_event(kind, **data), from the engine thread.
 """
 import base64
+import collections
 import contextlib
 import dataclasses
 import fcntl
@@ -438,6 +439,8 @@ class Job:
     finished: float = 0.0
     resumed: int = 0        # answers already done when this sitting started
     earlier: list = dataclasses.field(default_factory=list)  # earlier sittings (see Engine._sittings)
+    stop_early: bool = False  # stop once the pack's FAIL is certain (verdict.fail_certain)
+    first: list = dataclasses.field(default_factory=list)   # test ids round 1 asks first (most failed)
 
     @property
     def label(self):
@@ -1152,14 +1155,15 @@ class Engine:
         return not current and header_matches(head, pack.fingerprint, settings)
 
     # ---- planning
-    def plan(self, labels, pack_names, repeat=None, tier="screen", force=False, tests=None):
+    def plan(self, labels, pack_names, repeat=None, tier="screen", force=False, tests=None, stop_early=None):
         """Jobs for models x packs. tier: smoke (3 tests), screen (a spread sample), certify (all
         tests). repeat: times each question is asked; None = the tier default (1 for smoke and
         screen, the pack's certification repeats for certify). tests: {pack: [test ids]} runs just
         those tests of a pack instead of the tier's sample (unknown ids raise PackError). Earlier
         matching results are kept and only missing answers are run, unless force; force with
         picked tests reruns those and keeps the pack's other answers. force is True, False, or the
-        keys ("label/pack") of the jobs to retest, the others resuming."""
+        keys ("label/pack") of the jobs to retest, the others resuming. stop_early: stop each pack
+        once its FAIL is certain, asking first what models fail most; None = each pack's stop_early."""
         if tier not in TIERS:
             raise ValueError(f"tier must be one of {TIERS}")
         jobs = []
@@ -1173,7 +1177,10 @@ class Engine:
                 job = Job(m, pack, reps, pack.pick(picked) if picked else pack.select(tier),
                           self.result_path(label, name, tier), effective_sampling(self.cfg, m), tier=tier,
                           merge=not retest or bool(picked), redo=retest and bool(picked), fresh=retest,
-                          settings=self.settings(m))
+                          settings=self.settings(m),
+                          stop_early=pack.stop_early if stop_early is None else bool(stop_early))
+                if job.stop_early:
+                    job.first = self.most_failed_first(pack)
                 sv = self.serving(m)
                 status = self.result_status(label, name, tier)
                 job.discards = self._discards(job)
@@ -2005,13 +2012,30 @@ class Engine:
 
     @staticmethod
     def round_order(job, rep):
-        """The tests in the order round `rep` asks them: the pack's order first, then a fixed
-        shuffle per round (the same on every run, so resuming and reruns ask in the same order)."""
+        """The tests in the order round `rep` asks them: the pack's order first (with an early stop,
+        the most failed first), then a fixed shuffle per round (the same on every run, so resuming
+        and reruns ask in the same order)."""
         tests = list(job.tests)
         if rep:
             random.Random(f"{job.pack.name}:{rep}").shuffle(tests)
+        elif job.first:
+            rank = {tid: i for i, tid in enumerate(job.first)}
+            tests.sort(key=lambda t: rank.get(t["id"], len(rank)))
         return tests
 
+    def most_failed_first(self, pack):
+        """The pack's test ids, the ones models failed most often first (critical failures count
+        double), from every model's stored results; ties keep the pack's order."""
+        fails, seen = collections.Counter(), collections.Counter()
+        for m in self.cfg["models"]:
+            for r in self.records(m["label"], pack.name):
+                if client.server_error_row(r):
+                    continue
+                seen[r["test"]] += 1
+                fails[r["test"]] += (not r["pass"]) + (r.get("severity") == "critical")
+        order = {t["id"]: i for i, t in enumerate(pack.tests)}
+        return sorted(order, key=lambda tid: (-(fails[tid] / seen[tid] if seen[tid] else 0), order[tid]))
+
     # ---- sittings: a pack run over several sessions (stopped and resumed) keeps its total time
     SITTING_KEYS = ("load_s", "peak_rss_mb", "gpu_peak_gb", "vram_peak_gb", "kernel_share_max", "machine",
                     "loaded_alongside")
@@ -2143,7 +2167,10 @@ class Engine:
         # a hosted API has no /tokenize to split reasoning from answer tokens (it reports the split)
         tokenize_url = None if self.cfg["servers"][job.model["server"]].get("api_key_env") else base_url
         warned_reuse = False
+        stopped = self._stop_reason(job, mine)
         for rep in range(job.repeat):
+            if stopped:
+                break
             for test in self.round_order(job, rep):
                 if (test["id"], rep) in done_keys:
                     continue
@@ -2237,6 +2264,10 @@ class Engine:
                 job.passed += bool(g["pass"])
                 job.failed += not g["pass"]
                 self.emit("request_done", job=job, record=rec)
+                mine.append(rec)
+                stopped = self._stop_reason(job, mine)
+                if stopped:
+                    break
         job.status, job.finished = "done", time.time()
         merged = {(r["test"], r["repeat"]): r for r in kept}
         merged.update({(r["test"], r["repeat"]): r for r in new})
@@ -2246,9 +2277,17 @@ class Engine:
         os.remove(partial)
         with contextlib.suppress(OSError):
             os.remove(self._sittings_path(job))
-        job.note = f"{job.passed}/{job.done} passed"
+        job.note = f"{job.passed}/{job.done} passed" + (f"; stopped early, FAIL is certain: {stopped}" if stopped else "")
         self.emit("job_done", job=job)
 
+    def _stop_reason(self, job, rows):
+        """Why the job stops early now (verdict.fail_certain), or None."""
+        if not job.stop_early:
+            return None
+        from . import verdict
+        return verdict.fail_certain(job.pack, [r for r in rows if not client.server_error_row(r)],
+                                    self.cfg["defaults"]["repeat"])
+
     def _partial_to_history(self, job, keep=False):
         """Keep an unfinished run that is set aside in history/; keep: copy it (some of its answers stay)."""
         partial = job.out_path + ".partial.jsonl"

{tuieval-0.2.0.dev77 → tuieval-0.2.0.dev79}/src/tuieval/packs.py

@@ -49,6 +49,7 @@ class Pack:
     gate: dict             # release criteria (see verdict.py)
     screen: int            # tests in a screening run
     certify_repeat: int    # repeats in a certification run (0 = models.toml default)
+    stop_early: bool = False   # stop the pack once its FAIL is certain (pack.toml stop_early)
 
     @property
     def modules(self):
@@ -221,7 +222,8 @@ def load_pack(path):
                 system=system, needs=list(needs), tools=tools, tests=tests,
                 fingerprint=_fingerprint(system, tools, meta.get("grader", "answer"), tests, images),
                 gate=gate, screen=int(meta.get("screen", 10)),
-                certify_repeat=int(meta.get("certify", {}).get("repeat", 0)))
+                certify_repeat=int(meta.get("certify", {}).get("repeat", 0)),
+                stop_early=bool(meta.get("stop_early", False)))
 
 
 def load_packs(packs_dir=None, errors=None):

{tuieval-0.2.0.dev77 → tuieval-0.2.0.dev79}/src/tuieval/run_evals.py

@@ -745,6 +745,9 @@ def main(argv=None):
                                    "pack in --packs, else pack:id (and --packs defaults to those packs)")
     p.add_argument("--retest", "--force", action="store_true", dest="force",
                    help="ask again even if already answered: the packs, or just the --tests (earlier answers go to history/)")
+    p.add_argument("--stop-early", action="store_true",
+                   help="stop each pack once its FAIL is certain (a critical failure, or a bar out of reach), "
+                        "asking first what models fail most (default: each pack's stop_early)")
     p.add_argument("--repeat", type=int, help="times each question is asked, any tier (default: 1 for smoke and "
                                                   "screen, the pack's certification count for certify)")
     p.add_argument("--tier", choices=engine.TIERS, default="screen",
@@ -800,7 +803,7 @@ def main(argv=None):
     if args.tests:
         tests, pack_names = parse_tests(args.tests, pack_names if (args.packs or args.preset) else None, e.packs)
     try:
-        jobs = e.plan(labels, pack_names, args.repeat, args.tier, args.force, tests)
+        jobs = e.plan(labels, pack_names, args.repeat, args.tier, args.force, tests, args.stop_early or None)
     except packs_mod.PackError as err:
         sys.exit(str(err))
 

{tuieval-0.2.0.dev77 → tuieval-0.2.0.dev79}/src/tuieval/tui.py

@@ -390,6 +390,10 @@ picked with e, if any). Without it, answered questions are skipped. Before anyth
 the packs with earlier answers and lets you choose for each: Zero (ask everything again), Continue
 (keep what's answered, ask the rest) or Leave out. Replaced answers go to history/, never deleted.
 
+[b]Stop a pack once it fails[/b]: stop each pack as soon as its FAIL is certain (a critical failure, or
+an accuracy or truncation bar out of reach) and move on, asking first the questions models fail most.
+Packs with stop_early = true in pack.toml always do this. A later Certify fills in the rest.
+
 [b]Keys[/b]
   s  start (with a run going: add to it, or queue it)   r  results
   a  add a model (a GGUF, a model id, openrouter:<id>)  m  scan model folders for new GGUFs
@@ -652,6 +656,8 @@ class SetupScreen(Screen):
             yield Label("Models at a time")
             yield Input("", id="parallel", type="integer", max_length=1, placeholder="1")
             yield Checkbox("Retest (ask again, even if already answered)", id="force")
+        with Horizontal(id="options2"):
+            yield Checkbox("Stop a pack once it fails (a critical failure, or a bar out of reach)", id="stop-early")
         yield Static(id="estimate")
         with Horizontal(id="buttons"):
             yield Button("Start  [s]", id="start", variant="success")
@@ -1009,6 +1015,8 @@ class SetupScreen(Screen):
             self.query_one(f"#tier-{tier}", RadioButton).value = True
         if "force" in sel:
             self.query_one("#force", Checkbox).value = bool(sel["force"])
+        if "stop_early" in sel:
+            self.query_one("#stop-early", Checkbox).value = bool(sel["stop_early"])
         self.query_one("#filter", Input).value = ""
         self.refresh_models()
         if not quiet:
@@ -1043,7 +1051,7 @@ class SetupScreen(Screen):
             return
         picks = self.current_picks(suites)
         sel = {"models": labels, "packs": suites, "repeat": repeat, "tier": tier, "force": force, "tests": picks,
-               "parallel": self.parallel()}
+               "parallel": self.parallel(), "stop_early": self.query_one("#stop-early", Checkbox).value}
         e = self.app.engine
         jobs = e.plan(labels, suites, repeat, tier, True, picks)
         if not any(j.status == "waiting" for j in jobs):
@@ -1573,7 +1581,7 @@ class QueuedRun:
         """(jobs, None), or (None, why it can't run) now."""
         sel = self.sel
         jobs = [j for j in eng.plan(sel["models"], sel["packs"], sel["repeat"], sel["tier"], self.retest,
-                                    self.tests(eng)) if j.key not in self.leave]
+                                    self.tests(eng), sel.get("stop_early") or None) if j.key not in self.leave]
         waiting = [j for j in jobs if j.status == "waiting"]
         if not waiting:
             return None, "every pack left is already done"
@@ -3076,7 +3084,7 @@ class EvalsApp(App):
     #tier-row Label { padding: 1 1 0 0; }
     #tier-row RadioSet { layout: horizontal; width: auto; height: auto; }
     #tier-row RadioButton { width: auto; margin-right: 2; }
-    #options { height: 3; padding: 0 1; align-vertical: middle; }
+    #options, #options2 { height: 3; padding: 0 1; align-vertical: middle; }
     #options Label { padding: 1 1 0 0; }
     #options Input { width: 12; }
     #options #parallel { width: 8; }

{tuieval-0.2.0.dev77 → tuieval-0.2.0.dev79}/src/tuieval/verdict.py

@@ -73,6 +73,26 @@ def consistency(records):
     return ok / len(full), len(full)
 
 
+def fail_certain(pack, records, repeat_default=3):
+    """Why this pack's verdict is FAIL whatever the remaining answers of its certification say,
+    or None: a critical failure, or a bar out of reach even if every answer still to come passes
+    (accuracy) or none of them is cut off (truncation). What an early stop checks after each answer."""
+    crit = next((r for r in records if r.get("severity") == "critical"), None)
+    if crit:
+        return f"critical failure ({crit['test']})"
+    n = len(records)
+    total = max(n, len(pack.tests) * (pack.certify_repeat or repeat_default))
+    passed = sum(bool(r["pass"]) for r in records)
+    bar = pack.gate.get("min_accuracy", 0.8)
+    if n and wilson(passed + total - n, total)[1] < bar:
+        return (f"{n - passed} wrong of {n}: under the {100 * bar:.0f}% bar even if all "
+                f"{total - n} answers left pass")
+    cut = sum(r.get("finish") == "length" for r in records)
+    if "max_truncation" in pack.gate and cut / total > pack.gate["max_truncation"]:
+        return f"{cut} answers cut off by max_tokens: over the {100 * pack.gate['max_truncation']:g}% limit for the pack"
+    return None
+
+
 def evaluate(pack, records, repeat_default=3, speed_gate=False):
     """Quality verdict for one model on one pack. `records` are result rows from
     results/<label>/<pack>.json. The p90 time limit is judged per machine by latency() unless
@@ -112,6 +132,9 @@ def evaluate(pack, records, repeat_default=3, speed_gate=False):
     if hi < min_acc:
         fails.append(f"accuracy {100 * passed / n:.1f}% (95% CI {100 * lo:.0f}-{100 * hi:.0f}%) is below "
                      f"the {100 * min_acc:.0f}% bar")
+    certain = fail_certain(pack, records, repeat_default)
+    if certain and not crit and not fails:   # a run stopped early (or a partial one) that can't pass any more
+        fails.append(certain)
     if "min_consistency" in gate and cons is not None and certified and cons < gate["min_consistency"]:
         fails.append(f"only {100 * cons:.0f}% of {groups} rephrased cases answered consistently "
                      f"(needs {100 * gate['min_consistency']:.0f}%)")

{tuieval-0.2.0.dev77 → tuieval-0.2.0.dev79}/tests/test_tuieval.py

@@ -151,6 +151,52 @@ class Starters(unittest.TestCase):
         self.assertIn("separate the models", tuieval(self.ws, "compare", "--speed").stdout)
 
 
+class EarlyStop(unittest.TestCase):
+    """--stop-early ends a pack once its FAIL is certain, and asks what models fail most first."""
+
+    def test_wrong_model_stops_and_good_model_finishes(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            ws = os.path.join(tmp, "ws")
+            tuieval(tmp, "init", ws)
+            tuieval(ws, "new-pack", "demo", "--grader", "answer")
+            packs = os.path.join(ws, "packs")
+            good, bad = Mock(packs, "oracle"), Mock(packs, "wrong")
+            try:
+                with open(os.path.join(ws, "models.toml"), "a") as f:
+                    f.write(f'\n[servers.good]\nurl = "http://127.0.0.1:{good.port}"\n'
+                            f'\n[servers.bad]\nurl = "http://127.0.0.1:{bad.port}"\n')
+                tuieval(ws, "add", "mock", "--server", "good", "--label", "good-mock")
+                tuieval(ws, "add", "mock", "--server", "bad", "--label", "bad-mock")
+                out = tuieval(ws, "run", "--tier", "certify", "--stop-early").stdout
+            finally:
+                good.stop()
+                bad.stop()
+            self.assertIn("bad-mock/demo: done", out)
+            self.assertIn("stopped early, FAIL is certain", out)
+            bad_res = json.loads(read(os.path.join(ws, "results", "bad-mock", "demo.json")))["results"]
+            good_res = json.loads(read(os.path.join(ws, "results", "good-mock", "demo.json")))["results"]
+            self.assertEqual(len(good_res), 18)
+            self.assertLess(len(bad_res), 18)
+            self.assertIn("bad-mock: not ready for", tuieval(ws, "verdict").stdout.split("Details")[0])
+            # bad-mock's failures now come first for the next model with an early stop
+            code = ("from tuieval import engine; e = engine.Engine(); p = e.packs['demo']; "
+                    "first = e.most_failed_first(p); fails = {r['test'] for r in e.records('bad-mock', 'demo') "
+                    "if not r['pass']}; print(set(first[:len(fails)]) == fails)")
+            self.assertEqual(subprocess.run([sys.executable, "-c", code], capture_output=True, text=True,
+                                            env=dict(os.environ, TUIEVAL_HOME=ws)).stdout.strip(), "True")
+
+    def test_fail_certain(self):
+        from tuieval import verdict
+        pack = types.SimpleNamespace(tests=[{"id": str(i)} for i in range(10)], certify_repeat=3,
+                                     gate={"min_accuracy": 0.8, "max_truncation": 0.05})
+        row = lambda ok, **kw: {"test": "0", "pass": ok, **kw}
+        self.assertIsNone(verdict.fail_certain(pack, [row(False)] * 2))           # 28 of 30 could still pass
+        self.assertIsNone(verdict.fail_certain(pack, [row(False)] * 9))           # best case 21/30: CI reaches 83%
+        self.assertIn("under the 80% bar", verdict.fail_certain(pack, [row(False)] * 12))
+        self.assertIn("critical failure", verdict.fail_certain(pack, [row(False, severity="critical")]))
+        self.assertIn("cut off", verdict.fail_certain(pack, [row(True, finish="length")] * 2))  # 2 of 30 > 5%
+
+
 class WorkspaceGraders(unittest.TestCase):
     """A workspace's graders/ and a pack's grader.py are loaded, with their critical flag."""