tuieval 0.2.0.dev77__py3-none-any.whl → 0.2.0.dev81__py3-none-any.whl

tuieval/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.2.0.dev77'
-__version_tuple__ = version_tuple = (0, 2, 0, 'dev77')
+__version__ = version = '0.2.0.dev81'
+__version_tuple__ = version_tuple = (0, 2, 0, 'dev81')
 
 __commit_id__ = commit_id = None

tuieval/engine.py

@@ -28,6 +28,7 @@ interrupted run resumes from its .partial.jsonl, request by request.
 Events are delivered as on_event(kind, **data), from the engine thread.
 """
 import base64
+import collections
 import contextlib
 import dataclasses
 import fcntl
@@ -46,6 +47,7 @@ import subprocess
 import threading
 import time
 import tomllib
+import urllib.parse
 import urllib.request
 
 from . import client
@@ -438,6 +440,8 @@ class Job:
     finished: float = 0.0
     resumed: int = 0        # answers already done when this sitting started
     earlier: list = dataclasses.field(default_factory=list)  # earlier sittings (see Engine._sittings)
+    stop_early: bool = False  # stop once the pack's FAIL is certain (verdict.fail_certain)
+    first: list = dataclasses.field(default_factory=list)   # test ids round 1 asks first (most failed)
 
     @property
     def label(self):
@@ -700,6 +704,7 @@ class Engine:
         self._procs = []
         self._stop_lock = threading.Lock()
         self._run_lock = threading.Lock()
+        self._machine_take = threading.Lock()   # one lane takes the machine lock for a hosted-only run
         self._run = None          # the run going: {"jobs", "pending", "open", "machine", "cond"} (add_jobs)
         self._header_cache = {}
         self._serving = {}
@@ -1152,14 +1157,15 @@ class Engine:
         return not current and header_matches(head, pack.fingerprint, settings)
 
     # ---- planning
-    def plan(self, labels, pack_names, repeat=None, tier="screen", force=False, tests=None):
+    def plan(self, labels, pack_names, repeat=None, tier="screen", force=False, tests=None, stop_early=None):
         """Jobs for models x packs. tier: smoke (3 tests), screen (a spread sample), certify (all
         tests). repeat: times each question is asked; None = the tier default (1 for smoke and
         screen, the pack's certification repeats for certify). tests: {pack: [test ids]} runs just
         those tests of a pack instead of the tier's sample (unknown ids raise PackError). Earlier
         matching results are kept and only missing answers are run, unless force; force with
         picked tests reruns those and keeps the pack's other answers. force is True, False, or the
-        keys ("label/pack") of the jobs to retest, the others resuming."""
+        keys ("label/pack") of the jobs to retest, the others resuming. stop_early: stop each pack
+        once its FAIL is certain, asking first what models fail most; None = each pack's stop_early."""
         if tier not in TIERS:
             raise ValueError(f"tier must be one of {TIERS}")
         jobs = []
@@ -1173,7 +1179,10 @@ class Engine:
                 job = Job(m, pack, reps, pack.pick(picked) if picked else pack.select(tier),
                           self.result_path(label, name, tier), effective_sampling(self.cfg, m), tier=tier,
                           merge=not retest or bool(picked), redo=retest and bool(picked), fresh=retest,
-                          settings=self.settings(m))
+                          settings=self.settings(m),
+                          stop_early=pack.stop_early if stop_early is None else bool(stop_early))
+                if job.stop_early:
+                    job.first = self.most_failed_first(pack)
                 sv = self.serving(m)
                 status = self.result_status(label, name, tier)
                 job.discards = self._discards(job)
@@ -1251,6 +1260,36 @@ class Engine:
         except (TypeError, ValueError):
             return 1
 
+    def parallel_hosted(self):
+        """How many hosted models (APIs) a run serves at a time, besides the local ones: models.toml
+        [machines.<id>] parallel_hosted (default 4). They don't use this machine's GPU or memory."""
+        try:
+            return max(1, int(self.machine_settings().get("parallel_hosted", 4)))
+        except (TypeError, ValueError):
+            return 4
+
+    def models_at_once(self, labels, parallel=None, parallel_hosted=None):
+        """How many of these models run at the same time: local ones up to parallel (parallel_models),
+        hosted ones beside them up to parallel_hosted."""
+        labels = set(labels)
+        hosted = sum(self.hosted(self.model(l)) for l in labels)
+        return (min(parallel or self.parallel_models(), len(labels) - hosted)
+                + min(parallel_hosted or self.parallel_hosted(), hosted))
+
+    def hosted(self, m):
+        """A model served by an API elsewhere: tuieval doesn't start its server and its url isn't this
+        machine (or its server sets hosted = true). A server you started yourself on localhost still
+        counts as local (it uses the GPU)."""
+        if m.get("remote"):
+            return True
+        server = self.cfg["servers"][m["server"]]
+        if "hosted" in server:   # say so for a server the url doesn't give away (e.g. a local proxy to an API)
+            return bool(server["hosted"])
+        if server.get("cmd"):
+            return False
+        host = urllib.parse.urlparse(server.get("url") or "").hostname or ""
+        return bool(host) and host not in ("localhost", "127.0.0.1", "::1", "0.0.0.0")
+
     def memory_need_gb(self, m):
         """GPU memory a model takes while served here: 0 for one we don't start (a hosted API or a
         server already running), the fit check's estimate for a local GGUF, else None (unknown)."""
@@ -1609,17 +1648,17 @@ class Engine:
         return any(self.cfg["servers"][j.model["server"]].get("cmd") for j in jobs if j.status == "waiting")
 
     # ---- running
-    def run(self, jobs, parallel=None):
+    def run(self, jobs, parallel=None, parallel_hosted=None):
         """Run the jobs (blocking). Jobs of the same model share one server start. parallel: how many
-        models are served at a time (default: parallel_models for this machine). add_jobs() can add
-        more while it runs; they run after the ones before them."""
+        local models are served at a time (default: parallel_models for this machine); hosted models
+        (APIs) run beside them, up to parallel_hosted at a time (default: parallel_hosted). add_jobs()
+        can add more while it runs; they run after the ones before them."""
         self._cancel.clear()
         self.emit("queue_started", jobs=jobs)
         models = sorted({j.label for j in jobs if j.status == "waiting"})
         machine = bool(jobs) and self.needs_machine(jobs)
-        lock = (self.machine_lock(f"{jobs[0].tier} run: {', '.join(models[:3])}"
-                                  + (f" +{len(models) - 3}" if len(models) > 3 else ""))
-                if machine else contextlib.nullcontext())
+        what = f"{jobs[0].tier} run: {', '.join(models[:3])}" + (f" +{len(models) - 3}" if len(models) > 3 else "")
+        stack = contextlib.ExitStack()   # holds the machine lock; a hosted-only run takes it if local models are added
         order = []
         for j in jobs:
             if j.label not in order:
@@ -1627,10 +1666,13 @@ class Engine:
         groups = [(label, [j for j in jobs if j.label == label and j.status == "waiting"]) for label in order]
         with self._run_lock:
             self._run = {"jobs": jobs, "pending": [g for g in groups if g[1]], "open": True, "machine": machine,
-                         "cond": threading.Condition(self._run_lock)}
+                         "cond": threading.Condition(self._run_lock), "stack": stack, "what": what}
         try:
-            with lock:
-                self._run_lanes(max(1, int(parallel or self.parallel_models())))
+            with stack:
+                if machine:
+                    stack.enter_context(self.machine_lock(what))
+                self._run_lanes(max(1, int(parallel or self.parallel_models())),
+                                max(1, int(parallel_hosted or self.parallel_hosted())))
             if self._cancel.is_set():
                 raise Cancelled()
         except Cancelled:
@@ -1657,14 +1699,13 @@ class Engine:
     def add_jobs(self, new):
         """Add planned jobs to the run going on, after its other jobs. Returns (added, why not): a
         pack already waiting or running in it isn't added twice, and nothing is added once the run is
-        finishing or cancelled, or when it holds no machine lock (hosted only) and these need one.
+        finishing or cancelled. Local models added to a hosted-only run take the machine lock when the
+        first of them starts.
         Runs on the caller's thread, so the caller shows what was added (no event)."""
         with self._run_lock:
             r = self._run
             if not r or not r["open"] or self._cancel.is_set():
                 return [], "the run is finishing"
-            if not r["machine"] and self.needs_machine(new):
-                return [], "the run going uses only hosted models, and these need this machine"
             busy = {j.key for j in r["jobs"] if j.status in ("waiting", "loading", "running")}
             added = [j for j in new if j.status == "waiting" and j.key not in busy]
             r["jobs"].extend(added)
@@ -1677,33 +1718,51 @@ class Engine:
             r["cond"].notify_all()
         return added, None if added else "everything picked is already in the run"
 
-    def _run_lanes(self, parallel):
-        """Run each model's jobs in its own lane, up to `parallel` at a time, in order: the next
-        model starts once a lane is free, nothing running stops it (runs_alone), and its memory fits
-        next to the ones running (memory_need_gb; an unknown need is left to the setting). Lanes
-        wait for jobs added meanwhile (add_jobs) until every model is done."""
+    def _run_lanes(self, parallel, parallel_hosted=4):
+        """Run each model's jobs in its own lane: up to `parallel` local models at a time, in order (the
+        next one starts once a lane is free, nothing running stops it (runs_alone), and its memory fits
+        next to the ones running (memory_need_gb; an unknown need is left to the setting)), and beside
+        them up to `parallel_hosted` hosted models, which need none of this machine. Lanes wait for jobs
+        added meanwhile (add_jobs) until every model is done."""
         pending, running, told = self._run["pending"], {}, set()
         cond = self._run["cond"]
+        hosted = {}
+
+        def is_hosted(label):
+            if label not in hosted:
+                hosted[label] = self.hosted(self.model(label))
+            return hosted[label]
 
         def blocked(label):
             """Why the next model can't start now (None: it can)."""
-            if not running:
+            if is_hosted(label):
+                return "" if sum(map(is_hosted, running)) >= parallel_hosted else None
+            local = [l for l in running if not is_hosted(l)]
+            if not local:
                 return None
-            if len(running) >= parallel:
+            if len(local) >= parallel:
                 return ""
             m = self.model(label)
-            if self.runs_alone(m) or any(self.runs_alone(self.model(l)) for l in running):
-                alone = label if self.runs_alone(m) else next(l for l in running if self.runs_alone(self.model(l)))
+            if self.runs_alone(m) or any(self.runs_alone(self.model(l)) for l in local):
+                alone = label if self.runs_alone(m) else next(l for l in local if self.runs_alone(self.model(l)))
                 return (f"{alone}'s server runs a before_start step that may stop other servers, so it runs "
                         "on its own")
             need = self.memory_need_gb(m)
-            used = sum(n for n in running.values() if n)
+            used = sum(running[l] or 0 for l in local)
             room = self.memory_available_gb()
             if need and used + need > room:
-                return (f"needs ~{need:.0f} GB; {', '.join(running)} take ~{used:.0f} of {room:.0f} GB, so it "
+                return (f"needs ~{need:.0f} GB; {', '.join(local)} take ~{used:.0f} of {room:.0f} GB, so it "
                         "starts when there's room")
             return None
 
+        def take_machine():
+            """A hosted-only run that was given local models takes the machine lock before the first
+            of them starts (it may wait for another tuieval window, like any run)."""
+            with self._machine_take:
+                if not self._run["machine"]:
+                    self._run["stack"].enter_context(self.machine_lock(self._run["what"]))
+                    self._run["machine"] = True
+
         def lane():
             while True:
                 mine, note = None, None
@@ -1713,33 +1772,41 @@ class Engine:
                         cond.notify_all()
                         return
                     # in order, except that a model added again while it runs waits for itself to end
-                    # without holding up the ones behind it
-                    nxt = next((i for i, g in enumerate(pending) if g[0] not in running), None)
-                    label = pending[nxt][0] if nxt is not None else None
-                    why = blocked(label) if label else ""   # none can start: a model still runs, more may come
-                    if why is None:
-                        label, mine = pending.pop(nxt)
-                        running[label] = self.memory_need_gb(self.model(label))
-                    elif why and label not in told:
-                        told.add(label)
-                        note = f"{label} waits: {why}"
-                    else:
-                        cond.wait(1.0)
+                    # without holding up the ones behind it; the first waiting hosted model doesn't wait
+                    # for local ones (and the first local one doesn't wait for hosted ones)
+                    waiting = [i for i, g in enumerate(pending) if g[0] not in running]
+                    first = {}
+                    for i in waiting:
+                        first.setdefault(is_hosted(pending[i][0]), i)
+                    why, label = "", None
+                    for i in sorted(first.values()):
+                        why = blocked(pending[i][0])
+                        if why is None:
+                            label, mine = pending.pop(i)
+                            running[label] = self.memory_need_gb(self.model(label))
+                            break
+                    if mine is None:
+                        label = pending[min(first.values())][0] if first else None
+                        if why and label and label not in told:
+                            told.add(label)
+                            note = f"{label} waits: {why}"
+                        else:
+                            cond.wait(1.0)
                 if note:
                     self.emit("model_waiting", label=label, message=note)
                 if mine is None:
                     continue
                 try:
+                    if not is_hosted(label) and self.cfg["servers"][mine[0].model["server"]].get("cmd"):
+                        take_machine()
                     self._run_lane(label, mine)
                 finally:
                     with cond:
                         running.pop(label, None)
                         cond.notify_all()
 
-        if parallel == 1:
-            lane()
-            return
-        threads = [threading.Thread(target=lane, name=f"lane-{i + 1}") for i in range(parallel)]
+        # lanes for hosted models too, even if none is queued yet: add_jobs may bring some
+        threads = [threading.Thread(target=lane, name=f"lane-{i + 1}") for i in range(parallel + parallel_hosted)]
         for t in threads:
             t.start()
         for t in threads:
@@ -2005,13 +2072,30 @@ class Engine:
 
     @staticmethod
     def round_order(job, rep):
-        """The tests in the order round `rep` asks them: the pack's order first, then a fixed
-        shuffle per round (the same on every run, so resuming and reruns ask in the same order)."""
+        """The tests in the order round `rep` asks them: the pack's order first (with an early stop,
+        the most failed first), then a fixed shuffle per round (the same on every run, so resuming
+        and reruns ask in the same order)."""
         tests = list(job.tests)
         if rep:
             random.Random(f"{job.pack.name}:{rep}").shuffle(tests)
+        elif job.first:
+            rank = {tid: i for i, tid in enumerate(job.first)}
+            tests.sort(key=lambda t: rank.get(t["id"], len(rank)))
         return tests
 
+    def most_failed_first(self, pack):
+        """The pack's test ids, the ones models failed most often first (critical failures count
+        double), from every model's stored results; ties keep the pack's order."""
+        fails, seen = collections.Counter(), collections.Counter()
+        for m in self.cfg["models"]:
+            for r in self.records(m["label"], pack.name):
+                if client.server_error_row(r):
+                    continue
+                seen[r["test"]] += 1
+                fails[r["test"]] += (not r["pass"]) + (r.get("severity") == "critical")
+        order = {t["id"]: i for i, t in enumerate(pack.tests)}
+        return sorted(order, key=lambda tid: (-(fails[tid] / seen[tid] if seen[tid] else 0), order[tid]))
+
     # ---- sittings: a pack run over several sessions (stopped and resumed) keeps its total time
     SITTING_KEYS = ("load_s", "peak_rss_mb", "gpu_peak_gb", "vram_peak_gb", "kernel_share_max", "machine",
                     "loaded_alongside")
@@ -2143,7 +2227,10 @@ class Engine:
         # a hosted API has no /tokenize to split reasoning from answer tokens (it reports the split)
         tokenize_url = None if self.cfg["servers"][job.model["server"]].get("api_key_env") else base_url
         warned_reuse = False
+        stopped = self._stop_reason(job, mine)
         for rep in range(job.repeat):
+            if stopped:
+                break
             for test in self.round_order(job, rep):
                 if (test["id"], rep) in done_keys:
                     continue
@@ -2237,6 +2324,10 @@ class Engine:
                 job.passed += bool(g["pass"])
                 job.failed += not g["pass"]
                 self.emit("request_done", job=job, record=rec)
+                mine.append(rec)
+                stopped = self._stop_reason(job, mine)
+                if stopped:
+                    break
         job.status, job.finished = "done", time.time()
         merged = {(r["test"], r["repeat"]): r for r in kept}
         merged.update({(r["test"], r["repeat"]): r for r in new})
@@ -2246,9 +2337,17 @@ class Engine:
         os.remove(partial)
         with contextlib.suppress(OSError):
             os.remove(self._sittings_path(job))
-        job.note = f"{job.passed}/{job.done} passed"
+        job.note = f"{job.passed}/{job.done} passed" + (f"; stopped early, FAIL is certain: {stopped}" if stopped else "")
         self.emit("job_done", job=job)
 
+    def _stop_reason(self, job, rows):
+        """Why the job stops early now (verdict.fail_certain), or None."""
+        if not job.stop_early:
+            return None
+        from . import verdict
+        return verdict.fail_certain(job.pack, [r for r in rows if not client.server_error_row(r)],
+                                    self.cfg["defaults"]["repeat"])
+
     def _partial_to_history(self, job, keep=False):
         """Keep an unfinished run that is set aside in history/; keep: copy it (some of its answers stay)."""
         partial = job.out_path + ".partial.jsonl"

tuieval/packs.py

@@ -49,6 +49,7 @@ class Pack:
     gate: dict             # release criteria (see verdict.py)
     screen: int            # tests in a screening run
     certify_repeat: int    # repeats in a certification run (0 = models.toml default)
+    stop_early: bool = False   # stop the pack once its FAIL is certain (pack.toml stop_early)
 
     @property
     def modules(self):
@@ -221,7 +222,8 @@ def load_pack(path):
                 system=system, needs=list(needs), tools=tools, tests=tests,
                 fingerprint=_fingerprint(system, tools, meta.get("grader", "answer"), tests, images),
                 gate=gate, screen=int(meta.get("screen", 10)),
-                certify_repeat=int(meta.get("certify", {}).get("repeat", 0)))
+                certify_repeat=int(meta.get("certify", {}).get("repeat", 0)),
+                stop_early=bool(meta.get("stop_early", False)))
 
 
 def load_packs(packs_dir=None, errors=None):

tuieval/run_evals.py

@@ -17,6 +17,7 @@
     tuieval run --tier smoke --only openrouter:qwen/qwen3-32b   # any OpenRouter model, no models.toml edit
     tuieval run --dry-run                         # print the plan and server commands only
     tuieval run --parallel 2                      # serve two models at a time (parallel_models in models.toml)
+    tuieval run --parallel-hosted 6               # hosted models at a time beside them (default 4)
     tuieval add ~/models/New-Model-Q4_K_M.gguf    # register a model (--vision/--mmproj, --no-think, --tags)
     tuieval scan                                  # GGUFs in model_dirs that aren't registered yet
     tuieval list                                  # the models tuieval knows; hidden ones listed separately
@@ -745,6 +746,9 @@ def main(argv=None):
                                    "pack in --packs, else pack:id (and --packs defaults to those packs)")
     p.add_argument("--retest", "--force", action="store_true", dest="force",
                    help="ask again even if already answered: the packs, or just the --tests (earlier answers go to history/)")
+    p.add_argument("--stop-early", action="store_true",
+                   help="stop each pack once its FAIL is certain (a critical failure, or a bar out of reach), "
+                        "asking first what models fail most (default: each pack's stop_early)")
     p.add_argument("--repeat", type=int, help="times each question is asked, any tier (default: 1 for smoke and "
                                                   "screen, the pack's certification count for certify)")
     p.add_argument("--tier", choices=engine.TIERS, default="screen",
@@ -752,6 +756,9 @@ def main(argv=None):
     p.add_argument("--smoke", action="store_const", const="smoke", dest="tier", help="same as --tier smoke")
     p.add_argument("--parallel", type=int, metavar="N",
                    help="models served at a time (default: parallel_models for this machine in models.toml, else 1)")
+    p.add_argument("--parallel-hosted", type=int, metavar="N",
+                   help="hosted models (APIs) served at a time beside the local ones (default: parallel_hosted "
+                        "in models.toml, else 4)")
     p.add_argument("--dry-run", action="store_true", help="print the plan and commands without running")
     p.add_argument("--brief", action="store_true", help="end with one line per model instead of the scorecards")
     p.add_argument("--models", default=None, help="default: the workspace's models.toml")
@@ -766,6 +773,9 @@ def main(argv=None):
     if args.parallel is not None and args.parallel < 1:
         sys.exit("--parallel must be 1 or more")
     parallel = args.parallel or e.parallel_models()
+    if args.parallel_hosted is not None and args.parallel_hosted < 1:
+        sys.exit("--parallel-hosted must be 1 or more")
+    parallel_hosted = args.parallel_hosted or e.parallel_hosted()
     models = e.cfg["models"]
     labels = [m["label"] for m in models if not m.get("remote")]   # hosted APIs cost money: only by name
     if args.only:
@@ -800,7 +810,7 @@ def main(argv=None):
     if args.tests:
         tests, pack_names = parse_tests(args.tests, pack_names if (args.packs or args.preset) else None, e.packs)
     try:
-        jobs = e.plan(labels, pack_names, args.repeat, args.tier, args.force, tests)
+        jobs = e.plan(labels, pack_names, args.repeat, args.tier, args.force, tests, args.stop_early or None)
     except packs_mod.PackError as err:
         sys.exit(str(err))
 
@@ -834,12 +844,12 @@ def main(argv=None):
 
     signal.signal(signal.SIGINT, stop)
     signal.signal(signal.SIGTERM, stop)
-    side_by_side = min(parallel, len({j.label for j in jobs if j.status == "waiting"}))
+    side_by_side = e.models_at_once({j.label for j in jobs if j.status == "waiting"}, parallel, parallel_hosted)
     if side_by_side > 1:
         printer.side_by_side = True
         say(f"up to {side_by_side} models at a time: one line per answer; each answer notes the models "
             "it ran alongside, since they share this machine's speed")
-    worker = threading.Thread(target=e.run, args=(jobs, parallel))
+    worker = threading.Thread(target=e.run, args=(jobs, parallel, parallel_hosted))
     worker.start()
     while worker.is_alive():
         worker.join(0.5)

tuieval/tui.py

@@ -390,6 +390,10 @@ picked with e, if any). Without it, answered questions are skipped. Before anyth
 the packs with earlier answers and lets you choose for each: Zero (ask everything again), Continue
 (keep what's answered, ask the rest) or Leave out. Replaced answers go to history/, never deleted.
 
+[b]Stop a pack once it fails[/b]: stop each pack as soon as its FAIL is certain (a critical failure, or
+an accuracy or truncation bar out of reach) and move on, asking first the questions models fail most.
+Packs with stop_early = true in pack.toml always do this. A later Certify fills in the rest.
+
 [b]Keys[/b]
   s  start (with a run going: add to it, or queue it)   r  results
   a  add a model (a GGUF, a model id, openrouter:<id>)  m  scan model folders for new GGUFs
@@ -652,6 +656,8 @@ class SetupScreen(Screen):
             yield Label("Models at a time")
             yield Input("", id="parallel", type="integer", max_length=1, placeholder="1")
             yield Checkbox("Retest (ask again, even if already answered)", id="force")
+        with Horizontal(id="options2"):
+            yield Checkbox("Stop a pack once it fails (a critical failure, or a bar out of reach)", id="stop-early")
         yield Static(id="estimate")
         with Horizontal(id="buttons"):
             yield Button("Start  [s]", id="start", variant="success")
@@ -969,7 +975,7 @@ class SetupScreen(Screen):
         resumed = [j for j in run if "done earlier" in j.note]
         if resumed:
             text += f"   [yellow]{len(resumed)} pack(s) continue from earlier results[/yellow]"
-        at_once = min(self.parallel() or e.parallel_models(), len({j.label for j in run}))
+        at_once = e.models_at_once({j.label for j in run}, self.parallel())
         if at_once > 1:
             text += (f"\n[cyan]Up to {at_once} models at a time[/cyan][dim] (the time above assumes one at a "
                      "time). Answers note the models they ran alongside, since those share the machine's "
@@ -1009,6 +1015,8 @@ class SetupScreen(Screen):
             self.query_one(f"#tier-{tier}", RadioButton).value = True
         if "force" in sel:
             self.query_one("#force", Checkbox).value = bool(sel["force"])
+        if "stop_early" in sel:
+            self.query_one("#stop-early", Checkbox).value = bool(sel["stop_early"])
         self.query_one("#filter", Input).value = ""
         self.refresh_models()
         if not quiet:
@@ -1043,7 +1051,7 @@ class SetupScreen(Screen):
             return
         picks = self.current_picks(suites)
         sel = {"models": labels, "packs": suites, "repeat": repeat, "tier": tier, "force": force, "tests": picks,
-               "parallel": self.parallel()}
+               "parallel": self.parallel(), "stop_early": self.query_one("#stop-early", Checkbox).value}
         e = self.app.engine
         jobs = e.plan(labels, suites, repeat, tier, True, picks)
         if not any(j.status == "waiting" for j in jobs):
@@ -1573,7 +1581,7 @@ class QueuedRun:
         """(jobs, None), or (None, why it can't run) now."""
         sel = self.sel
         jobs = [j for j in eng.plan(sel["models"], sel["packs"], sel["repeat"], sel["tier"], self.retest,
-                                    self.tests(eng)) if j.key not in self.leave]
+                                    self.tests(eng), sel.get("stop_early") or None) if j.key not in self.leave]
         waiting = [j for j in jobs if j.status == "waiting"]
         if not waiting:
             return None, "every pack left is already done"
@@ -2122,8 +2130,7 @@ class RunScreen(Screen):
     @property
     def side_by_side(self):
         """Several models run at a time in this run."""
-        return min(self.parallel or self.app.engine.parallel_models(),
-                   len({j.label for j in self.jobs if j.status != "skipped"})) > 1
+        return self.app.engine.models_at_once({j.label for j in self.jobs if j.status != "skipped"}, self.parallel) > 1
 
     def active(self):
         """Models being loaded or run now, in queue order."""
@@ -2438,7 +2445,7 @@ class RunScreen(Screen):
                 rough |= j.key in self.rough
             left += (j.total - j.done) * spr
             models.add(j.label)
-        lanes = max(1, min(self.parallel or self.app.engine.parallel_models(), len(models)))
+        lanes = max(1, self.app.engine.models_at_once(models, self.parallel))
         return left / lanes, rough
 
     def update_overall(self):
@@ -3076,7 +3083,7 @@ class EvalsApp(App):
     #tier-row Label { padding: 1 1 0 0; }
     #tier-row RadioSet { layout: horizontal; width: auto; height: auto; }
     #tier-row RadioButton { width: auto; margin-right: 2; }
-    #options { height: 3; padding: 0 1; align-vertical: middle; }
+    #options, #options2 { height: 3; padding: 0 1; align-vertical: middle; }
     #options Label { padding: 1 1 0 0; }
     #options Input { width: 12; }
     #options #parallel { width: 8; }

tuieval/verdict.py

@@ -73,6 +73,26 @@ def consistency(records):
     return ok / len(full), len(full)
 
 
+def fail_certain(pack, records, repeat_default=3):
+    """Why this pack's verdict is FAIL whatever the remaining answers of its certification say,
+    or None: a critical failure, or a bar out of reach even if every answer still to come passes
+    (accuracy) or none of them is cut off (truncation). What an early stop checks after each answer."""
+    crit = next((r for r in records if r.get("severity") == "critical"), None)
+    if crit:
+        return f"critical failure ({crit['test']})"
+    n = len(records)
+    total = max(n, len(pack.tests) * (pack.certify_repeat or repeat_default))
+    passed = sum(bool(r["pass"]) for r in records)
+    bar = pack.gate.get("min_accuracy", 0.8)
+    if n and wilson(passed + total - n, total)[1] < bar:
+        return (f"{n - passed} wrong of {n}: under the {100 * bar:.0f}% bar even if all "
+                f"{total - n} answers left pass")
+    cut = sum(r.get("finish") == "length" for r in records)
+    if "max_truncation" in pack.gate and cut / total > pack.gate["max_truncation"]:
+        return f"{cut} answers cut off by max_tokens: over the {100 * pack.gate['max_truncation']:g}% limit for the pack"
+    return None
+
+
 def evaluate(pack, records, repeat_default=3, speed_gate=False):
     """Quality verdict for one model on one pack. `records` are result rows from
     results/<label>/<pack>.json. The p90 time limit is judged per machine by latency() unless
@@ -112,6 +132,9 @@ def evaluate(pack, records, repeat_default=3, speed_gate=False):
     if hi < min_acc:
         fails.append(f"accuracy {100 * passed / n:.1f}% (95% CI {100 * lo:.0f}-{100 * hi:.0f}%) is below "
                      f"the {100 * min_acc:.0f}% bar")
+    certain = fail_certain(pack, records, repeat_default)
+    if certain and not crit and not fails:   # a run stopped early (or a partial one) that can't pass any more
+        fails.append(certain)
     if "min_consistency" in gate and cons is not None and certified and cons < gate["min_consistency"]:
         fails.append(f"only {100 * cons:.0f}% of {groups} rephrased cases answered consistently "
                      f"(needs {100 * gate['min_consistency']:.0f}%)")

{tuieval-0.2.0.dev77.dist-info → tuieval-0.2.0.dev81.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: tuieval
-Version: 0.2.0.dev77
+Version: 0.2.0.dev81
 Summary: Evaluate local and frontier LLMs on your own questions: accuracy, speed, tokens and PASS/FAIL verdicts, in the terminal.
 Project-URL: Homepage, https://github.com/ashe-wb/tuieval
 Project-URL: Issues, https://github.com/ashe-wb/tuieval/issues

{tuieval-0.2.0.dev77.dist-info → tuieval-0.2.0.dev81.dist-info}/RECORD

@@ -1,26 +1,26 @@
 tuieval/__init__.py,sha256=myVXVX7htbwezW6k3x_FuKgn1RSZ2WcTaDEMByGn6Wg,312
 tuieval/__main__.py,sha256=E6Gls0DNz8GQK2K-kOUIx8cYhgANW_CH54VKrfCfs14,52
-tuieval/_version.py,sha256=qX-ZwL3llLIy2WRHxAygX62S9Li2VQEb17v_xj4E0wE,535
+tuieval/_version.py,sha256=wvLQcBRh9Ke_k0afcu8VifHptYL1UWBFOCSFcnpeLi0,535
 tuieval/builtin.toml,sha256=Kxw22RZCJhURUyu5Wz8tj5Y1nM85A8hFVRbA4ocoPUg,4293
 tuieval/cli.py,sha256=VwiaU1F9bZtU8C-gEowJq58kPeYQY4t6SY_HwrV-qgc,4702
 tuieval/client.py,sha256=kpsyLEZKfxCsvIGBHBbBy83TpI97CvnGCDq-4AbJS4Y,9986
 tuieval/compare.py,sha256=uXU_Yzf4zDSHoupHQrTmBhsfc8wCyI1nMaZQN81ybiA,30522
 tuieval/doctor.py,sha256=j2MdDORstsAGPnRmpsnxau1_fR6sTc_oEFGM5zugTJU,7246
-tuieval/engine.py,sha256=pGGlumC65FJMy9Jnpni4FoFNXhUHBDLgHYXi_A3qv7k,118847
+tuieval/engine.py,sha256=yMWvYq85WmYmk8fpMLRzYcxJxo7XcjEIUTa_y2VK7Kk,124538
 tuieval/export.py,sha256=dbHQgL_ubWtIPM1sLzAew_gETnrDJyd0zgqVA5vkpXM,9657
 tuieval/logs.py,sha256=01gZK6tf53dIThaIrs1bEsz9eqKYVlcxpyaLnSPTVxo,8127
 tuieval/machines.py,sha256=t9tu6CZeXZON4EtOVnEwP4VtaynpVLf9Cz8TNjf7jtY,20124
 tuieval/onboard.py,sha256=IYETVDWCCo9vF0Dtj4_rr9VanYeCky8PwP3-2g5A_l0,6215
-tuieval/packs.py,sha256=tiLmxsrX_M5EUMuR4Yvb7pLBLEI15wamHSjflD6wjoY,11299
+tuieval/packs.py,sha256=8bkF3xIDnWlt1JD2t3seHpMXW_RbMBgrCdIGEC-w-ZU,11458
 tuieval/profiles.py,sha256=soeLxjdB-l-OxSY8jZ4yjwfdSnsBV-QICcwKxRJS7nQ,3663
 tuieval/pta.py,sha256=i5oPj-Pz3N9u7bzzuZwsQvMGdxWdAncww2nR15K6PHY,12108
 tuieval/remove.py,sha256=Vwh43nm9lVb9U9aN-h6CMYbkTwdR3UWfZObwweF72n0,5157
-tuieval/run_evals.py,sha256=LtzXYWlPDdYKMWLmrmCpD6ff2ALLQRcUlgXGUdIFjQE,47005
+tuieval/run_evals.py,sha256=FU3bvu3jt9ZWg3YLsXNINbdeClXMioyc4zQ4yEhmBQs,47855
 tuieval/scaffold.py,sha256=YBOqniJpf_GtKKAFF3_OHxxMGbmTWdFjcClGcOnrrKE,12130
 tuieval/selftest.py,sha256=nExkrOidgymlmvkxImEBfrQG5iTUN5WuRMIKtqusCCs,9835
-tuieval/tui.py,sha256=MUHk9EDMpqWa-dn-fuC2LZ9F6C_wJ6A2YTKyU6XcDdg,172739
+tuieval/tui.py,sha256=bGAy0cP55rWkg9Cj3iAsSqR3tNM67Cj3G3nfvCitN8A,173351
 tuieval/tune.py,sha256=azxxthgJeag8JyoQUgaRxq6oI1XvEgzb9vkszjJZH7w,39645
-tuieval/verdict.py,sha256=VB-0LTxR9y8bJHT9RicxtE9K7cUHBdCu40hOCrsbmA4,19150
+tuieval/verdict.py,sha256=3SM8qrVdWNIFDENagaR5ZPBYEnjsBtkYB9WS5TDVssU,20517
 tuieval/watch_proxy.py,sha256=bnDO1i6n6yUs1hODoUWcMppJMGPavoQqs6aNYrIxpT8,11416
 tuieval/workspace.py,sha256=V_HhkaN1odr0OpjNsPYradRWLpzDldM7SwpvxohBL4U,895
 tuieval/yamlout.py,sha256=r1IokjLqW-Ztpg1fSn33ZW3VK00v1VvxymmU5Qy7GEw,606
@@ -47,8 +47,8 @@ tuieval/templates/packs/tool_call/pack.toml,sha256=n4bx0lGJHSmW41uVqhfIXJ2BWZpmT
 tuieval/templates/packs/tool_call/system.txt,sha256=mKxu-lB-0YjLHXdgeFJa8x2C180OvcNz5cHv2N5Bg3U,254
 tuieval/templates/packs/tool_call/tests.yaml,sha256=Nv7c4syrzJ9cU8EBKSK_1ZSFx9V02aCjW4-2TBCPT7s,1859
 tuieval/templates/packs/tool_call/tools.yaml,sha256=HgxCkh0On2VwbHSsjAGM8tNCK7ZtKLGrs7lMCn3Fprs,1048
-tuieval-0.2.0.dev77.dist-info/METADATA,sha256=LvwdGOxerLqRU5dR6g4Soi1TYEx3k1i32ei87hi9xnA,14580
-tuieval-0.2.0.dev77.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
-tuieval-0.2.0.dev77.dist-info/entry_points.txt,sha256=7eIhi1GpFxBP2OLJV02Nn8LwsNuO512eKOGt1IBbj04,45
-tuieval-0.2.0.dev77.dist-info/licenses/LICENSE,sha256=7wfRgGtBGH1aXhZkIn47oUznqKC18kEPMvUVgol7Ffc,1077
-tuieval-0.2.0.dev77.dist-info/RECORD,,
+tuieval-0.2.0.dev81.dist-info/METADATA,sha256=362fng3M9sOKj7AGpBf-ebNRTQgTjW84h1nfrn3YeKg,14580
+tuieval-0.2.0.dev81.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+tuieval-0.2.0.dev81.dist-info/entry_points.txt,sha256=7eIhi1GpFxBP2OLJV02Nn8LwsNuO512eKOGt1IBbj04,45
+tuieval-0.2.0.dev81.dist-info/licenses/LICENSE,sha256=7wfRgGtBGH1aXhZkIn47oUznqKC18kEPMvUVgol7Ffc,1077
+tuieval-0.2.0.dev81.dist-info/RECORD,,