tuieval 0.2.0.dev11__py3-none-any.whl → 0.2.0.dev23__py3-none-any.whl

tuieval/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.2.0.dev11'
-__version_tuple__ = version_tuple = (0, 2, 0, 'dev11')
+__version__ = version = '0.2.0.dev23'
+__version_tuple__ = version_tuple = (0, 2, 0, 'dev23')
 
 __commit_id__ = commit_id = None

tuieval/builtin.toml

@@ -0,0 +1,83 @@
+# Built-in defaults and servers, used when a workspace's models.toml doesn't define them.
+# [defaults] fill in key by key; [sampling] only when models.toml has no [sampling] at all (it's part of
+# every result's fingerprint); a server here is replaced whole by one of the same name in models.toml.
+
+[defaults]
+repeat = 3
+request_timeout_ms = 1800000   # per request; long thinking runs need it
+ready_timeout_s = 1200         # how long a model may take to load before giving up
+model_dirs = ["~/models"]      # where "Scan for models" looks for new GGUFs
+
+# Sent with every request. A model may override it (see `thinking` / `sampling` below); results
+# record the settings each model actually ran with, and the scorecard shows any difference.
+[sampling]
+temperature = 0.7
+top_p = 0.95
+top_k = 20
+min_p = 0.0
+max_tokens = 16384
+enable_thinking = true         # test every model in the SAME mode
+
+# llama.cpp's server (https://github.com/ggml-org/llama.cpp), started for each GGUF model.
+[servers.llama]
+# Output-affecting flags only. Anything here must be identical wherever results are compared,
+# so it's part of each result's fingerprint. Speed-only flags live in `perf` and `tune` below.
+cmd = [
+  "llama-server",
+  "-m", "{model}",
+  "--alias", "{served_name}",
+  "--host", "127.0.0.1", "--port", "{port}",
+  "--jinja",
+  "--metrics",
+  "-np", "1",
+  "-ctk", "{kv_type}", "-ctv", "{kv_type}",
+  "-c", "{ctx}",
+  "--reasoning-format", "auto",
+]
+port = 8080
+model_is_path = true           # check the GGUF exists and fits before starting
+vision_args = ["--mmproj", "{mmproj}"]   # appended only for models that set mmproj
+kv_type = "f16"                # a model's own `kv_type` (e.g. "q8_0") makes it a different model entry
+max_ctx = 65536                # upper bound; the fit check lowers it per machine (see machines.py)
+version_cmd = ["llama-server", "--version"]   # recorded with every result
+# Added to every request. cache_prompt = false: each answer is computed from scratch, so a repeat of
+# the same question reuses nothing from the last time it was asked (and its TTFT is honest).
+# Answers record any prompt tokens the server reused.
+request = { cache_prompt = false }
+
+# Speed-only flags, always applied (not tuned).
+perf = ["-ngl", "99"]
+
+# Speed-only knobs `tuieval tune` tries, one at a time from the best so far. The first option of
+# each knob is the default for models not tuned on this machine yet. Placeholders: {p} P-cores,
+# {p_minus_2}, {all} all cores. Options that stop the server from starting are skipped.
+# Check the flags against `llama-server --help` for your build.
+[servers.llama.tune]
+threads = [["-t", "{p}", "-tb", "{p}"], ["-t", "{p_minus_2}", "-tb", "{p}"], ["-t", "{all}", "-tb", "{all}"]]
+ubatch = [["-ub", "512"], ["-ub", "256"], ["-ub", "1024"]]
+batch = [["-b", "2048"], ["-b", "4096"]]
+flash_attn = [["-fa", "on"], ["-fa", "off"]]
+
+# A server that's already running (LM Studio, Ollama, vLLM, a remote box): give its URL and no cmd.
+# Its models are added with `tuieval add <model id> --server local`.
+[servers.local]
+url = "http://127.0.0.1:1234"   # LM Studio's default; Ollama is http://127.0.0.1:11434, vLLM :8000
+
+# OpenRouter: ANY model it serves, with no [[models]] entry. Name it when you run:
+#     tuieval run --tier smoke --only openrouter:qwen/qwen3-235b-a22b-2507
+#     TUI: tick "+ openrouter: any model" in the model list, or press "a" and type openrouter:<model id>
+# Results go to results/or-<id>/ and the model stays listed once it has results. A plain
+# `tuieval run` never includes OpenRouter models (they cost money); name them.
+# Needs OPENROUTER_API_KEY in the environment of the shell that starts tuieval.
+# pin_endpoint: every answer of a model comes from one provider endpoint (the closest to the released
+# weights), with no fallback, since providers run different quantizations.
+# To keep providers that store or train on prompts (and could learn your tests) out entirely, add
+#   request = { provider = { data_collection = "deny" } }
+[servers.openrouter]
+url = "https://openrouter.ai/api/v1"
+any_model = true
+label_prefix = "or"
+pin_endpoint = true
+api_key_env = "OPENROUTER_API_KEY"
+thinking_param = "reasoning"     # [sampling] enable_thinking -> OpenRouter's reasoning.enabled
+headers = { "X-Title" = "tuieval" }

tuieval/cli.py

@@ -2,28 +2,33 @@
 
 Usage: tuieval [--workspace DIR] [command] [options]
 
-  tuieval                     open the TUI
-  tuieval init [DIR]          create a workspace (models.toml, packs/, graders/)
-  tuieval new-pack NAME       a new pack from a starter template (--grader answer|rag|reply|tool_call|code)
-  tuieval add PATH|ID         register a model in models.toml (a GGUF, or a model id)
-  tuieval scan                list GGUFs in model_dirs that aren't in models.toml yet
-  tuieval list                the models tuieval knows, hidden ones separately (--packs: the packs)
-  tuieval remove MODEL        remove models, or every hidden one (--hidden); archived in removed/
+Start here
+  tuieval init [DIR]          create a workspace; in a terminal, a guided setup to a first result
+  tuieval                     open the TUI (? explains any screen)
+  tuieval doctor              check servers, models, packs and API keys, and say what to fix
+  tuieval add [PATH|ID]       add a model: a GGUF, a model id, or none to pick from running servers
+  tuieval new-pack NAME       a new pack of example questions (--from CSV, --about TOPIC, --grader …)
+  tuieval selftest            check every test's reference and wrong answers against its grader
   tuieval run [...]           run evals from the command line (tuieval run --help)
+  tuieval verdict             which model is ready for what, and what to run next
+
+Day to day
   tuieval compare [...]       scorecards (--speed, --pairwise, --failures)
-  tuieval regrade             re-score stored answers after changing a grader
-  tuieval verdict             production readiness (PASS/FAIL per model and use case)
-  tuieval report              the readiness verdicts with their evidence, as a markdown file
+  tuieval report              the verdicts with their evidence, as a markdown file
   tuieval history             every verdict change, newest first
-  tuieval selftest            check every test's reference and wrong answers against its grader
   tuieval items               tests that don't separate models or look broken
   tuieval capture LOG --pack  turn a real failure (logs/live/*.txt) into a new test
+  tuieval regrade             re-score stored answers after changing a grader
+  tuieval list                the models tuieval knows, hidden ones separately (--packs: the packs)
+  tuieval remove MODEL        remove models, or every hidden one (--hidden); archived in removed/
+
+Advanced
+  tuieval scan                list GGUFs in model_dirs that aren't in models.toml yet
   tuieval machines            this machine, other recorded machines, and fit and tuning per model
   tuieval tune MODEL          find the fastest speed-only server flags for a model on this machine
   tuieval export pi MODEL     serve a model in the pi coding agent with its tuned flags
   tuieval watch               a proxy that shows the reasoning of any app using your server
-  tuieval help                this help
-  tuieval --version
+  tuieval help · --version
 
 The workspace is --workspace DIR, else $TUIEVAL_HOME, else the current folder.
 """
@@ -67,6 +72,9 @@ def main(argv=None):
         if not workspace.is_workspace():
             sys.exit(f"{workspace.root()} isn't a tuieval workspace; run `tuieval init` first")
         return scaffold.cmd_new_pack(rest)
+    if cmd == "doctor":
+        from . import doctor
+        return doctor.cmd_doctor(rest)
     if cmd == "run":
         from . import run_evals
         return run_evals.main(rest)

tuieval/doctor.py

@@ -0,0 +1,131 @@
+"""tuieval doctor: checks the setup and says what to fix (servers, models, packs, API keys).
+
+    tuieval doctor
+
+Each line is ✓ (fine), ! (worth knowing) or ✗ (a run would fail), with the fix. Exits 1 if any ✗.
+"""
+import argparse
+import os
+import platform
+import sys
+
+from . import engine
+from . import workspace
+
+OK, WARN, BAD = "✓", "!", "✗"
+COLOR = {OK: "\033[32m", WARN: "\033[33m", BAD: "\033[31m"}
+
+
+class Report:
+    def __init__(self):
+        self.bad = 0
+        self.color = sys.stdout.isatty() and not os.environ.get("NO_COLOR")
+
+    def line(self, mark, text, fix=""):
+        self.bad += mark == BAD
+        m = f"{COLOR[mark]}{mark}\033[0m" if self.color else mark
+        print(f"  {m} {text}" + (f"\n      → {fix}" if fix else ""))
+
+    def head(self, text):
+        print(f"\n{text}")
+
+
+def check(models_path=None):
+    """Print the report; returns the number of problems that would make a run fail."""
+    r = Report()
+    r.head("tuieval")
+    py = platform.python_version()
+    r.line(OK if sys.version_info >= (3, 11) else BAD, f"Python {py}", "" if sys.version_info >= (3, 11)
+           else "tuieval needs Python 3.11 or newer")
+    try:
+        e = engine.Engine(models_path=models_path)
+    except Exception as ex:   # a broken models.toml: nothing else can be checked
+        r.line(BAD, f"models.toml in {workspace.root()}: {ex}", "fix the file (tuieval init makes a fresh one elsewhere)")
+        return r.bad
+    r.line(OK, f"workspace {e.root}")
+
+    r.head("Packs")
+    for err in e.pack_errors:
+        r.line(BAD, err, "fix the pack, then: tuieval selftest")
+    if e.packs:
+        tests = sum(len(p.tests) for p in e.packs.values())
+        r.line(OK, f"{len(e.packs)} pack(s), {tests} tests  (tuieval selftest checks their answers)")
+    elif not e.pack_errors:
+        r.line(BAD, "no packs yet", "tuieval new-pack my-first-pack   (a pack of example questions to edit)")
+
+    r.head("Servers")
+    found = engine.detect_servers(engine.detect_ports(e.cfg))
+    used = {m["server"] for m in e.cfg["models"]}
+    answering = {}
+    for name, server in e.cfg["servers"].items():
+        if server.get("any_model") or server.get("api_key_env"):
+            key = server.get("api_key_env")
+            if key and os.environ.get(key):
+                r.line(OK, f"{name}: {key} is set")
+            elif key:
+                r.line(WARN, f"{name}: {key} isn't set (needed only for {name} models)",
+                       f"export {key}=… in the shell you start tuieval from")
+            continue
+        if server.get("cmd"):
+            missing = engine.missing_program([str(a) for a in server["cmd"]],
+                                             engine.expand(server["cwd"]) if server.get("cwd") else None)
+            if missing:
+                r.line(BAD if name in used else WARN,
+                       f"{name}: {missing} isn't installed" + ("" if name in used else " (no model uses it yet)"),
+                       engine.install_hint(missing))
+            else:
+                r.line(OK, f"{name}: {server['cmd'][0]} is installed; tuieval starts it for each model")
+            continue
+        url = server.get("url", "").rstrip("/").removesuffix("/v1")
+        ids = engine.probe_models(url) if url else None
+        answering[name] = ids
+        if ids is not None:
+            r.line(OK, f"{name}: {url} answers, serving {len(ids)} model(s)" + (f": {', '.join(ids[:5])}" if ids else ""))
+        else:
+            elsewhere = [f for f in found if f["url"] != url]
+            fix = (f"start the server at {url}, or "
+                   + ("run `tuieval add` to add the models of the one running at " + ", ".join(
+                       f"{f['url']}" + (f" ({f['usual']})" if f["usual"] else "") for f in elsewhere)
+                      if elsewhere else engine.url_fix(e.cfg, name)))
+            r.line(BAD if name in used else WARN,
+                   f"{name}: nothing answers at {url}" + ("" if name in used else " (no model uses it yet)"), fix)
+    known = {s.get("url", "").rstrip("/").removesuffix("/v1") for s in e.cfg["servers"].values()}
+    for f in found:
+        if f["url"] not in known:
+            r.line(WARN, f"found a server at {f['url']}" + (f" ({f['usual']}'s usual port)" if f["usual"] else "")
+                   + f" serving {len(f['models'])} model(s), not in models.toml",
+                   "add its models with: tuieval add")
+
+    r.head("Models")
+    if not e.cfg["models"]:
+        r.line(BAD, "no models yet", "tuieval add <GGUF path or model id>, or press a in the TUI")
+    for m in e.cfg["models"]:
+        server = e.cfg["servers"][m["server"]]
+        label = f"{m['label']} ({m['server']})"
+        if server.get("model_is_path"):
+            path = engine.expand(m["model"])
+            r.line(OK if os.path.exists(path) else BAD, f"{label}: {path}",
+                   "" if os.path.exists(path) else "the file is gone; fix `model` in models.toml or tuieval remove it")
+        elif m["server"] in answering:
+            ids = answering[m["server"]]
+            if ids is None:
+                r.line(BAD, f"{label}: its server isn't running", "see Servers above")
+            elif engine.serves(ids, m["served_name"]):
+                r.line(OK, f"{label}: served")
+            else:
+                r.line(BAD, f"{label}: the server doesn't list {m['served_name']!r}",
+                       f"load it in the server, or use one of: {', '.join(ids[:8]) or 'none listed'}")
+        else:
+            r.line(OK, label)
+
+    print("\n" + ("Ready: open the TUI with `tuieval`, or run `tuieval run --tier smoke`." if not r.bad
+                  else f"{r.bad} problem(s) to fix before a run (✗ above)."))
+    return r.bad
+
+
+def cmd_doctor(argv):
+    p = argparse.ArgumentParser(prog="tuieval doctor", description=__doc__,
+                                formatter_class=argparse.RawDescriptionHelpFormatter)
+    p.add_argument("--models", default=None, help="default: the workspace's models.toml")
+    a = p.parse_args(argv)
+    sys.exit(1 if check(a.models) else 0)

tuieval/engine.py

@@ -38,6 +38,7 @@ import mimetypes
 import os
 import random
 import re
+import shutil
 import signal
 import socket
 import statistics
@@ -72,10 +73,26 @@ def slug(model):
     return re.sub(r"[^a-z0-9._-]+", "-", name.lower()).strip("-.")
 
 
+BUILTIN_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "builtin.toml")
+
+
+def builtin_config():
+    with open(BUILTIN_PATH, "rb") as f:
+        return tomllib.load(f)
+
+
 def load_config(path):
-    """Read models.toml and fill in each model's defaults."""
+    """Read models.toml, fill in the built-ins it leaves out (builtin.toml) and each model's defaults.
+    [defaults] are filled key by key; [sampling] only when models.toml has none (sampling is part of
+    every result's fingerprint, so a workspace's own keys are never added to); servers by name."""
     with open(path, "rb") as f:
         cfg = tomllib.load(f)
+    builtin = builtin_config()
+    cfg["defaults"] = {**builtin["defaults"], **cfg.get("defaults", {})}
+    cfg.setdefault("sampling", builtin["sampling"])
+    own = cfg.get("servers", {})
+    cfg["servers"] = {**builtin["servers"], **own}
+    cfg["builtin_servers"] = sorted(set(builtin["servers"]) - set(own))   # not defined in models.toml
     cfg.setdefault("models", [])
     seen = set()
     for m in cfg["models"]:
@@ -218,6 +235,8 @@ def infer_server(cfg, model):
     is_file = model.lower().endswith(".gguf")
     choices = [n for n, s in cfg["servers"].items()
                if bool(s.get("model_is_path")) == is_file and not s.get("any_model")]
+    own = [n for n in choices if n not in cfg.get("builtin_servers", ())]   # the workspace's own come first
+    choices = own or choices
     return choices[0] if len(choices) == 1 else None
 
 
@@ -358,6 +377,7 @@ def load_presets(path):
         tags = ["moe"]            # models with any of these tags …
         models = ["foo", "bar"]   # … and/or these labels (omit both for every model)
         repeat = 3
+        tests = { coding = ["parse-dates"] }   # optional: only these tests of a pack
     """
     if not os.path.isfile(path):
         return {}
@@ -366,19 +386,24 @@ def load_presets(path):
 
 
 def resolve_preset(cfg, preset, pack_names):
-    """(labels, packs, repeat) for a preset; unknown names are dropped."""
+    """(labels, packs, repeat, tests) for a preset; unknown names are dropped. tests: {pack: [test
+    ids]} for packs where only some tests run."""
     labels = [m["label"] for m in cfg["models"]]
     if preset.get("models") or preset.get("tags"):
         want, tags = set(preset.get("models", [])), set(preset.get("tags", []))
         labels = [m["label"] for m in cfg["models"] if m["label"] in want or tags & set(m["tags"])]
     packs = [p for p in preset.get("packs", pack_names) if p in pack_names]
-    return labels, packs, preset.get("repeat")
+    tests = {p: list(ids) for p, ids in (preset.get("tests") or {}).items() if p in packs and ids}
+    return labels, packs, preset.get("repeat"), tests
 
 
-def save_preset(path, name, labels, packs, repeat):
+def save_preset(path, name, labels, packs, repeat, tests=None):
     """Add or replace one preset, keeping the rest of the file as written."""
+    tests = {p: ids for p, ids in (tests or {}).items() if p in packs and ids}
     block = (f"[{name}]\nmodels = [{', '.join(json.dumps(l) for l in labels)}]\n"
-             f"packs = [{', '.join(json.dumps(p) for p in packs)}]\n" + (f"repeat = {int(repeat)}\n" if repeat else ""))
+             f"packs = [{', '.join(json.dumps(p) for p in packs)}]\n" + (f"repeat = {int(repeat)}\n" if repeat else "")
+             + ("tests = { " + ", ".join(f"{json.dumps(p)} = [{', '.join(json.dumps(i) for i in ids)}]"
+                                         for p, ids in tests.items()) + " }\n" if tests else ""))
     text = open(path).read() if os.path.isfile(path) else \
         "# Saved selections for the TUI (p) and tuieval run --preset <name>. See engine.load_presets.\n"
     pattern = re.compile(rf"^\[{re.escape(name)}\]\n(?:(?!\[).*\n?)*", re.MULTILINE)
@@ -394,12 +419,13 @@ class Job:
     model: dict
     pack: packs_mod.Pack
     repeat: int
-    tests: list             # the tests this tier runs
+    tests: list             # the tests this tier runs (or the ones picked)
     out_path: str
     sampling: dict
     tier: str = "certify"
     settings: dict = dataclasses.field(default_factory=dict)  # sampling + serving identity (fingerprinted)
     merge: bool = True      # keep matching earlier results and only run what's missing
+    redo: bool = False      # rerun the picked tests, keeping the pack's other answers
     status: str = "waiting"  # waiting, loading, running, done, failed, skipped
     note: str = ""
     discards: list = dataclasses.field(default_factory=list)  # [(what, answers, why)] set aside on start
@@ -504,6 +530,115 @@ def server_error(log, start):
         return line.split("]: ", 1)[-1]
     return line.removeprefix("error: ")
 
+def missing_program(cmd, cwd=None):
+    """The program a server command runs, if it isn't installed (not on PATH, or no such file); else None."""
+    exe = next((a for a in cmd if a != "env" and not re.match(r"^[A-Za-z_][A-Za-z0-9_]*=", a)), None)
+    if not exe:
+        return None
+    if "/" in exe:
+        return None if os.path.isfile(os.path.join(cwd or "", expand(exe))) else exe
+    return None if shutil.which(exe) else exe
+
+
+def install_hint(program):
+    """How to get a server program, for the ones tuieval's templates use."""
+    if os.path.basename(program) == "llama-server":
+        return ("install llama.cpp (macOS: brew install llama.cpp; others: a release from "
+                "https://github.com/ggml-org/llama.cpp/releases) so llama-server is on PATH")
+    return "install it, or fix the server's cmd in models.toml"
+
+
+def probe_models(base_url, timeout=2.0, headers=None):
+    """The model ids an OpenAI-compatible server lists at base_url, or None when nothing answers."""
+    req = urllib.request.Request(base_url.rstrip("/").removesuffix("/v1") + "/v1/models", headers=headers or {})
+    try:
+        with urllib.request.urlopen(req, timeout=timeout) as r:
+            return [str(x.get("id", "")) for x in json.load(r).get("data", []) if isinstance(x, dict)]
+    except (OSError, ValueError, AttributeError):
+        return None
+
+
+# Where local OpenAI-compatible servers usually listen. models.toml [defaults] detect_ports
+# replaces the list (e.g. a server on a custom port, or [] to never look).
+DETECT_PORTS = {1234: "LM Studio", 11434: "Ollama", 8080: "llama-server", 8000: "vLLM"}
+
+
+SERVER_NAMES = {1234: "lmstudio", 11434: "ollama", 8080: "llamacpp", 8000: "vllm"}   # for new [servers.*] entries
+
+
+def detect_ports(cfg=None):
+    """Ports to look for running servers on: $TUIEVAL_DETECT_PORTS (comma-separated, empty = none),
+    else models.toml [defaults] detect_ports, else the usual ones."""
+    env = os.environ.get("TUIEVAL_DETECT_PORTS")
+    if env is not None:
+        return [int(p) for p in env.replace(" ", "").split(",") if p]
+    return [int(p) for p in ((cfg or {}).get("defaults") or {}).get("detect_ports", list(DETECT_PORTS))]
+
+
+def _norm_url(url):
+    return (url or "").rstrip("/").removesuffix("/v1").replace("localhost", "127.0.0.1")
+
+
+def server_for_url(cfg, url):
+    """The models.toml server already pointing at url (a server tuieval doesn't start), or None."""
+    return next((n for n, s in cfg["servers"].items() if not s.get("cmd") and _norm_url(s.get("url")) == _norm_url(url)),
+                None)
+
+
+def ensure_server(models_path, url, port=None):
+    """The name of a models.toml server for an already-running server at url, adding a
+    [servers.<name>] block when there's none (named after the app that usually uses the port)."""
+    cfg = load_config(models_path)
+    name = server_for_url(cfg, url)
+    if name:
+        return name
+    base = SERVER_NAMES.get(port, f"local-{port}" if port else "local-server")
+    name = base if base not in cfg["servers"] else f"{base}-{port}"
+    with open(models_path) as f:
+        original = f.read()
+    with open(models_path, "w") as f:
+        f.write(original.rstrip("\n") + f'\n\n[servers.{name}]\nurl = "{_norm_url(url)}"\n')
+    try:
+        load_config(models_path)
+    except BaseException:
+        with open(models_path, "w") as f:
+            f.write(original)
+        raise
+    return name
+
+
+def url_fix(cfg, name):
+    """How to point a server at another URL in models.toml (a built-in one has no section there yet)."""
+    if name in cfg.get("builtin_servers", ()):
+        return f'add [servers.{name}] with url = "http://127.0.0.1:<port>" to models.toml'
+    return f"fix `url` under [servers.{name}] in models.toml"
+
+
+def find_running(cfg, model):
+    """Running servers (detect_servers) that list this model id, best match first."""
+    found = detect_servers(detect_ports(cfg))
+    exact = [f for f in found if model in f["models"]]
+    return exact or [f for f in found if serves(f["models"], model)]
+
+
+def detect_servers(ports, timeout=1.0):
+    """[{url, port, usual, models}] for each local port with an OpenAI-compatible server answering.
+    usual: the app that normally uses that port (a hint, not a check)."""
+    found = []
+    for port in ports:
+        url = f"http://127.0.0.1:{port}"
+        ids = probe_models(url, timeout)
+        if ids is not None:
+            found.append({"url": url, "port": port, "usual": DETECT_PORTS.get(port), "models": ids})
+    return found
+
+
+def serves(ids, name):
+    """Does a server listing these model ids serve this model (names match loosely: aliases, paths)?"""
+    want = name.lower()
+    return any(want == i.lower() or want in i.lower() or (i and i.lower() in want) for i in ids)
+
+
 def pack_min_context(pack):
     """Rough context a pack needs: its longest prompt (~4 chars/token) plus room to answer."""
     return max(len(t["input"]) for t in pack.tests) // 4 + 4096
@@ -614,6 +749,15 @@ class Engine:
                 raise ConfigError(f"couldn't read {name}'s model list ({e})") from None
         return self._catalogues[name]
 
+    def has_results(self):
+        """Has any model run anything yet (smoke runs included)?"""
+        for base in (self.results_dir, os.path.join(self.results_dir, "smoke")):
+            for label in os.listdir(base) if os.path.isdir(base) else []:
+                d = os.path.join(base, label)
+                if os.path.isdir(d) and any(f.endswith(".json") for f in os.listdir(d)):
+                    return True
+        return False
+
     def _add_remote_models(self):
         """Add the models named at run time this session, and ones that have results from an
         earlier session (their result files record the model), to cfg["models"]."""
@@ -868,7 +1012,7 @@ class Engine:
         if job.merge:
             data = self._result(job.out_path)
             if data and data["pack"]["fingerprint"] == job.pack.fingerprint and settings_match(data, job.settings):
-                kept = data["results"]
+                kept = self._kept(job, data["results"])
                 if kept:
                     earlier = self._file_sittings(data["run"])
         head, new = self._partial_lines(job.out_path + ".partial.jsonl")
@@ -877,6 +1021,14 @@ class Engine:
         saved = self._saved_sittings(job)
         return kept, new, earlier + saved + self._rebuild_sittings(new, saved + earlier)
 
+    @staticmethod
+    def _kept(job, rows):
+        """A finished file's rows that stay: all of them, less the picked tests' when redoing them."""
+        if not job.redo:
+            return rows
+        redo = {t["id"] for t in job.tests}
+        return [r for r in rows if r["test"] not in redo]
+
     def _fill_progress(self, job):
         """Set a job's done/passed/failed and earlier sittings from what's stored."""
         kept, new, job.earlier = self._stored(job)
@@ -927,11 +1079,13 @@ class Engine:
         return not current and header_matches(head, pack.fingerprint, settings)
 
     # ---- planning
-    def plan(self, labels, pack_names, repeat=None, tier="screen", force=False):
+    def plan(self, labels, pack_names, repeat=None, tier="screen", force=False, tests=None):
         """Jobs for models x packs. tier: smoke (3 tests), screen (a spread sample), certify (all
         tests). repeat: times each question is asked; None = the tier default (1 for smoke and
-        screen, the pack's certification repeats for certify). Earlier matching results are kept
-        and only missing answers are run, unless force."""
+        screen, the pack's certification repeats for certify). tests: {pack: [test ids]} runs just
+        those tests of a pack instead of the tier's sample (unknown ids raise PackError). Earlier
+        matching results are kept and only missing answers are run, unless force; force with
+        picked tests reruns those and keeps the pack's other answers."""
         if tier not in TIERS:
             raise ValueError(f"tier must be one of {TIERS}")
         jobs = []
@@ -940,8 +1094,10 @@ class Engine:
             for name in pack_names:
                 pack = self.packs[name]
                 reps = repeat or self.default_repeat(pack, tier)
-                job = Job(m, pack, reps, pack.select(tier), self.result_path(label, name, tier),
-                          effective_sampling(self.cfg, m), tier=tier, merge=not force, settings=self.settings(m))
+                picked = (tests or {}).get(name)
+                job = Job(m, pack, reps, pack.pick(picked) if picked else pack.select(tier),
+                          self.result_path(label, name, tier), effective_sampling(self.cfg, m), tier=tier,
+                          merge=not force or bool(picked), redo=force and bool(picked), settings=self.settings(m))
                 sv = self.serving(m)
                 status = self.result_status(label, name, tier)
                 job.discards = self._discards(job)
@@ -958,7 +1114,7 @@ class Engine:
                 elif missing:
                     job.status, job.note = "skipped", f"{', '.join(missing)} not installed in tuieval's Python"
                 elif status and status.startswith("outdated"):
-                    job.merge, job.note = False, status.split(": ", 1)[-1] + ", rerunning"
+                    job.merge, job.redo, job.note = False, False, status.split(": ", 1)[-1] + ", rerunning"
                     if not force:   # a rerun under the new settings already under way resumes
                         self._fill_progress(job)
                         if job.done:
@@ -1387,6 +1543,9 @@ class Engine:
                     raise ModelFailed(f"mmproj file not found: {expand(m['mmproj'])}")
                 if cwd and not os.path.isdir(cwd):
                     raise ModelFailed(f"server directory not found: {cwd}")
+                missing = missing_program(cmd, cwd)
+                if missing:
+                    raise ModelFailed(f"{missing} isn't installed: {install_hint(missing)}")
                 if port_in_use(port):
                     raise ModelFailed(f"port {port} is already in use; stop the server running there")
                 os.makedirs(os.path.join(self.log_dir, "server"), exist_ok=True)
@@ -1411,7 +1570,7 @@ class Engine:
                 key_env = server.get("api_key_env")
                 if key_env and not os.environ.get(key_env):
                     raise ModelFailed(f"{key_env} is not set; export it in the shell you start tuieval from")
-                where = f"already running at {base_url}"
+                where = f"using the server at {base_url}"
                 if server.get("pin_endpoint"):
                     ep = self.endpoint(m)
                     if not ep or not ep.get("tag"):
@@ -1421,6 +1580,14 @@ class Engine:
                 self.emit("model_loading", label=m["label"], command=[f"({where})"], log="")
                 t0 = time.time()
             deadline = time.time() + defaults["ready_timeout_s"]
+            if srv is None and not server.get("health") and probe_models(base_url, 5, self.request_headers(m)) is None:
+                # a server we don't start: if nothing answers now, waiting won't help
+                time.sleep(3)
+                if probe_models(base_url, 5, self.request_headers(m)) is None:
+                    raise ModelFailed(
+                        f"nothing is answering at {base_url}. Start that server, or {url_fix(self.cfg, m['server'])} "
+                        "(common local ports: LM Studio 1234, Ollama 11434, llama-server 8080, vLLM 8000); "
+                        "`tuieval doctor` checks them all")
             while True:
                 self._check()
                 if srv is not None and srv.poll() is not None:
@@ -1442,9 +1609,8 @@ class Engine:
                     raise ModelFailed(f"server not ready after {defaults['ready_timeout_s']}s")
                 time.sleep(0.5)
             info["load_s"] = time.time() - t0 if srv is not None else None
-            want = m["served_name"].lower()
             # A health-checked server we started ourselves serves exactly the model we gave it.
-            if not (server.get("health") and srv is not None) and not any(want == i.lower() or want in i.lower() or (i and i.lower() in want) for i in ids):
+            if not (server.get("health") and srv is not None) and not serves(ids, m["served_name"]):
                 raise ModelFailed(f"server reports {ids}, expected {m['served_name']!r}; "
                                   "not running to avoid mislabelled results")
             if log and server.get("log_facts"):
@@ -1656,7 +1822,7 @@ class Engine:
             data = self._result(job.out_path)
             if data and data["pack"]["fingerprint"] == job.pack.fingerprint and \
                     settings_match(data, job.settings):
-                kept = data["results"]
+                kept = self._kept(job, data["results"])
                 if kept:
                     job.earlier = self._file_sittings(data["run"])
         new = []
@@ -1789,18 +1955,19 @@ class Engine:
         job.note = f"{job.passed}/{job.done} passed"
         self.emit("job_done", job=job)
 
-    def _archive(self, job):
-        """Keep a result file that is about to be replaced (rerun or outdated) in history/."""
+    def _archive(self, job, keep=False):
+        """Keep a result file that is about to be replaced (rerun or outdated) in history/; keep:
+        copy it (some of its answers stay in the new file)."""
         if job.tier == "smoke" or not os.path.isfile(job.out_path):
             return
         hist = os.path.join(os.path.dirname(job.out_path), "history")
         os.makedirs(hist, exist_ok=True)
         stamp = time.strftime("%Y%m%d-%H%M%S", time.localtime(os.path.getmtime(job.out_path)))
-        os.replace(job.out_path, os.path.join(hist, f"{job.pack.name}-{stamp}.json"))
+        (shutil.copy2 if keep else os.replace)(job.out_path, os.path.join(hist, f"{job.pack.name}-{stamp}.json"))
 
     def _write_result(self, job, records, run_info):
-        if not job.merge:
-            self._archive(job)
+        if not job.merge or job.redo:
+            self._archive(job, keep=job.redo)
         m = job.model
         data = {
             "format": RESULT_FORMAT,