tuieval 0.2.0.dev14__py3-none-any.whl → 0.2.0.dev25__py3-none-any.whl

tuieval/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.2.0.dev14'
-__version_tuple__ = version_tuple = (0, 2, 0, 'dev14')
+__version__ = version = '0.2.0.dev25'
+__version_tuple__ = version_tuple = (0, 2, 0, 'dev25')
 
 __commit_id__ = commit_id = None

tuieval/builtin.toml

@@ -0,0 +1,83 @@
+# Built-in defaults and servers, used when a workspace's models.toml doesn't define them.
+# [defaults] fill in key by key; [sampling] only when models.toml has no [sampling] at all (it's part of
+# every result's fingerprint); a server here is replaced whole by one of the same name in models.toml.
+
+[defaults]
+repeat = 3
+request_timeout_ms = 1800000   # per request; long thinking runs need it
+ready_timeout_s = 1200         # how long a model may take to load before giving up
+model_dirs = ["~/models"]      # where "Scan for models" looks for new GGUFs
+
+# Sent with every request. A model may override it (see `thinking` / `sampling` below); results
+# record the settings each model actually ran with, and the scorecard shows any difference.
+[sampling]
+temperature = 0.7
+top_p = 0.95
+top_k = 20
+min_p = 0.0
+max_tokens = 16384
+enable_thinking = true         # test every model in the SAME mode
+
+# llama.cpp's server (https://github.com/ggml-org/llama.cpp), started for each GGUF model.
+[servers.llama]
+# Output-affecting flags only. Anything here must be identical wherever results are compared,
+# so it's part of each result's fingerprint. Speed-only flags live in `perf` and `tune` below.
+cmd = [
+  "llama-server",
+  "-m", "{model}",
+  "--alias", "{served_name}",
+  "--host", "127.0.0.1", "--port", "{port}",
+  "--jinja",
+  "--metrics",
+  "-np", "1",
+  "-ctk", "{kv_type}", "-ctv", "{kv_type}",
+  "-c", "{ctx}",
+  "--reasoning-format", "auto",
+]
+port = 8080
+model_is_path = true           # check the GGUF exists and fits before starting
+vision_args = ["--mmproj", "{mmproj}"]   # appended only for models that set mmproj
+kv_type = "f16"                # a model's own `kv_type` (e.g. "q8_0") makes it a different model entry
+max_ctx = 65536                # upper bound; the fit check lowers it per machine (see machines.py)
+version_cmd = ["llama-server", "--version"]   # recorded with every result
+# Added to every request. cache_prompt = false: each answer is computed from scratch, so a repeat of
+# the same question reuses nothing from the last time it was asked (and its TTFT is honest).
+# Answers record any prompt tokens the server reused.
+request = { cache_prompt = false }
+
+# Speed-only flags, always applied (not tuned).
+perf = ["-ngl", "99"]
+
+# Speed-only knobs `tuieval tune` tries, one at a time from the best so far. The first option of
+# each knob is the default for models not tuned on this machine yet. Placeholders: {p} P-cores,
+# {p_minus_2}, {all} all cores. Options that stop the server from starting are skipped.
+# Check the flags against `llama-server --help` for your build.
+[servers.llama.tune]
+threads = [["-t", "{p}", "-tb", "{p}"], ["-t", "{p_minus_2}", "-tb", "{p}"], ["-t", "{all}", "-tb", "{all}"]]
+ubatch = [["-ub", "512"], ["-ub", "256"], ["-ub", "1024"]]
+batch = [["-b", "2048"], ["-b", "4096"]]
+flash_attn = [["-fa", "on"], ["-fa", "off"]]
+
+# A server that's already running (LM Studio, Ollama, vLLM, a remote box): give its URL and no cmd.
+# Its models are added with `tuieval add <model id> --server local`.
+[servers.local]
+url = "http://127.0.0.1:1234"   # LM Studio's default; Ollama is http://127.0.0.1:11434, vLLM :8000
+
+# OpenRouter: ANY model it serves, with no [[models]] entry. Name it when you run:
+#     tuieval run --tier smoke --only openrouter:qwen/qwen3-235b-a22b-2507
+#     TUI: tick "+ openrouter: any model" in the model list, or press "a" and type openrouter:<model id>
+# Results go to results/or-<id>/ and the model stays listed once it has results. A plain
+# `tuieval run` never includes OpenRouter models (they cost money); name them.
+# Needs OPENROUTER_API_KEY in the environment of the shell that starts tuieval.
+# pin_endpoint: every answer of a model comes from one provider endpoint (the closest to the released
+# weights), with no fallback, since providers run different quantizations.
+# To keep providers that store or train on prompts (and could learn your tests) out entirely, add
+#   request = { provider = { data_collection = "deny" } }
+[servers.openrouter]
+url = "https://openrouter.ai/api/v1"
+any_model = true
+label_prefix = "or"
+pin_endpoint = true
+api_key_env = "OPENROUTER_API_KEY"
+thinking_param = "reasoning"     # [sampling] enable_thinking -> OpenRouter's reasoning.enabled
+headers = { "X-Title" = "tuieval" }

tuieval/cli.py

@@ -2,28 +2,33 @@
 
 Usage: tuieval [--workspace DIR] [command] [options]
 
-  tuieval                     open the TUI
-  tuieval init [DIR]          create a workspace (models.toml, packs/, graders/)
-  tuieval new-pack NAME       a new pack from a starter template (--grader answer|rag|reply|tool_call|code)
-  tuieval add PATH|ID         register a model in models.toml (a GGUF, or a model id)
-  tuieval scan                list GGUFs in model_dirs that aren't in models.toml yet
-  tuieval list                the models tuieval knows, hidden ones separately (--packs: the packs)
-  tuieval remove MODEL        remove models, or every hidden one (--hidden); archived in removed/
+Start here
+  tuieval init [DIR]          create a workspace; in a terminal, a guided setup to a first result
+  tuieval                     open the TUI (? explains any screen)
+  tuieval doctor              check servers, models, packs and API keys, and say what to fix
+  tuieval add [PATH|ID]       add a model: a GGUF, a model id, or none to pick from running servers
+  tuieval new-pack NAME       a new pack of example questions (--from CSV, --about TOPIC, --grader …)
+  tuieval selftest            check every test's reference and wrong answers against its grader
   tuieval run [...]           run evals from the command line (tuieval run --help)
+  tuieval verdict             which model is ready for what, and what to run next
+
+Day to day
   tuieval compare [...]       scorecards (--speed, --pairwise, --failures)
-  tuieval regrade             re-score stored answers after changing a grader
-  tuieval verdict             production readiness (PASS/FAIL per model and use case)
-  tuieval report              the readiness verdicts with their evidence, as a markdown file
+  tuieval report              the verdicts with their evidence, as a markdown file
   tuieval history             every verdict change, newest first
-  tuieval selftest            check every test's reference and wrong answers against its grader
   tuieval items               tests that don't separate models or look broken
   tuieval capture LOG --pack  turn a real failure (logs/live/*.txt) into a new test
+  tuieval regrade             re-score stored answers after changing a grader
+  tuieval list                the models tuieval knows, hidden ones separately (--packs: the packs)
+  tuieval remove MODEL        remove models, or every hidden one (--hidden); archived in removed/
+
+Advanced
+  tuieval scan                list GGUFs in model_dirs that aren't in models.toml yet
   tuieval machines            this machine, other recorded machines, and fit and tuning per model
   tuieval tune MODEL          find the fastest speed-only server flags for a model on this machine
   tuieval export pi MODEL     serve a model in the pi coding agent with its tuned flags
   tuieval watch               a proxy that shows the reasoning of any app using your server
-  tuieval help                this help
-  tuieval --version
+  tuieval help · --version
 
 The workspace is --workspace DIR, else $TUIEVAL_HOME, else the current folder.
 """
@@ -67,6 +72,9 @@ def main(argv=None):
         if not workspace.is_workspace():
             sys.exit(f"{workspace.root()} isn't a tuieval workspace; run `tuieval init` first")
         return scaffold.cmd_new_pack(rest)
+    if cmd == "doctor":
+        from . import doctor
+        return doctor.cmd_doctor(rest)
     if cmd == "run":
         from . import run_evals
         return run_evals.main(rest)

tuieval/doctor.py

@@ -0,0 +1,131 @@
+"""tuieval doctor: checks the setup and says what to fix (servers, models, packs, API keys).
+
+    tuieval doctor
+
+Each line is ✓ (fine), ! (worth knowing) or ✗ (a run would fail), with the fix. Exits 1 if any ✗.
+"""
+import argparse
+import os
+import platform
+import sys
+
+from . import engine
+from . import workspace
+
+OK, WARN, BAD = "✓", "!", "✗"
+COLOR = {OK: "\033[32m", WARN: "\033[33m", BAD: "\033[31m"}
+
+
+class Report:
+    def __init__(self):
+        self.bad = 0
+        self.color = sys.stdout.isatty() and not os.environ.get("NO_COLOR")
+
+    def line(self, mark, text, fix=""):
+        self.bad += mark == BAD
+        m = f"{COLOR[mark]}{mark}\033[0m" if self.color else mark
+        print(f"  {m} {text}" + (f"\n      → {fix}" if fix else ""))
+
+    def head(self, text):
+        print(f"\n{text}")
+
+
+def check(models_path=None):
+    """Print the report; returns the number of problems that would make a run fail."""
+    r = Report()
+    r.head("tuieval")
+    py = platform.python_version()
+    r.line(OK if sys.version_info >= (3, 11) else BAD, f"Python {py}", "" if sys.version_info >= (3, 11)
+           else "tuieval needs Python 3.11 or newer")
+    try:
+        e = engine.Engine(models_path=models_path)
+    except Exception as ex:   # a broken models.toml: nothing else can be checked
+        r.line(BAD, f"models.toml in {workspace.root()}: {ex}", "fix the file (tuieval init makes a fresh one elsewhere)")
+        return r.bad
+    r.line(OK, f"workspace {e.root}")
+
+    r.head("Packs")
+    for err in e.pack_errors:
+        r.line(BAD, err, "fix the pack, then: tuieval selftest")
+    if e.packs:
+        tests = sum(len(p.tests) for p in e.packs.values())
+        r.line(OK, f"{len(e.packs)} pack(s), {tests} tests  (tuieval selftest checks their answers)")
+    elif not e.pack_errors:
+        r.line(BAD, "no packs yet", "tuieval new-pack my-first-pack   (a pack of example questions to edit)")
+
+    r.head("Servers")
+    found = engine.detect_servers(engine.detect_ports(e.cfg))
+    used = {m["server"] for m in e.cfg["models"]}
+    answering = {}
+    for name, server in e.cfg["servers"].items():
+        if server.get("any_model") or server.get("api_key_env"):
+            key = server.get("api_key_env")
+            if key and os.environ.get(key):
+                r.line(OK, f"{name}: {key} is set")
+            elif key:
+                r.line(WARN, f"{name}: {key} isn't set (needed only for {name} models)",
+                       f"export {key}=… in the shell you start tuieval from")
+            continue
+        if server.get("cmd"):
+            missing = engine.missing_program([str(a) for a in server["cmd"]],
+                                             engine.expand(server["cwd"]) if server.get("cwd") else None)
+            if missing:
+                r.line(BAD if name in used else WARN,
+                       f"{name}: {missing} isn't installed" + ("" if name in used else " (no model uses it yet)"),
+                       engine.install_hint(missing))
+            else:
+                r.line(OK, f"{name}: {server['cmd'][0]} is installed; tuieval starts it for each model")
+            continue
+        url = server.get("url", "").rstrip("/").removesuffix("/v1")
+        ids = engine.probe_models(url) if url else None
+        answering[name] = ids
+        if ids is not None:
+            r.line(OK, f"{name}: {url} answers, serving {len(ids)} model(s)" + (f": {', '.join(ids[:5])}" if ids else ""))
+        else:
+            elsewhere = [f for f in found if f["url"] != url]
+            fix = (f"start the server at {url}, or "
+                   + ("run `tuieval add` to add the models of the one running at " + ", ".join(
+                       f"{f['url']}" + (f" ({f['usual']})" if f["usual"] else "") for f in elsewhere)
+                      if elsewhere else engine.url_fix(e.cfg, name)))
+            r.line(BAD if name in used else WARN,
+                   f"{name}: nothing answers at {url}" + ("" if name in used else " (no model uses it yet)"), fix)
+    known = {s.get("url", "").rstrip("/").removesuffix("/v1") for s in e.cfg["servers"].values()}
+    for f in found:
+        if f["url"] not in known:
+            r.line(WARN, f"found a server at {f['url']}" + (f" ({f['usual']}'s usual port)" if f["usual"] else "")
+                   + f" serving {len(f['models'])} model(s), not in models.toml",
+                   "add its models with: tuieval add")
+
+    r.head("Models")
+    if not e.cfg["models"]:
+        r.line(BAD, "no models yet", "tuieval add <GGUF path or model id>, or press a in the TUI")
+    for m in e.cfg["models"]:
+        server = e.cfg["servers"][m["server"]]
+        label = f"{m['label']} ({m['server']})"
+        if server.get("model_is_path"):
+            path = engine.expand(m["model"])
+            r.line(OK if os.path.exists(path) else BAD, f"{label}: {path}",
+                   "" if os.path.exists(path) else "the file is gone; fix `model` in models.toml or tuieval remove it")
+        elif m["server"] in answering:
+            ids = answering[m["server"]]
+            if ids is None:
+                r.line(BAD, f"{label}: its server isn't running", "see Servers above")
+            elif engine.serves(ids, m["served_name"]):
+                r.line(OK, f"{label}: served")
+            else:
+                r.line(BAD, f"{label}: the server doesn't list {m['served_name']!r}",
+                       f"load it in the server, or use one of: {', '.join(ids[:8]) or 'none listed'}")
+        else:
+            r.line(OK, label)
+
+    print("\n" + ("Ready: open the TUI with `tuieval`, or run `tuieval run --tier smoke`." if not r.bad
+                  else f"{r.bad} problem(s) to fix before a run (✗ above)."))
+    return r.bad
+
+
+def cmd_doctor(argv):
+    p = argparse.ArgumentParser(prog="tuieval doctor", description=__doc__,
+                                formatter_class=argparse.RawDescriptionHelpFormatter)
+    p.add_argument("--models", default=None, help="default: the workspace's models.toml")
+    a = p.parse_args(argv)
+    sys.exit(1 if check(a.models) else 0)

tuieval/engine.py

@@ -73,10 +73,26 @@ def slug(model):
     return re.sub(r"[^a-z0-9._-]+", "-", name.lower()).strip("-.")
 
 
+BUILTIN_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "builtin.toml")
+
+
+def builtin_config():
+    with open(BUILTIN_PATH, "rb") as f:
+        return tomllib.load(f)
+
+
 def load_config(path):
-    """Read models.toml and fill in each model's defaults."""
+    """Read models.toml, fill in the built-ins it leaves out (builtin.toml) and each model's defaults.
+    [defaults] are filled key by key; [sampling] only when models.toml has none (sampling is part of
+    every result's fingerprint, so a workspace's own keys are never added to); servers by name."""
     with open(path, "rb") as f:
         cfg = tomllib.load(f)
+    builtin = builtin_config()
+    cfg["defaults"] = {**builtin["defaults"], **cfg.get("defaults", {})}
+    cfg.setdefault("sampling", builtin["sampling"])
+    own = cfg.get("servers", {})
+    cfg["servers"] = {**builtin["servers"], **own}
+    cfg["builtin_servers"] = sorted(set(builtin["servers"]) - set(own))   # not defined in models.toml
     cfg.setdefault("models", [])
     seen = set()
     for m in cfg["models"]:
@@ -219,6 +235,8 @@ def infer_server(cfg, model):
     is_file = model.lower().endswith(".gguf")
     choices = [n for n, s in cfg["servers"].items()
                if bool(s.get("model_is_path")) == is_file and not s.get("any_model")]
+    own = [n for n in choices if n not in cfg.get("builtin_servers", ())]   # the workspace's own come first
+    choices = own or choices
     return choices[0] if len(choices) == 1 else None
 
 
@@ -512,6 +530,115 @@ def server_error(log, start):
         return line.split("]: ", 1)[-1]
     return line.removeprefix("error: ")
 
+def missing_program(cmd, cwd=None):
+    """The program a server command runs, if it isn't installed (not on PATH, or no such file); else None."""
+    exe = next((a for a in cmd if a != "env" and not re.match(r"^[A-Za-z_][A-Za-z0-9_]*=", a)), None)
+    if not exe:
+        return None
+    if "/" in exe:
+        return None if os.path.isfile(os.path.join(cwd or "", expand(exe))) else exe
+    return None if shutil.which(exe) else exe
+
+
+def install_hint(program):
+    """How to get a server program, for the ones tuieval's templates use."""
+    if os.path.basename(program) == "llama-server":
+        return ("install llama.cpp (macOS: brew install llama.cpp; others: a release from "
+                "https://github.com/ggml-org/llama.cpp/releases) so llama-server is on PATH")
+    return "install it, or fix the server's cmd in models.toml"
+
+
+def probe_models(base_url, timeout=2.0, headers=None):
+    """The model ids an OpenAI-compatible server lists at base_url, or None when nothing answers."""
+    req = urllib.request.Request(base_url.rstrip("/").removesuffix("/v1") + "/v1/models", headers=headers or {})
+    try:
+        with urllib.request.urlopen(req, timeout=timeout) as r:
+            return [str(x.get("id", "")) for x in json.load(r).get("data", []) if isinstance(x, dict)]
+    except (OSError, ValueError, AttributeError):
+        return None
+
+
+# Where local OpenAI-compatible servers usually listen. models.toml [defaults] detect_ports
+# replaces the list (e.g. a server on a custom port, or [] to never look).
+DETECT_PORTS = {1234: "LM Studio", 11434: "Ollama", 8080: "llama-server", 8000: "vLLM"}
+
+
+SERVER_NAMES = {1234: "lmstudio", 11434: "ollama", 8080: "llamacpp", 8000: "vllm"}   # for new [servers.*] entries
+
+
+def detect_ports(cfg=None):
+    """Ports to look for running servers on: $TUIEVAL_DETECT_PORTS (comma-separated, empty = none),
+    else models.toml [defaults] detect_ports, else the usual ones."""
+    env = os.environ.get("TUIEVAL_DETECT_PORTS")
+    if env is not None:
+        return [int(p) for p in env.replace(" ", "").split(",") if p]
+    return [int(p) for p in ((cfg or {}).get("defaults") or {}).get("detect_ports", list(DETECT_PORTS))]
+
+
+def _norm_url(url):
+    return (url or "").rstrip("/").removesuffix("/v1").replace("localhost", "127.0.0.1")
+
+
+def server_for_url(cfg, url):
+    """The models.toml server already pointing at url (a server tuieval doesn't start), or None."""
+    return next((n for n, s in cfg["servers"].items() if not s.get("cmd") and _norm_url(s.get("url")) == _norm_url(url)),
+                None)
+
+
+def ensure_server(models_path, url, port=None):
+    """The name of a models.toml server for an already-running server at url, adding a
+    [servers.<name>] block when there's none (named after the app that usually uses the port)."""
+    cfg = load_config(models_path)
+    name = server_for_url(cfg, url)
+    if name:
+        return name
+    base = SERVER_NAMES.get(port, f"local-{port}" if port else "local-server")
+    name = base if base not in cfg["servers"] else f"{base}-{port}"
+    with open(models_path) as f:
+        original = f.read()
+    with open(models_path, "w") as f:
+        f.write(original.rstrip("\n") + f'\n\n[servers.{name}]\nurl = "{_norm_url(url)}"\n')
+    try:
+        load_config(models_path)
+    except BaseException:
+        with open(models_path, "w") as f:
+            f.write(original)
+        raise
+    return name
+
+
+def url_fix(cfg, name):
+    """How to point a server at another URL in models.toml (a built-in one has no section there yet)."""
+    if name in cfg.get("builtin_servers", ()):
+        return f'add [servers.{name}] with url = "http://127.0.0.1:<port>" to models.toml'
+    return f"fix `url` under [servers.{name}] in models.toml"
+
+
+def find_running(cfg, model):
+    """Running servers (detect_servers) that list this model id, best match first."""
+    found = detect_servers(detect_ports(cfg))
+    exact = [f for f in found if model in f["models"]]
+    return exact or [f for f in found if serves(f["models"], model)]
+
+
+def detect_servers(ports, timeout=1.0):
+    """[{url, port, usual, models}] for each local port with an OpenAI-compatible server answering.
+    usual: the app that normally uses that port (a hint, not a check)."""
+    found = []
+    for port in ports:
+        url = f"http://127.0.0.1:{port}"
+        ids = probe_models(url, timeout)
+        if ids is not None:
+            found.append({"url": url, "port": port, "usual": DETECT_PORTS.get(port), "models": ids})
+    return found
+
+
+def serves(ids, name):
+    """Does a server listing these model ids serve this model (names match loosely: aliases, paths)?"""
+    want = name.lower()
+    return any(want == i.lower() or want in i.lower() or (i and i.lower() in want) for i in ids)
+
+
 def pack_min_context(pack):
     """Rough context a pack needs: its longest prompt (~4 chars/token) plus room to answer."""
     return max(len(t["input"]) for t in pack.tests) // 4 + 4096
@@ -622,6 +749,15 @@ class Engine:
                 raise ConfigError(f"couldn't read {name}'s model list ({e})") from None
         return self._catalogues[name]
 
+    def has_results(self):
+        """Has any model run anything yet (smoke runs included)?"""
+        for base in (self.results_dir, os.path.join(self.results_dir, "smoke")):
+            for label in os.listdir(base) if os.path.isdir(base) else []:
+                d = os.path.join(base, label)
+                if os.path.isdir(d) and any(f.endswith(".json") for f in os.listdir(d)):
+                    return True
+        return False
+
     def _add_remote_models(self):
         """Add the models named at run time this session, and ones that have results from an
         earlier session (their result files record the model), to cfg["models"]."""
@@ -1407,6 +1543,9 @@ class Engine:
                     raise ModelFailed(f"mmproj file not found: {expand(m['mmproj'])}")
                 if cwd and not os.path.isdir(cwd):
                     raise ModelFailed(f"server directory not found: {cwd}")
+                missing = missing_program(cmd, cwd)
+                if missing:
+                    raise ModelFailed(f"{missing} isn't installed: {install_hint(missing)}")
                 if port_in_use(port):
                     raise ModelFailed(f"port {port} is already in use; stop the server running there")
                 os.makedirs(os.path.join(self.log_dir, "server"), exist_ok=True)
@@ -1431,7 +1570,7 @@ class Engine:
                 key_env = server.get("api_key_env")
                 if key_env and not os.environ.get(key_env):
                     raise ModelFailed(f"{key_env} is not set; export it in the shell you start tuieval from")
-                where = f"already running at {base_url}"
+                where = f"using the server at {base_url}"
                 if server.get("pin_endpoint"):
                     ep = self.endpoint(m)
                     if not ep or not ep.get("tag"):
@@ -1441,6 +1580,14 @@ class Engine:
                 self.emit("model_loading", label=m["label"], command=[f"({where})"], log="")
                 t0 = time.time()
             deadline = time.time() + defaults["ready_timeout_s"]
+            if srv is None and not server.get("health") and probe_models(base_url, 5, self.request_headers(m)) is None:
+                # a server we don't start: if nothing answers now, waiting won't help
+                time.sleep(3)
+                if probe_models(base_url, 5, self.request_headers(m)) is None:
+                    raise ModelFailed(
+                        f"nothing is answering at {base_url}. Start that server, or {url_fix(self.cfg, m['server'])} "
+                        "(common local ports: LM Studio 1234, Ollama 11434, llama-server 8080, vLLM 8000); "
+                        "`tuieval doctor` checks them all")
             while True:
                 self._check()
                 if srv is not None and srv.poll() is not None:
@@ -1462,9 +1609,8 @@ class Engine:
                     raise ModelFailed(f"server not ready after {defaults['ready_timeout_s']}s")
                 time.sleep(0.5)
             info["load_s"] = time.time() - t0 if srv is not None else None
-            want = m["served_name"].lower()
             # A health-checked server we started ourselves serves exactly the model we gave it.
-            if not (server.get("health") and srv is not None) and not any(want == i.lower() or want in i.lower() or (i and i.lower() in want) for i in ids):
+            if not (server.get("health") and srv is not None) and not serves(ids, m["served_name"]):
                 raise ModelFailed(f"server reports {ids}, expected {m['served_name']!r}; "
                                   "not running to avoid mislabelled results")
             if log and server.get("log_facts"):

tuieval/graders/code.py

@@ -53,6 +53,7 @@ def _err(e):
     return last[:200]
 try:
     exec(compile(CANDIDATE, "candidate.py", "exec"), ns)
+    ns = dict(ns)  # tests get a copy, so their imports can't rebind the candidate's globals
     exec(compile(SETUP, "setup.py", "exec"), ns)
     setup_error = None
 except BaseException as e:

tuieval/onboard.py

@@ -0,0 +1,132 @@
+"""Getting started: find the model servers already running on this machine, add their models,
+and (tuieval init in a terminal) walk a new workspace to its first Smoke check."""
+import os
+import sys
+
+from . import engine
+from . import workspace
+
+
+def ask(question, default="", yes=False):
+    """input() with a default; yes (or no terminal) takes the default without asking."""
+    if yes or not sys.stdin.isatty():
+        return default
+    try:
+        answer = input(f"{question} ").strip()
+    except EOFError:
+        return default
+    return answer or default
+
+
+def unregistered(cfg, found):
+    """[(model id, detected server)] for models on running servers that models.toml doesn't have yet."""
+    out = []
+    for f in found:
+        name = engine.server_for_url(cfg, f["url"])
+        have = {m["served_name"] for m in cfg["models"] if m["server"] == name}
+        out += [(mid, f) for mid in f["models"] if mid and mid not in have]
+    return out
+
+
+def where(f):
+    return f["url"] + (f" ({f['usual']})" if f["usual"] else "")
+
+
+def no_servers_help(cfg):
+    """What to do when nothing is running: the three ways to get a model."""
+    llama = next((s for s in cfg["servers"].values() if s.get("model_is_path") and s.get("cmd")), None)
+    missing = engine.missing_program([str(a) for a in llama["cmd"]]) if llama else "llama-server"
+    key = next((s.get("api_key_env") for s in cfg["servers"].values() if s.get("any_model")), None)
+    ports = ", ".join(map(str, engine.detect_ports(cfg))) or "none (detect_ports is empty)"
+    return [
+        f"No model server is running (looked on ports {ports}). Three ways to get a model:",
+        "  • An app that serves models (LM Studio, Ollama, vLLM): start its server, load a model, then run",
+        "    tuieval add   (it finds the running server and lists its models)",
+        "  • A GGUF file: tuieval add ~/path/to/model.gguf   (tuieval starts llama-server for it; "
+        + ("installed ✓)" if not missing else f"{engine.install_hint(missing)})"),
+        "  • A hosted model on OpenRouter: tuieval run --only openrouter:<model id>   "
+        + (f"({key} is set ✓)" if key and os.environ.get(key) else f"(export {key or 'OPENROUTER_API_KEY'} first)"),
+    ]
+
+
+def add_found(models_path, picks):
+    """Add (model id, detected server) picks to models.toml; returns the labels added."""
+    labels = []
+    for mid, f in picks:
+        try:
+            server = engine.ensure_server(models_path, f["url"], f["port"])
+            label, _ = engine.add_model(models_path, mid, server)
+            labels.append(label)
+            print(f"  added {label}  ({mid} on {where(f)}, server [servers.{server}])")
+        except engine.ConfigError as e:
+            print(f"  not added: {mid}: {e}")
+    return labels
+
+
+def choose(items, yes=False, default="all"):
+    """Numbered pick from items (strings): returns the chosen indexes. Accepts 'all', 'none' or '1,3'."""
+    for i, text in enumerate(items, 1):
+        print(f"  {i}) {text}")
+    answer = ask(f"Add which? numbers like 1,3 · all · none [{default}]:", default, yes).lower()
+    if answer in ("all", "a", "y", "yes"):
+        return list(range(len(items)))
+    if answer in ("none", "n", "no", ""):
+        return []
+    picked = []
+    for part in answer.replace(" ", "").split(","):
+        if part.isdigit() and 1 <= int(part) <= len(items):
+            picked.append(int(part) - 1)
+    return picked
+
+
+def add_detected(models_path, yes=False):
+    """tuieval add without a model: list what's running and add the chosen models. Returns labels."""
+    cfg = engine.load_config(models_path)
+    found = engine.detect_servers(engine.detect_ports(cfg))
+    if not found:
+        print("\n".join(no_servers_help(cfg)))
+        return []
+    items = unregistered(cfg, found)
+    if not items:
+        print("Every model on the running servers is already added: " + ", ".join(where(f) for f in found))
+        return []
+    print(f"Models on running servers ({len(items)}):")
+    picks = choose([f"{mid}  on {where(f)}" for mid, f in items], yes,
+                   default="all" if len(items) <= 3 else "none")
+    return add_found(models_path, [items[i] for i in picks])
+
+
+def guided_init(folder, yes=False):
+    """After `tuieval init` in a terminal: add running models, offer a starter pack, run Smoke."""
+    from . import scaffold
+    os.environ[workspace.ENV] = folder
+    models_path = os.path.join(folder, "models.toml")
+    cfg = engine.load_config(models_path)
+    labels = []
+    if not cfg["models"]:
+        print("\nLooking for model servers running on this machine…")
+        labels = add_detected(models_path, yes)
+    cfg = engine.load_config(models_path)
+    packs_dir = os.path.join(folder, "packs")
+    if not any(os.path.isfile(os.path.join(packs_dir, d, "pack.toml")) for d in os.listdir(packs_dir)):
+        if ask("\nCreate a starter pack of example questions to try? [Y/n]", "y", yes).lower().startswith("y"):
+            scaffold.new_pack("starter", packs_dir=packs_dir)
+            print("  created packs/starter/ (example questions: replace them with your own later)")
+    ready = [m["label"] for m in cfg["models"]]
+    has_pack = any(os.path.isfile(os.path.join(packs_dir, d, "pack.toml")) for d in os.listdir(packs_dir))
+    shown = os.path.relpath(folder) if folder.startswith(os.getcwd()) else folder
+    cd = f"cd {shown} && " if os.path.abspath(folder) != os.getcwd() else ""
+    if ready and has_pack and ask(f"\nRun a Smoke check now (3 questions per pack, {', '.join(ready)})? [Y/n]",
+                                  "y", yes).lower().startswith("y"):
+        from . import run_evals
+        try:
+            run_evals.main(["--tier", "smoke", "--brief", "--only", ",".join(ready)])
+        except SystemExit as e:
+            if e.code not in (None, 0):
+                print(e.code if isinstance(e.code, str) else "the Smoke check didn't finish")
+        print(f"\nNext: {cd}tuieval   (opens the TUI: r shows results, ? explains any screen)")
+    elif not ready:
+        print(f"\nNext: add a model (see above), then {cd}tuieval")
+    else:
+        print(f"\nNext: {cd}tuieval   (opens the TUI; ? explains any screen)")
+    return labels

tuieval/packs.py

@@ -177,9 +177,15 @@ def load_pack(path):
     for tf in test_files:
         tpath = os.path.join(path, tf)
         for i, t in enumerate(_read_tests(tpath)):
-            if not isinstance(t, dict) or "input" not in t:
-                raise PackError(f"{tpath}: test #{i + 1} has no `input`")
+            if not isinstance(t, dict) or ("input" not in t and "input_file" not in t):
+                raise PackError(f"{tpath}: test #{i + 1} has no `input` (or `input_file`)")
             t = dict(t)
+            if "input" not in t:   # the prompt lives in its own file, relative to the pack
+                src = os.path.normpath(os.path.join(path, str(t["input_file"])))
+                if not src.startswith(os.path.normpath(path) + os.sep) or not os.path.isfile(src):
+                    raise PackError(f"{tpath}: test #{i + 1}: input_file {t['input_file']!r} isn't a file in the pack")
+                with open(src) as f:
+                    t["input"] = f.read().strip()
             t.setdefault("description", str(t.get("id") or t["input"])[:80])
             t.setdefault("id", _slug(t["description"]) or f"{tf}-{i + 1}")
             if t["id"] in seen: