tuieval 0.2.0.dev68__tar.gz → 0.2.0.dev72__tar.gz

{tuieval-0.2.0.dev68 → tuieval-0.2.0.dev72}/CHANGELOG.md

@@ -2,6 +2,8 @@
 
 ## Unreleased
 
+- Retest: the setup checkbox "Rerun existing results" is now "Retest (ask again, even if already answered)", and `tuieval run --retest` is the same (`--force` still works).
+- `tuieval logs` follows the server log of the model running now and moves on with the run; `logs live` shows each answer as it finishes, `--list` every log.
 - Results tables sort by a click on a column name; a second click reverses. Numbers, times and levels sort by value; empty cells stay last.
 - Nvidia and AMD GPUs on Linux: VRAM from `nvidia-smi`, `amd-smi` or `rocm-smi` sizes models (`gpu_memory_gb` overrides it), runs record peak VRAM, and `tuieval doctor` shows the GPU.
 - Results has four tabs: PTA index, Production readiness, Per question and Failures. The PTA table adds tok/s, TTFT and memory; Per question can show only where models disagree; the settings warning is one line. Scorecard, pairwise, history and test quality stay in `tuieval compare`, `history` and `items`.

{tuieval-0.2.0.dev68 → tuieval-0.2.0.dev72}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: tuieval
-Version: 0.2.0.dev68
+Version: 0.2.0.dev72
 Summary: Evaluate local and frontier LLMs on your own questions: accuracy, speed, tokens and PASS/FAIL verdicts, in the terminal.
 Project-URL: Homepage, https://github.com/ashe-wb/tuieval
 Project-URL: Issues, https://github.com/ashe-wb/tuieval/issues
@@ -162,6 +162,8 @@ tuieval history                               # every verdict change over time
 tuieval compare --speed --pairwise --failures # scorecards
 tuieval selftest                              # check every test's reference and wrong answers
 tuieval items                                 # tests that don't separate models or look broken
+tuieval logs                                  # follow the server log of the model running now
+tuieval logs live                             # each answer as it finishes (--list: every log)
 tuieval capture logs/live/<file> --pack X     # turn a real failure into a new test
 tuieval regrade                               # re-score stored answers after changing a grader
 tuieval regrade --only <model> --packs X      # just one model's results (and packs)

{tuieval-0.2.0.dev68 → tuieval-0.2.0.dev72}/README.md

@@ -136,6 +136,8 @@ tuieval history                               # every verdict change over time
 tuieval compare --speed --pairwise --failures # scorecards
 tuieval selftest                              # check every test's reference and wrong answers
 tuieval items                                 # tests that don't separate models or look broken
+tuieval logs                                  # follow the server log of the model running now
+tuieval logs live                             # each answer as it finishes (--list: every log)
 tuieval capture logs/live/<file> --pack X     # turn a real failure into a new test
 tuieval regrade                               # re-score stored answers after changing a grader
 tuieval regrade --only <model> --packs X      # just one model's results (and packs)

{tuieval-0.2.0.dev68 → tuieval-0.2.0.dev72}/src/tuieval/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.2.0.dev68'
-__version_tuple__ = version_tuple = (0, 2, 0, 'dev68')
+__version__ = version = '0.2.0.dev72'
+__version_tuple__ = version_tuple = (0, 2, 0, 'dev72')
 
 __commit_id__ = commit_id = None

{tuieval-0.2.0.dev68 → tuieval-0.2.0.dev72}/src/tuieval/cli.py

@@ -18,6 +18,7 @@ Day to day
   tuieval report              the verdicts with their evidence, as a markdown file
   tuieval history             every verdict change, newest first
   tuieval items               tests that don't separate models or look broken
+  tuieval logs                follow the server log of the model running now (live: its answers)
   tuieval capture LOG --pack  turn a real failure (logs/live/*.txt) into a new test
   tuieval regrade             re-score stored answers after changing a grader (--only, --packs)
   tuieval list                the models tuieval knows, hidden ones separately (--packs: the packs)
@@ -85,6 +86,9 @@ def main(argv=None):
     if cmd == "watch":
         from . import watch_proxy
         return watch_proxy.main(rest)
+    if cmd == "logs":
+        from . import logs
+        return logs.main(rest)
     from . import run_evals
     if cmd in run_evals.COMMANDS:
         return run_evals.COMMANDS[cmd](rest)

tuieval-0.2.0.dev72/src/tuieval/logs.py

@@ -0,0 +1,220 @@
+"""tuieval logs: follow the server log of the model running now, without naming it.
+
+    tuieval logs              the newest server log, then every log that grows (a header says
+                              which model each part is from), so it moves on with the run
+    tuieval logs live         each answer (reasoning and answer, from logs/live/) as it finishes
+    tuieval logs --list       every model's server log, newest first
+    tuieval logs MODEL        one model's log (any unambiguous part of its label)
+
+Read-only: it never touches servers or results. Only servers tuieval starts write a log here;
+OpenRouter and servers that were already running don't.
+"""
+import argparse
+import datetime
+import os
+import re
+import sys
+import time
+
+from . import workspace
+
+POLL_S = 0.5
+
+
+def server_dir():
+    return workspace.path("logs", "server")
+
+
+def live_dir():
+    return workspace.path("logs", "live")
+
+
+def _files(d, ext):
+    """{path: (size, mtime)} for the files in d with this extension (none if d doesn't exist)."""
+    out = {}
+    try:
+        names = os.listdir(d)
+    except OSError:
+        return out
+    for n in names:
+        if n.endswith(ext):
+            p = os.path.join(d, n)
+            try:
+                st = os.stat(p)
+            except OSError:
+                continue
+            out[p] = (st.st_size, st.st_mtime)
+    return out
+
+
+def _label(path):
+    return os.path.basename(path)[:-len(".log")]
+
+
+def _tail_offset(path, lines):
+    """Where the last `lines` lines of a file start."""
+    with open(path, "rb") as f:
+        f.seek(0, os.SEEK_END)
+        end = pos = f.tell()
+        found, chunk = 0, 8192
+        while pos > 0 and found <= lines:
+            pos = max(0, pos - chunk)
+            f.seek(pos)
+            found = f.read(end - pos).count(b"\n")
+        if pos == 0 and found <= lines:
+            return 0
+        f.seek(pos)
+        data = f.read(end - pos)
+    cut = len(data)
+    for _ in range(lines + 1):
+        cut = data.rfind(b"\n", 0, cut)
+    return pos + cut + 1
+
+
+class Follower:
+    """Follows server logs. `poll()` returns the text written since the last poll, with a
+    `── label ──` header whenever it comes from another log than the text before it.
+    only: follow just this log. Otherwise every log in the folder, including new ones."""
+
+    def __init__(self, folder, lines=40, only=None):
+        self.folder, self.only = folder, only
+        self.offsets = {p: size for p, (size, _) in _files(folder, ".log").items()}
+        self.last = None        # the log the previous text came from
+        self.held = {}          # path -> size at the last poll, for a line still being written
+        start = only or max(self.offsets, key=lambda p: os.path.getmtime(p), default=None)
+        if start and start in self.offsets:
+            self.offsets[start] = _tail_offset(start, lines)
+
+    def poll(self):
+        out = []
+        files = _files(self.folder, ".log")
+        if self.only:
+            files = {p: v for p, v in files.items() if p == self.only}
+        for path, (size, _) in sorted(files.items(), key=lambda kv: kv[1][1]):   # oldest change first
+            off = self.offsets.setdefault(path, 0)
+            if size < off:          # truncated or replaced: read it from the start
+                off = 0
+            if size == off:
+                continue
+            with open(path, "rb") as f:
+                f.seek(off)
+                data = f.read(size - off)
+            cut = data.rfind(b"\n") + 1
+            if not cut:             # no complete line yet: wait one poll for the rest of it
+                if self.held.get(path) != size:
+                    self.held[path] = size
+                    continue
+                cut = len(data)
+            self.held.pop(path, None)
+            self.offsets[path] = off + cut
+            if path != self.last:
+                out.append(f"── {_label(path)} ──\n")
+                self.last = path
+            out.append(data[:cut].decode("utf-8", "replace"))
+        return "".join(out)
+
+
+FOOTER = re.compile(r"^=== .*finish=.* ===$", re.M)
+
+
+class LiveFollower:
+    """New answers in logs/live/ as they're written: each one from its reasoning on (the prompt is
+    in the file), under a header with the file name."""
+
+    def __init__(self, folder, last=1):
+        self.folder = folder
+        files = sorted(_files(folder, ".txt").items(), key=lambda kv: kv[1][1])
+        self.seen = {p for p, _ in files[:max(0, len(files) - last)]}
+
+    def poll(self):
+        out = []
+        for path, (_, mtime) in sorted(_files(self.folder, ".txt").items(), key=lambda kv: kv[1][1]):
+            if path in self.seen:
+                continue
+            try:
+                with open(path, errors="replace") as f:
+                    text = f.read()
+            except OSError:
+                continue
+            if not FOOTER.search(text) and time.time() - mtime < 2:   # still being written
+                continue
+            self.seen.add(path)
+            i = text.find("=== REASONING ===")
+            out.append(f"── {os.path.basename(path)[:-4]} ──\n{text[i:] if i >= 0 else text}\n")
+        return "".join(out)
+
+
+def _ago(mtime):
+    s = max(0, int(time.time() - mtime))
+    if s < 60:
+        return f"{s}s ago"
+    if s < 3600:
+        return f"{s // 60} min ago"
+    if s < 86400:
+        return f"{s // 3600} h ago"
+    return datetime.datetime.fromtimestamp(mtime).strftime("%Y-%m-%d %H:%M")
+
+
+def _size(n):
+    for unit in ("B", "KB", "MB", "GB"):
+        if n < 1024 or unit == "GB":
+            return f"{n:.0f} {unit}" if unit == "B" else f"{n:.1f} {unit}"
+        n /= 1024
+
+
+def list_logs(folder):
+    files = sorted(_files(folder, ".log").items(), key=lambda kv: -kv[1][1])
+    if not files:
+        return f"No server logs yet in {os.path.relpath(folder)}/."
+    w = max(len(_label(p)) for p, _ in files)
+    return "\n".join(f"{_label(p):<{w}}  {_size(size):>9}  {_ago(mtime)}" for p, (size, mtime) in files)
+
+
+def find_log(folder, name):
+    """The log whose label is `name`, or the only one containing it. Raises SystemExit otherwise."""
+    labels = {_label(p): p for p in _files(folder, ".log")}
+    if name in labels:
+        return labels[name]
+    hits = sorted(l for l in labels if name.lower() in l.lower())
+    if len(hits) == 1:
+        return labels[hits[0]]
+    if not hits:
+        sys.exit(f"No server log matches {name!r}; tuieval logs --list shows them.")
+    sys.exit(f"{name!r} matches {len(hits)} logs; say which:\n  " + "\n  ".join(hits))
+
+
+def follow(follower, waiting):
+    """Print what the follower finds until ctrl-c."""
+    said = False
+    try:
+        while True:
+            text = follower.poll()
+            if text:
+                sys.stdout.write(text)
+                sys.stdout.flush()
+                said = True
+            elif not said:
+                print(waiting, flush=True)
+                said = True
+            time.sleep(POLL_S)
+    except KeyboardInterrupt:
+        return 0
+
+
+def main(argv):
+    p = argparse.ArgumentParser(prog="tuieval logs", description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter,
+                                epilog="ctrl-c stops following (a run going on isn't affected).")
+    p.add_argument("what", nargs="?", help="'live' for answers, or a model (any unambiguous part of its label)")
+    p.add_argument("--list", action="store_true", help="every model's server log, newest first")
+    p.add_argument("-n", "--lines", type=int, default=40, help="lines shown before following (default 40)")
+    a = p.parse_args(argv)
+    if a.list:
+        print(list_logs(server_dir()))
+        return 0
+    if a.what == "live":
+        return follow(LiveFollower(live_dir()),
+                      f"Waiting for answers in {os.path.relpath(live_dir())}/ (ctrl-c stops)")
+    only = find_log(server_dir(), a.what) if a.what else None
+    return follow(Follower(server_dir(), a.lines, only),
+                  f"Waiting for {'that model' if only else 'a server'} to log in "
+                  f"{os.path.relpath(server_dir())}/ (ctrl-c stops)")

{tuieval-0.2.0.dev68 → tuieval-0.2.0.dev72}/src/tuieval/run_evals.py

@@ -11,6 +11,7 @@
     tuieval run --only my-model --packs my-pack,other-pack
     tuieval run --tags moe --packs my-pack        # models tagged "moe"
     tuieval run --packs my-pack --tests a,b       # only these tests of a pack (or --tests my-pack:a,other:b)
+    tuieval run --only my-model --packs my-pack --tests a --retest   # ask a question again
     tuieval run --preset nightly                  # a selection saved in the TUI (presets.toml)
     tuieval run --tier smoke --only my-model      # 3 tests per pack, 1 repeat: check paths and flags
     tuieval run --tier smoke --only openrouter:qwen/qwen3-32b   # any OpenRouter model, no models.toml edit
@@ -742,7 +743,8 @@ def main(argv=None):
     p.add_argument("--packs", help="comma-separated packs (default: all)")
     p.add_argument("--tests", help="comma-separated test ids to run instead of the tier's sample: id with one "
                                    "pack in --packs, else pack:id (and --packs defaults to those packs)")
-    p.add_argument("--force", action="store_true", help="rerun packs (or the --tests) that already have current results")
+    p.add_argument("--retest", "--force", action="store_true", dest="force",
+                   help="ask again even if already answered: the packs, or just the --tests (earlier answers go to history/)")
     p.add_argument("--repeat", type=int, help="times each question is asked, any tier (default: 1 for smoke and "
                                                   "screen, the pack's certification count for certify)")
     p.add_argument("--tier", choices=engine.TIERS, default="screen",

{tuieval-0.2.0.dev68 → tuieval-0.2.0.dev72}/src/tuieval/tui.py

@@ -385,6 +385,9 @@ HELP = {
 [b]Models at a time[/b]: how many of the ticked models run side by side (blank: this machine's
 parallel_models in models.toml, default 1). Answers note the models they ran alongside.
 
+[b]Retest[/b]: ask the ticked questions again even if they're already answered (only the tests
+picked with e, if any). Without it, answered questions are skipped. Replaced answers go to history/.
+
 [b]Keys[/b]
   s  start (with a run going: add to it, or queue it)   r  results
   a  add a model (a GGUF, a model id, openrouter:<id>)  m  scan model folders for new GGUFs
@@ -646,7 +649,7 @@ class SetupScreen(Screen):
             yield Input("", id="repeat", type="integer", max_length=2, placeholder="1")
             yield Label("Models at a time")
             yield Input("", id="parallel", type="integer", max_length=1, placeholder="1")
-            yield Checkbox("Rerun existing results", id="force")
+            yield Checkbox("Retest (ask again, even if already answered)", id="force")
         yield Static(id="estimate")
         with Horizontal(id="buttons"):
             yield Button("Start  [s]", id="start", variant="success")
@@ -958,7 +961,7 @@ class SetupScreen(Screen):
         if not run:
             text += "\n[yellow]Nothing selected can run (see skipping).[/yellow]"
         elif rerun:
-            text += "\n[yellow]Everything selected has results; Start runs it again.[/yellow]"
+            text += "\n[yellow]Everything selected has results; Start retests it.[/yellow]"
         resumed = [j for j in run if "done earlier" in j.note]
         if resumed:
             text += f"   [yellow]{len(resumed)} pack(s) continue from earlier results[/yellow]"

{tuieval-0.2.0.dev68 → tuieval-0.2.0.dev72}/tests/test_tuieval.py

@@ -380,6 +380,9 @@ class PickTests(unittest.TestCase):
         hist = os.listdir(os.path.join(self.ws, "results", "m", "history"))
         self.assertTrue(any(f.startswith("apps-") for f in hist), hist)
         self.assertEqual(self.status(), "certified")
+        self.server("oracle")
+        tuieval(self.ws, "run", "--tests", "apps:b", "--retest")                  # --retest = --force
+        self.assertEqual(self.rows(), {"a": True, "b": True, "c": True, "d": True})
 
     def test_cli_errors(self):
         self.server("oracle")
@@ -1705,5 +1708,88 @@ class Remove(unittest.TestCase):
                 self.assertTrue(os.path.exists(os.path.join(ws, "removed", kept, rel)), rel)
 
 
+
+class Logs(unittest.TestCase):
+    """tuieval logs: follows whichever model's server log is being written, without naming it."""
+
+    def test_follows_the_log_being_written(self):
+        from tuieval import logs
+        with tempfile.TemporaryDirectory() as d:
+            a, b = os.path.join(d, "model-a.log"), os.path.join(d, "model-b.log")
+            write(a, "a old\n")
+            write(b, "".join(f"b {i}\n" for i in range(50)))
+            os.utime(a, (time.time() - 60,) * 2)
+            f = logs.Follower(d, lines=3)
+            self.assertEqual(f.poll(), "── model-b ──\nb 47\nb 48\nb 49\n")   # the newest, its last lines
+            self.assertEqual(f.poll(), "")
+            with open(a, "a") as fh:
+                fh.write("a new\n")                                          # the run moved on to model a
+            self.assertEqual(f.poll(), "── model-a ──\na new\n")
+            with open(b, "a") as fh:
+                fh.write("half a li")                                          # a line still being written
+            self.assertEqual(f.poll(), "")
+            self.assertEqual(f.poll(), "── model-b ──\nhalf a li")             # unchanged since: shown anyway
+            write(os.path.join(d, "model-c.log"), "c starts\n")              # a model that never ran before
+            self.assertEqual(f.poll(), "── model-c ──\nc starts\n")
+            write(a, "fresh\n")                                               # replaced: read from the start
+            self.assertEqual(f.poll(), "── model-a ──\nfresh\n")
+            only = logs.Follower(d, lines=1, only=b)
+            self.assertEqual(only.poll(), "── model-b ──\nb 49\n")             # complete lines first
+            self.assertEqual((only.poll(), only.poll()), ("", "half a li"))     # held one poll, as above
+            with open(a, "a") as fh:
+                fh.write("ignored\n")
+            self.assertEqual(only.poll(), "")
+
+    def test_live_answers(self):
+        from tuieval import logs
+        with tempfile.TemporaryDirectory() as d:
+            def answer(name, done=True, age=0):
+                path = os.path.join(d, name + ".txt")
+                write(path, "MODEL: m\nPROMPT: the question\n\n=== REASONING ===\nthink\n\n=== ANSWER ===\n42\n"
+                      + ("\n=== 10 tokens, 1.0s, finish=stop ===\n" if done else ""))
+                os.utime(path, (time.time() - age,) * 2)
+            answer("20260101_100000_m_pack_old", age=20)
+            answer("20260101_100100_m_pack_last", age=10)
+            f = logs.LiveFollower(d)
+            first = f.poll()
+            self.assertIn("── 20260101_100100_m_pack_last ──\n=== REASONING ===", first)   # the latest one only
+            self.assertNotIn("_old", first)
+            self.assertNotIn("PROMPT", first)
+            answer("20260101_100200_m_pack_new", done=False)                  # still being written
+            self.assertEqual(f.poll(), "")
+            answer("20260101_100200_m_pack_new")
+            self.assertIn("=== ANSWER ===\n42", f.poll())
+
+    def test_cli(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            ws = os.path.join(tmp, "ws")
+            tuieval(tmp, "init", ws)
+            self.assertIn("No server logs yet", tuieval(ws, "logs", "--list").stdout)
+            d = os.path.join(ws, "logs", "server")
+            os.makedirs(d, exist_ok=True)
+            for i, name in enumerate(("coder-q4", "coder-q5", "chat")):
+                write(os.path.join(d, name + ".log"), f"{name} line\n")
+                os.utime(os.path.join(d, name + ".log"), (time.time() - 100 + i,) * 2)
+            listed = tuieval(ws, "logs", "--list").stdout.splitlines()
+            self.assertEqual([l.split()[0] for l in listed], ["chat", "coder-q5", "coder-q4"])   # newest first
+            p = tuieval(ws, "logs", "coder", check=False)
+            self.assertNotEqual(p.returncode, 0)
+            self.assertIn("matches 2 logs", p.stderr)
+            self.assertIn("No server log matches", tuieval(ws, "logs", "nothing", check=False).stderr)
+            # following: the newest log, then whichever one grows; ctrl-c stops it cleanly
+            proc = subprocess.Popen([sys.executable, "-m", "tuieval", "logs", "-n", "1"], stdout=subprocess.PIPE,
+                                    text=True, env=dict(os.environ, TUIEVAL_HOME=ws))
+            try:
+                time.sleep(1.5)
+                with open(os.path.join(d, "coder-q4.log"), "a") as fh:
+                    fh.write("q4 server started\n")
+                time.sleep(1.5)
+            finally:
+                proc.send_signal(__import__("signal").SIGINT)
+                out, _ = proc.communicate(timeout=20)
+            self.assertEqual(proc.returncode, 0)
+            self.assertEqual(out, "── chat ──\nchat line\n── coder-q4 ──\nq4 server started\n")
+
+
 if __name__ == "__main__":
     unittest.main()