tuieval 0.2.0.dev68__tar.gz → 0.2.0.dev70__tar.gz

{tuieval-0.2.0.dev68 → tuieval-0.2.0.dev70}/CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## Unreleased
 
+- `tuieval logs` follows the server log of the model running now and moves on with the run; `logs live` shows each answer as it finishes, `--list` every log.
 - Results tables sort by a click on a column name; a second click reverses. Numbers, times and levels sort by value; empty cells stay last.
 - Nvidia and AMD GPUs on Linux: VRAM from `nvidia-smi`, `amd-smi` or `rocm-smi` sizes models (`gpu_memory_gb` overrides it), runs record peak VRAM, and `tuieval doctor` shows the GPU.
 - Results has four tabs: PTA index, Production readiness, Per question and Failures. The PTA table adds tok/s, TTFT and memory; Per question can show only where models disagree; the settings warning is one line. Scorecard, pairwise, history and test quality stay in `tuieval compare`, `history` and `items`.

{tuieval-0.2.0.dev68 → tuieval-0.2.0.dev70}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: tuieval
-Version: 0.2.0.dev68
+Version: 0.2.0.dev70
 Summary: Evaluate local and frontier LLMs on your own questions: accuracy, speed, tokens and PASS/FAIL verdicts, in the terminal.
 Project-URL: Homepage, https://github.com/ashe-wb/tuieval
 Project-URL: Issues, https://github.com/ashe-wb/tuieval/issues
@@ -162,6 +162,8 @@ tuieval history                               # every verdict change over time
 tuieval compare --speed --pairwise --failures # scorecards
 tuieval selftest                              # check every test's reference and wrong answers
 tuieval items                                 # tests that don't separate models or look broken
+tuieval logs                                  # follow the server log of the model running now
+tuieval logs live                             # each answer as it finishes (--list: every log)
 tuieval capture logs/live/<file> --pack X     # turn a real failure into a new test
 tuieval regrade                               # re-score stored answers after changing a grader
 tuieval regrade --only <model> --packs X      # just one model's results (and packs)

{tuieval-0.2.0.dev68 → tuieval-0.2.0.dev70}/README.md

@@ -136,6 +136,8 @@ tuieval history                               # every verdict change over time
 tuieval compare --speed --pairwise --failures # scorecards
 tuieval selftest                              # check every test's reference and wrong answers
 tuieval items                                 # tests that don't separate models or look broken
+tuieval logs                                  # follow the server log of the model running now
+tuieval logs live                             # each answer as it finishes (--list: every log)
 tuieval capture logs/live/<file> --pack X     # turn a real failure into a new test
 tuieval regrade                               # re-score stored answers after changing a grader
 tuieval regrade --only <model> --packs X      # just one model's results (and packs)

{tuieval-0.2.0.dev68 → tuieval-0.2.0.dev70}/src/tuieval/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.2.0.dev68'
-__version_tuple__ = version_tuple = (0, 2, 0, 'dev68')
+__version__ = version = '0.2.0.dev70'
+__version_tuple__ = version_tuple = (0, 2, 0, 'dev70')
 
 __commit_id__ = commit_id = None

{tuieval-0.2.0.dev68 → tuieval-0.2.0.dev70}/src/tuieval/cli.py

@@ -18,6 +18,7 @@ Day to day
   tuieval report              the verdicts with their evidence, as a markdown file
   tuieval history             every verdict change, newest first
   tuieval items               tests that don't separate models or look broken
+  tuieval logs                follow the server log of the model running now (live: its answers)
   tuieval capture LOG --pack  turn a real failure (logs/live/*.txt) into a new test
   tuieval regrade             re-score stored answers after changing a grader (--only, --packs)
   tuieval list                the models tuieval knows, hidden ones separately (--packs: the packs)
@@ -85,6 +86,9 @@ def main(argv=None):
     if cmd == "watch":
         from . import watch_proxy
         return watch_proxy.main(rest)
+    if cmd == "logs":
+        from . import logs
+        return logs.main(rest)
     from . import run_evals
     if cmd in run_evals.COMMANDS:
         return run_evals.COMMANDS[cmd](rest)

tuieval-0.2.0.dev70/src/tuieval/logs.py

@@ -0,0 +1,220 @@
+"""tuieval logs: follow the server log of the model running now, without naming it.
+
+    tuieval logs              the newest server log, then every log that grows (a header says
+                              which model each part is from), so it moves on with the run
+    tuieval logs live         each answer (reasoning and answer, from logs/live/) as it finishes
+    tuieval logs --list       every model's server log, newest first
+    tuieval logs MODEL        one model's log (any unambiguous part of its label)
+
+Read-only: it never touches servers or results. Only servers tuieval starts write a log here;
+OpenRouter and servers that were already running don't.
+"""
+import argparse
+import datetime
+import os
+import re
+import sys
+import time
+
+from . import workspace
+
+POLL_S = 0.5
+
+
+def server_dir():
+    return workspace.path("logs", "server")
+
+
+def live_dir():
+    return workspace.path("logs", "live")
+
+
+def _files(d, ext):
+    """{path: (size, mtime)} for the files in d with this extension (none if d doesn't exist)."""
+    out = {}
+    try:
+        names = os.listdir(d)
+    except OSError:
+        return out
+    for n in names:
+        if n.endswith(ext):
+            p = os.path.join(d, n)
+            try:
+                st = os.stat(p)
+            except OSError:
+                continue
+            out[p] = (st.st_size, st.st_mtime)
+    return out
+
+
+def _label(path):
+    return os.path.basename(path)[:-len(".log")]
+
+
+def _tail_offset(path, lines):
+    """Where the last `lines` lines of a file start."""
+    with open(path, "rb") as f:
+        f.seek(0, os.SEEK_END)
+        end = pos = f.tell()
+        found, chunk = 0, 8192
+        while pos > 0 and found <= lines:
+            pos = max(0, pos - chunk)
+            f.seek(pos)
+            found = f.read(end - pos).count(b"\n")
+        if pos == 0 and found <= lines:
+            return 0
+        f.seek(pos)
+        data = f.read(end - pos)
+    cut = len(data)
+    for _ in range(lines + 1):
+        cut = data.rfind(b"\n", 0, cut)
+    return pos + cut + 1
+
+
+class Follower:
+    """Follows server logs. `poll()` returns the text written since the last poll, with a
+    `── label ──` header whenever it comes from another log than the text before it.
+    only: follow just this log. Otherwise every log in the folder, including new ones."""
+
+    def __init__(self, folder, lines=40, only=None):
+        self.folder, self.only = folder, only
+        self.offsets = {p: size for p, (size, _) in _files(folder, ".log").items()}
+        self.last = None        # the log the previous text came from
+        self.held = {}          # path -> size at the last poll, for a line still being written
+        start = only or max(self.offsets, key=lambda p: os.path.getmtime(p), default=None)
+        if start and start in self.offsets:
+            self.offsets[start] = _tail_offset(start, lines)
+
+    def poll(self):
+        out = []
+        files = _files(self.folder, ".log")
+        if self.only:
+            files = {p: v for p, v in files.items() if p == self.only}
+        for path, (size, _) in sorted(files.items(), key=lambda kv: kv[1][1]):   # oldest change first
+            off = self.offsets.setdefault(path, 0)
+            if size < off:          # truncated or replaced: read it from the start
+                off = 0
+            if size == off:
+                continue
+            with open(path, "rb") as f:
+                f.seek(off)
+                data = f.read(size - off)
+            cut = data.rfind(b"\n") + 1
+            if not cut:             # no complete line yet: wait one poll for the rest of it
+                if self.held.get(path) != size:
+                    self.held[path] = size
+                    continue
+                cut = len(data)
+            self.held.pop(path, None)
+            self.offsets[path] = off + cut
+            if path != self.last:
+                out.append(f"── {_label(path)} ──\n")
+                self.last = path
+            out.append(data[:cut].decode("utf-8", "replace"))
+        return "".join(out)
+
+
+FOOTER = re.compile(r"^=== .*finish=.* ===$", re.M)
+
+
+class LiveFollower:
+    """New answers in logs/live/ as they're written: each one from its reasoning on (the prompt is
+    in the file), under a header with the file name."""
+
+    def __init__(self, folder, last=1):
+        self.folder = folder
+        files = sorted(_files(folder, ".txt").items(), key=lambda kv: kv[1][1])
+        self.seen = {p for p, _ in files[:max(0, len(files) - last)]}
+
+    def poll(self):
+        out = []
+        for path, (_, mtime) in sorted(_files(self.folder, ".txt").items(), key=lambda kv: kv[1][1]):
+            if path in self.seen:
+                continue
+            try:
+                with open(path, errors="replace") as f:
+                    text = f.read()
+            except OSError:
+                continue
+            if not FOOTER.search(text) and time.time() - mtime < 2:   # still being written
+                continue
+            self.seen.add(path)
+            i = text.find("=== REASONING ===")
+            out.append(f"── {os.path.basename(path)[:-4]} ──\n{text[i:] if i >= 0 else text}\n")
+        return "".join(out)
+
+
+def _ago(mtime):
+    s = max(0, int(time.time() - mtime))
+    if s < 60:
+        return f"{s}s ago"
+    if s < 3600:
+        return f"{s // 60} min ago"
+    if s < 86400:
+        return f"{s // 3600} h ago"
+    return datetime.datetime.fromtimestamp(mtime).strftime("%Y-%m-%d %H:%M")
+
+
+def _size(n):
+    for unit in ("B", "KB", "MB", "GB"):
+        if n < 1024 or unit == "GB":
+            return f"{n:.0f} {unit}" if unit == "B" else f"{n:.1f} {unit}"
+        n /= 1024
+
+
+def list_logs(folder):
+    files = sorted(_files(folder, ".log").items(), key=lambda kv: -kv[1][1])
+    if not files:
+        return f"No server logs yet in {os.path.relpath(folder)}/."
+    w = max(len(_label(p)) for p, _ in files)
+    return "\n".join(f"{_label(p):<{w}}  {_size(size):>9}  {_ago(mtime)}" for p, (size, mtime) in files)
+
+
+def find_log(folder, name):
+    """The log whose label is `name`, or the only one containing it. Raises SystemExit otherwise."""
+    labels = {_label(p): p for p in _files(folder, ".log")}
+    if name in labels:
+        return labels[name]
+    hits = sorted(l for l in labels if name.lower() in l.lower())
+    if len(hits) == 1:
+        return labels[hits[0]]
+    if not hits:
+        sys.exit(f"No server log matches {name!r}; tuieval logs --list shows them.")
+    sys.exit(f"{name!r} matches {len(hits)} logs; say which:\n  " + "\n  ".join(hits))
+
+
+def follow(follower, waiting):
+    """Print what the follower finds until ctrl-c."""
+    said = False
+    try:
+        while True:
+            text = follower.poll()
+            if text:
+                sys.stdout.write(text)
+                sys.stdout.flush()
+                said = True
+            elif not said:
+                print(waiting, flush=True)
+                said = True
+            time.sleep(POLL_S)
+    except KeyboardInterrupt:
+        return 0
+
+
+def main(argv):
+    p = argparse.ArgumentParser(prog="tuieval logs", description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter,
+                                epilog="ctrl-c stops following (a run going on isn't affected).")
+    p.add_argument("what", nargs="?", help="'live' for answers, or a model (any unambiguous part of its label)")
+    p.add_argument("--list", action="store_true", help="every model's server log, newest first")
+    p.add_argument("-n", "--lines", type=int, default=40, help="lines shown before following (default 40)")
+    a = p.parse_args(argv)
+    if a.list:
+        print(list_logs(server_dir()))
+        return 0
+    if a.what == "live":
+        return follow(LiveFollower(live_dir()),
+                      f"Waiting for answers in {os.path.relpath(live_dir())}/ (ctrl-c stops)")
+    only = find_log(server_dir(), a.what) if a.what else None
+    return follow(Follower(server_dir(), a.lines, only),
+                  f"Waiting for {'that model' if only else 'a server'} to log in "
+                  f"{os.path.relpath(server_dir())}/ (ctrl-c stops)")

{tuieval-0.2.0.dev68 → tuieval-0.2.0.dev70}/tests/test_tuieval.py

@@ -1705,5 +1705,88 @@ class Remove(unittest.TestCase):
                 self.assertTrue(os.path.exists(os.path.join(ws, "removed", kept, rel)), rel)
 
 
+
+class Logs(unittest.TestCase):
+    """tuieval logs: follows whichever model's server log is being written, without naming it."""
+
+    def test_follows_the_log_being_written(self):
+        from tuieval import logs
+        with tempfile.TemporaryDirectory() as d:
+            a, b = os.path.join(d, "model-a.log"), os.path.join(d, "model-b.log")
+            write(a, "a old\n")
+            write(b, "".join(f"b {i}\n" for i in range(50)))
+            os.utime(a, (time.time() - 60,) * 2)
+            f = logs.Follower(d, lines=3)
+            self.assertEqual(f.poll(), "── model-b ──\nb 47\nb 48\nb 49\n")   # the newest, its last lines
+            self.assertEqual(f.poll(), "")
+            with open(a, "a") as fh:
+                fh.write("a new\n")                                          # the run moved on to model a
+            self.assertEqual(f.poll(), "── model-a ──\na new\n")
+            with open(b, "a") as fh:
+                fh.write("half a li")                                          # a line still being written
+            self.assertEqual(f.poll(), "")
+            self.assertEqual(f.poll(), "── model-b ──\nhalf a li")             # unchanged since: shown anyway
+            write(os.path.join(d, "model-c.log"), "c starts\n")              # a model that never ran before
+            self.assertEqual(f.poll(), "── model-c ──\nc starts\n")
+            write(a, "fresh\n")                                               # replaced: read from the start
+            self.assertEqual(f.poll(), "── model-a ──\nfresh\n")
+            only = logs.Follower(d, lines=1, only=b)
+            self.assertEqual(only.poll(), "── model-b ──\nb 49\n")             # complete lines first
+            self.assertEqual((only.poll(), only.poll()), ("", "half a li"))     # held one poll, as above
+            with open(a, "a") as fh:
+                fh.write("ignored\n")
+            self.assertEqual(only.poll(), "")
+
+    def test_live_answers(self):
+        from tuieval import logs
+        with tempfile.TemporaryDirectory() as d:
+            def answer(name, done=True, age=0):
+                path = os.path.join(d, name + ".txt")
+                write(path, "MODEL: m\nPROMPT: the question\n\n=== REASONING ===\nthink\n\n=== ANSWER ===\n42\n"
+                      + ("\n=== 10 tokens, 1.0s, finish=stop ===\n" if done else ""))
+                os.utime(path, (time.time() - age,) * 2)
+            answer("20260101_100000_m_pack_old", age=20)
+            answer("20260101_100100_m_pack_last", age=10)
+            f = logs.LiveFollower(d)
+            first = f.poll()
+            self.assertIn("── 20260101_100100_m_pack_last ──\n=== REASONING ===", first)   # the latest one only
+            self.assertNotIn("_old", first)
+            self.assertNotIn("PROMPT", first)
+            answer("20260101_100200_m_pack_new", done=False)                  # still being written
+            self.assertEqual(f.poll(), "")
+            answer("20260101_100200_m_pack_new")
+            self.assertIn("=== ANSWER ===\n42", f.poll())
+
+    def test_cli(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            ws = os.path.join(tmp, "ws")
+            tuieval(tmp, "init", ws)
+            self.assertIn("No server logs yet", tuieval(ws, "logs", "--list").stdout)
+            d = os.path.join(ws, "logs", "server")
+            os.makedirs(d, exist_ok=True)
+            for i, name in enumerate(("coder-q4", "coder-q5", "chat")):
+                write(os.path.join(d, name + ".log"), f"{name} line\n")
+                os.utime(os.path.join(d, name + ".log"), (time.time() - 100 + i,) * 2)
+            listed = tuieval(ws, "logs", "--list").stdout.splitlines()
+            self.assertEqual([l.split()[0] for l in listed], ["chat", "coder-q5", "coder-q4"])   # newest first
+            p = tuieval(ws, "logs", "coder", check=False)
+            self.assertNotEqual(p.returncode, 0)
+            self.assertIn("matches 2 logs", p.stderr)
+            self.assertIn("No server log matches", tuieval(ws, "logs", "nothing", check=False).stderr)
+            # following: the newest log, then whichever one grows; ctrl-c stops it cleanly
+            proc = subprocess.Popen([sys.executable, "-m", "tuieval", "logs", "-n", "1"], stdout=subprocess.PIPE,
+                                    text=True, env=dict(os.environ, TUIEVAL_HOME=ws))
+            try:
+                time.sleep(1.5)
+                with open(os.path.join(d, "coder-q4.log"), "a") as fh:
+                    fh.write("q4 server started\n")
+                time.sleep(1.5)
+            finally:
+                proc.send_signal(__import__("signal").SIGINT)
+                out, _ = proc.communicate(timeout=20)
+            self.assertEqual(proc.returncode, 0)
+            self.assertEqual(out, "── chat ──\nchat line\n── coder-q4 ──\nq4 server started\n")
+
+
 if __name__ == "__main__":
     unittest.main()