tuieval 0.2.0.dev68__py3-none-any.whl → 0.2.0.dev72__py3-none-any.whl

tuieval/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.2.0.dev68'
-__version_tuple__ = version_tuple = (0, 2, 0, 'dev68')
+__version__ = version = '0.2.0.dev72'
+__version_tuple__ = version_tuple = (0, 2, 0, 'dev72')
 
 __commit_id__ = commit_id = None

tuieval/cli.py

@@ -18,6 +18,7 @@ Day to day
   tuieval report              the verdicts with their evidence, as a markdown file
   tuieval history             every verdict change, newest first
   tuieval items               tests that don't separate models or look broken
+  tuieval logs                follow the server log of the model running now (live: its answers)
   tuieval capture LOG --pack  turn a real failure (logs/live/*.txt) into a new test
   tuieval regrade             re-score stored answers after changing a grader (--only, --packs)
   tuieval list                the models tuieval knows, hidden ones separately (--packs: the packs)
@@ -85,6 +86,9 @@ def main(argv=None):
     if cmd == "watch":
         from . import watch_proxy
         return watch_proxy.main(rest)
+    if cmd == "logs":
+        from . import logs
+        return logs.main(rest)
     from . import run_evals
     if cmd in run_evals.COMMANDS:
         return run_evals.COMMANDS[cmd](rest)

tuieval/logs.py

@@ -0,0 +1,220 @@
+"""tuieval logs: follow the server log of the model running now, without naming it.
+
+    tuieval logs              the newest server log, then every log that grows (a header says
+                              which model each part is from), so it moves on with the run
+    tuieval logs live         each answer (reasoning and answer, from logs/live/) as it finishes
+    tuieval logs --list       every model's server log, newest first
+    tuieval logs MODEL        one model's log (any unambiguous part of its label)
+
+Read-only: it never touches servers or results. Only servers tuieval starts write a log here;
+OpenRouter and servers that were already running don't.
+"""
+import argparse
+import datetime
+import os
+import re
+import sys
+import time
+
+from . import workspace
+
+POLL_S = 0.5
+
+
+def server_dir():
+    return workspace.path("logs", "server")
+
+
+def live_dir():
+    return workspace.path("logs", "live")
+
+
+def _files(d, ext):
+    """{path: (size, mtime)} for the files in d with this extension (none if d doesn't exist)."""
+    out = {}
+    try:
+        names = os.listdir(d)
+    except OSError:
+        return out
+    for n in names:
+        if n.endswith(ext):
+            p = os.path.join(d, n)
+            try:
+                st = os.stat(p)
+            except OSError:
+                continue
+            out[p] = (st.st_size, st.st_mtime)
+    return out
+
+
+def _label(path):
+    return os.path.basename(path)[:-len(".log")]
+
+
+def _tail_offset(path, lines):
+    """Where the last `lines` lines of a file start."""
+    with open(path, "rb") as f:
+        f.seek(0, os.SEEK_END)
+        end = pos = f.tell()
+        found, chunk = 0, 8192
+        while pos > 0 and found <= lines:
+            pos = max(0, pos - chunk)
+            f.seek(pos)
+            found = f.read(end - pos).count(b"\n")
+        if pos == 0 and found <= lines:
+            return 0
+        f.seek(pos)
+        data = f.read(end - pos)
+    cut = len(data)
+    for _ in range(lines + 1):
+        cut = data.rfind(b"\n", 0, cut)
+    return pos + cut + 1
+
+
+class Follower:
+    """Follows server logs. `poll()` returns the text written since the last poll, with a
+    `── label ──` header whenever it comes from another log than the text before it.
+    only: follow just this log. Otherwise every log in the folder, including new ones."""
+
+    def __init__(self, folder, lines=40, only=None):
+        self.folder, self.only = folder, only
+        self.offsets = {p: size for p, (size, _) in _files(folder, ".log").items()}
+        self.last = None        # the log the previous text came from
+        self.held = {}          # path -> size at the last poll, for a line still being written
+        start = only or max(self.offsets, key=lambda p: os.path.getmtime(p), default=None)
+        if start and start in self.offsets:
+            self.offsets[start] = _tail_offset(start, lines)
+
+    def poll(self):
+        out = []
+        files = _files(self.folder, ".log")
+        if self.only:
+            files = {p: v for p, v in files.items() if p == self.only}
+        for path, (size, _) in sorted(files.items(), key=lambda kv: kv[1][1]):   # oldest change first
+            off = self.offsets.setdefault(path, 0)
+            if size < off:          # truncated or replaced: read it from the start
+                off = 0
+            if size == off:
+                continue
+            with open(path, "rb") as f:
+                f.seek(off)
+                data = f.read(size - off)
+            cut = data.rfind(b"\n") + 1
+            if not cut:             # no complete line yet: wait one poll for the rest of it
+                if self.held.get(path) != size:
+                    self.held[path] = size
+                    continue
+                cut = len(data)
+            self.held.pop(path, None)
+            self.offsets[path] = off + cut
+            if path != self.last:
+                out.append(f"── {_label(path)} ──\n")
+                self.last = path
+            out.append(data[:cut].decode("utf-8", "replace"))
+        return "".join(out)
+
+
+FOOTER = re.compile(r"^=== .*finish=.* ===$", re.M)
+
+
+class LiveFollower:
+    """New answers in logs/live/ as they're written: each one from its reasoning on (the prompt is
+    in the file), under a header with the file name."""
+
+    def __init__(self, folder, last=1):
+        self.folder = folder
+        files = sorted(_files(folder, ".txt").items(), key=lambda kv: kv[1][1])
+        self.seen = {p for p, _ in files[:max(0, len(files) - last)]}
+
+    def poll(self):
+        out = []
+        for path, (_, mtime) in sorted(_files(self.folder, ".txt").items(), key=lambda kv: kv[1][1]):
+            if path in self.seen:
+                continue
+            try:
+                with open(path, errors="replace") as f:
+                    text = f.read()
+            except OSError:
+                continue
+            if not FOOTER.search(text) and time.time() - mtime < 2:   # still being written
+                continue
+            self.seen.add(path)
+            i = text.find("=== REASONING ===")
+            out.append(f"── {os.path.basename(path)[:-4]} ──\n{text[i:] if i >= 0 else text}\n")
+        return "".join(out)
+
+
+def _ago(mtime):
+    s = max(0, int(time.time() - mtime))
+    if s < 60:
+        return f"{s}s ago"
+    if s < 3600:
+        return f"{s // 60} min ago"
+    if s < 86400:
+        return f"{s // 3600} h ago"
+    return datetime.datetime.fromtimestamp(mtime).strftime("%Y-%m-%d %H:%M")
+
+
+def _size(n):
+    for unit in ("B", "KB", "MB", "GB"):
+        if n < 1024 or unit == "GB":
+            return f"{n:.0f} {unit}" if unit == "B" else f"{n:.1f} {unit}"
+        n /= 1024
+
+
+def list_logs(folder):
+    files = sorted(_files(folder, ".log").items(), key=lambda kv: -kv[1][1])
+    if not files:
+        return f"No server logs yet in {os.path.relpath(folder)}/."
+    w = max(len(_label(p)) for p, _ in files)
+    return "\n".join(f"{_label(p):<{w}}  {_size(size):>9}  {_ago(mtime)}" for p, (size, mtime) in files)
+
+
+def find_log(folder, name):
+    """The log whose label is `name`, or the only one containing it. Raises SystemExit otherwise."""
+    labels = {_label(p): p for p in _files(folder, ".log")}
+    if name in labels:
+        return labels[name]
+    hits = sorted(l for l in labels if name.lower() in l.lower())
+    if len(hits) == 1:
+        return labels[hits[0]]
+    if not hits:
+        sys.exit(f"No server log matches {name!r}; tuieval logs --list shows them.")
+    sys.exit(f"{name!r} matches {len(hits)} logs; say which:\n  " + "\n  ".join(hits))
+
+
+def follow(follower, waiting):
+    """Print what the follower finds until ctrl-c."""
+    said = False
+    try:
+        while True:
+            text = follower.poll()
+            if text:
+                sys.stdout.write(text)
+                sys.stdout.flush()
+                said = True
+            elif not said:
+                print(waiting, flush=True)
+                said = True
+            time.sleep(POLL_S)
+    except KeyboardInterrupt:
+        return 0
+
+
+def main(argv):
+    p = argparse.ArgumentParser(prog="tuieval logs", description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter,
+                                epilog="ctrl-c stops following (a run going on isn't affected).")
+    p.add_argument("what", nargs="?", help="'live' for answers, or a model (any unambiguous part of its label)")
+    p.add_argument("--list", action="store_true", help="every model's server log, newest first")
+    p.add_argument("-n", "--lines", type=int, default=40, help="lines shown before following (default 40)")
+    a = p.parse_args(argv)
+    if a.list:
+        print(list_logs(server_dir()))
+        return 0
+    if a.what == "live":
+        return follow(LiveFollower(live_dir()),
+                      f"Waiting for answers in {os.path.relpath(live_dir())}/ (ctrl-c stops)")
+    only = find_log(server_dir(), a.what) if a.what else None
+    return follow(Follower(server_dir(), a.lines, only),
+                  f"Waiting for {'that model' if only else 'a server'} to log in "
+                  f"{os.path.relpath(server_dir())}/ (ctrl-c stops)")

tuieval/run_evals.py

@@ -11,6 +11,7 @@
     tuieval run --only my-model --packs my-pack,other-pack
     tuieval run --tags moe --packs my-pack        # models tagged "moe"
     tuieval run --packs my-pack --tests a,b       # only these tests of a pack (or --tests my-pack:a,other:b)
+    tuieval run --only my-model --packs my-pack --tests a --retest   # ask a question again
     tuieval run --preset nightly                  # a selection saved in the TUI (presets.toml)
     tuieval run --tier smoke --only my-model      # 3 tests per pack, 1 repeat: check paths and flags
     tuieval run --tier smoke --only openrouter:qwen/qwen3-32b   # any OpenRouter model, no models.toml edit
@@ -742,7 +743,8 @@ def main(argv=None):
     p.add_argument("--packs", help="comma-separated packs (default: all)")
     p.add_argument("--tests", help="comma-separated test ids to run instead of the tier's sample: id with one "
                                    "pack in --packs, else pack:id (and --packs defaults to those packs)")
-    p.add_argument("--force", action="store_true", help="rerun packs (or the --tests) that already have current results")
+    p.add_argument("--retest", "--force", action="store_true", dest="force",
+                   help="ask again even if already answered: the packs, or just the --tests (earlier answers go to history/)")
     p.add_argument("--repeat", type=int, help="times each question is asked, any tier (default: 1 for smoke and "
                                                   "screen, the pack's certification count for certify)")
     p.add_argument("--tier", choices=engine.TIERS, default="screen",

tuieval/tui.py

@@ -385,6 +385,9 @@ HELP = {
 [b]Models at a time[/b]: how many of the ticked models run side by side (blank: this machine's
 parallel_models in models.toml, default 1). Answers note the models they ran alongside.
 
+[b]Retest[/b]: ask the ticked questions again even if they're already answered (only the tests
+picked with e, if any). Without it, answered questions are skipped. Replaced answers go to history/.
+
 [b]Keys[/b]
   s  start (with a run going: add to it, or queue it)   r  results
   a  add a model (a GGUF, a model id, openrouter:<id>)  m  scan model folders for new GGUFs
@@ -646,7 +649,7 @@ class SetupScreen(Screen):
             yield Input("", id="repeat", type="integer", max_length=2, placeholder="1")
             yield Label("Models at a time")
             yield Input("", id="parallel", type="integer", max_length=1, placeholder="1")
-            yield Checkbox("Rerun existing results", id="force")
+            yield Checkbox("Retest (ask again, even if already answered)", id="force")
         yield Static(id="estimate")
         with Horizontal(id="buttons"):
             yield Button("Start  [s]", id="start", variant="success")
@@ -958,7 +961,7 @@ class SetupScreen(Screen):
         if not run:
             text += "\n[yellow]Nothing selected can run (see skipping).[/yellow]"
         elif rerun:
-            text += "\n[yellow]Everything selected has results; Start runs it again.[/yellow]"
+            text += "\n[yellow]Everything selected has results; Start retests it.[/yellow]"
         resumed = [j for j in run if "done earlier" in j.note]
         if resumed:
             text += f"   [yellow]{len(resumed)} pack(s) continue from earlier results[/yellow]"

{tuieval-0.2.0.dev68.dist-info → tuieval-0.2.0.dev72.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: tuieval
-Version: 0.2.0.dev68
+Version: 0.2.0.dev72
 Summary: Evaluate local and frontier LLMs on your own questions: accuracy, speed, tokens and PASS/FAIL verdicts, in the terminal.
 Project-URL: Homepage, https://github.com/ashe-wb/tuieval
 Project-URL: Issues, https://github.com/ashe-wb/tuieval/issues
@@ -162,6 +162,8 @@ tuieval history                               # every verdict change over time
 tuieval compare --speed --pairwise --failures # scorecards
 tuieval selftest                              # check every test's reference and wrong answers
 tuieval items                                 # tests that don't separate models or look broken
+tuieval logs                                  # follow the server log of the model running now
+tuieval logs live                             # each answer as it finishes (--list: every log)
 tuieval capture logs/live/<file> --pack X     # turn a real failure into a new test
 tuieval regrade                               # re-score stored answers after changing a grader
 tuieval regrade --only <model> --packs X      # just one model's results (and packs)

{tuieval-0.2.0.dev68.dist-info → tuieval-0.2.0.dev72.dist-info}/RECORD

@@ -1,23 +1,24 @@
 tuieval/__init__.py,sha256=myVXVX7htbwezW6k3x_FuKgn1RSZ2WcTaDEMByGn6Wg,312
 tuieval/__main__.py,sha256=E6Gls0DNz8GQK2K-kOUIx8cYhgANW_CH54VKrfCfs14,52
-tuieval/_version.py,sha256=-5RoY-3h38Zwg0fzKJuoa3DtBDQFu5H5pzCM1Lh0aFE,535
+tuieval/_version.py,sha256=5bSTQoJyfnoC1UimCugL5jEW1WKsn9kwuboOZkyp3xk,535
 tuieval/builtin.toml,sha256=Kxw22RZCJhURUyu5Wz8tj5Y1nM85A8hFVRbA4ocoPUg,4293
-tuieval/cli.py,sha256=fn6z0cupM-aW3fY5fFgnS-HmOQmMzlDg-U76Y6LgL3o,4510
+tuieval/cli.py,sha256=JGtN4J6d4cr9G7OfBxDiNtxvVEew9MgUhJBlYKYPhJ4,4687
 tuieval/client.py,sha256=kpsyLEZKfxCsvIGBHBbBy83TpI97CvnGCDq-4AbJS4Y,9986
 tuieval/compare.py,sha256=T6a_PC7z-jHEvw09KEceo2dxoLMi-9cHYPQs4L6Ctig,26997
 tuieval/doctor.py,sha256=j2MdDORstsAGPnRmpsnxau1_fR6sTc_oEFGM5zugTJU,7246
 tuieval/engine.py,sha256=SsXqR1ydn827qBX8NoawyT2roNuelmrZB-hOuyGSesY,116514
 tuieval/export.py,sha256=dbHQgL_ubWtIPM1sLzAew_gETnrDJyd0zgqVA5vkpXM,9657
+tuieval/logs.py,sha256=01gZK6tf53dIThaIrs1bEsz9eqKYVlcxpyaLnSPTVxo,8127
 tuieval/machines.py,sha256=t9tu6CZeXZON4EtOVnEwP4VtaynpVLf9Cz8TNjf7jtY,20124
 tuieval/onboard.py,sha256=IYETVDWCCo9vF0Dtj4_rr9VanYeCky8PwP3-2g5A_l0,6215
 tuieval/packs.py,sha256=tiLmxsrX_M5EUMuR4Yvb7pLBLEI15wamHSjflD6wjoY,11299
 tuieval/profiles.py,sha256=soeLxjdB-l-OxSY8jZ4yjwfdSnsBV-QICcwKxRJS7nQ,3663
 tuieval/pta.py,sha256=i5oPj-Pz3N9u7bzzuZwsQvMGdxWdAncww2nR15K6PHY,12108
 tuieval/remove.py,sha256=Vwh43nm9lVb9U9aN-h6CMYbkTwdR3UWfZObwweF72n0,5157
-tuieval/run_evals.py,sha256=1zNkCdFt744cqpebqiyj-sZrbOI22c6IYBcXk9koz20,46538
+tuieval/run_evals.py,sha256=WVrGU418Fv4oP5ddKNd5k2_0QjNz4ZEiJHQhKrNDR4w,46712
 tuieval/scaffold.py,sha256=YBOqniJpf_GtKKAFF3_OHxxMGbmTWdFjcClGcOnrrKE,12130
 tuieval/selftest.py,sha256=nExkrOidgymlmvkxImEBfrQG5iTUN5WuRMIKtqusCCs,9835
-tuieval/tui.py,sha256=CkK7e3aJ3kli--u0cJrpaQKEoGAucs3grg3xt6x5v2c,166041
+tuieval/tui.py,sha256=q-ufpI9YdYPJhrmomDOZJME8sBw1u94_P3t4VMzEIpo,166257
 tuieval/tune.py,sha256=azxxthgJeag8JyoQUgaRxq6oI1XvEgzb9vkszjJZH7w,39645
 tuieval/verdict.py,sha256=VB-0LTxR9y8bJHT9RicxtE9K7cUHBdCu40hOCrsbmA4,19150
 tuieval/watch_proxy.py,sha256=bnDO1i6n6yUs1hODoUWcMppJMGPavoQqs6aNYrIxpT8,11416
@@ -46,8 +47,8 @@ tuieval/templates/packs/tool_call/pack.toml,sha256=n4bx0lGJHSmW41uVqhfIXJ2BWZpmT
 tuieval/templates/packs/tool_call/system.txt,sha256=mKxu-lB-0YjLHXdgeFJa8x2C180OvcNz5cHv2N5Bg3U,254
 tuieval/templates/packs/tool_call/tests.yaml,sha256=Nv7c4syrzJ9cU8EBKSK_1ZSFx9V02aCjW4-2TBCPT7s,1859
 tuieval/templates/packs/tool_call/tools.yaml,sha256=HgxCkh0On2VwbHSsjAGM8tNCK7ZtKLGrs7lMCn3Fprs,1048
-tuieval-0.2.0.dev68.dist-info/METADATA,sha256=GpjUswFVv-sJbkVrDXX4-rloDzF8Mngap9YKtG3yrZs,14025
-tuieval-0.2.0.dev68.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
-tuieval-0.2.0.dev68.dist-info/entry_points.txt,sha256=7eIhi1GpFxBP2OLJV02Nn8LwsNuO512eKOGt1IBbj04,45
-tuieval-0.2.0.dev68.dist-info/licenses/LICENSE,sha256=7wfRgGtBGH1aXhZkIn47oUznqKC18kEPMvUVgol7Ffc,1077
-tuieval-0.2.0.dev68.dist-info/RECORD,,
+tuieval-0.2.0.dev72.dist-info/METADATA,sha256=MbPZAuXA-J7aRQAolpHk2QRyyt97Y3Sua4RjKuER1bM,14215
+tuieval-0.2.0.dev72.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+tuieval-0.2.0.dev72.dist-info/entry_points.txt,sha256=7eIhi1GpFxBP2OLJV02Nn8LwsNuO512eKOGt1IBbj04,45
+tuieval-0.2.0.dev72.dist-info/licenses/LICENSE,sha256=7wfRgGtBGH1aXhZkIn47oUznqKC18kEPMvUVgol7Ffc,1077
+tuieval-0.2.0.dev72.dist-info/RECORD,,