mjolnir-cli 0.2.0__tar.gz → 0.2.1__tar.gz

{mjolnir_cli-0.2.0 → mjolnir_cli-0.2.1}/.gitignore

@@ -3,6 +3,12 @@ __pycache__/
 .venv/
 .pytest_cache/
 .ruff_cache/
+.env
+# coverage artifacts (the CI test stage writes these; they regenerate)
+.coverage
+.coverage.*
+coverage.xml
+.coverage_html/
 dist/
 build/
 *.egg-info/

{mjolnir_cli-0.2.0 → mjolnir_cli-0.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: mjolnir-cli
-Version: 0.2.0
+Version: 0.2.1
 Summary: Serve, benchmark and document LLMs on NVIDIA Jetson AGX Thor: YAML model configs, the Mjolnir vLLM image, KV-cache calibration, kernel / serving / accuracy benchmarks and auto-rendered reports.
 Project-URL: Homepage, https://github.com/thor-arena/mjolnir
 Project-URL: Documentation, https://github.com/thor-arena/mjolnir/tree/main/docs
@@ -18,13 +18,11 @@ Classifier: Environment :: Console
 Classifier: Intended Audience :: Developers
 Classifier: Operating System :: POSIX :: Linux
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.10
-Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Classifier: Topic :: System :: Benchmark
-Requires-Python: >=3.10
+Requires-Python: >=3.12
 Requires-Dist: jinja2>=3.1.6
 Requires-Dist: pyyaml>=6.0.3
 Requires-Dist: rich>=15.0.0

{mjolnir_cli-0.2.0 → mjolnir_cli-0.2.1}/mjolnir_cli/__init__.py

@@ -5,4 +5,4 @@ kernel against FlashInfer and FlashAttention 2, and render the
 data-driven README charts — one command surface.
 """
 
-__version__ = "0.2.0"
+__version__ = "0.2.1"

{mjolnir_cli-0.2.0 → mjolnir_cli-0.2.1}/mjolnir_cli/__main__.py

@@ -1,3 +1,5 @@
+"""Run the ``mjolnir`` CLI via ``python -m mjolnir_cli``."""
+
 from mjolnir_cli.cli import app
 
 if __name__ == "__main__":

{mjolnir_cli-0.2.0 → mjolnir_cli-0.2.1}/mjolnir_cli/accuracy.py

@@ -31,6 +31,8 @@ import time
 from pathlib import Path
 from statistics import mean
 
+from mjolnir_cli.config import by_model, model_of, model_short
+
 FORMAT = "mjolnir.accuracy/1"
 TOOL_REPO = "https://github.com/SeraphimSerapis/tool-eval-bench"
 # pinned: a score is only comparable to one produced by the same scenario set
@@ -109,7 +111,7 @@ def summarize_db(db: Path) -> dict:
     cats: dict[str, dict] = {}
     scen: dict[str, list[float]] = {}
     runs = []
-    for run_id, created, status, run_type, scores_json, config_json in rows:
+    for run_id, created, status, run_type, scores_json, _config_json in rows:
         runs.append({"run_id": run_id, "created_at": created, "status": status, "run_type": run_type})
         if status != "completed":
             continue
@@ -160,6 +162,11 @@ def summarize_db(db: Path) -> dict:
             continue
         last = dict(items[-1])
         last.pop("score", None)
+        if all("max" in i for i in items):  # every trial's points: deltas rescale each trial exactly
+            last["earneds"], last["maxes"] = [i["earned"] for i in items], [i["max"] for i in items]
+        comps = [i["completion_rate"] for i in items if i.get("completion_rate") is not None]
+        if comps:
+            last["completion_rate"] = min(comps)  # the worst trial: an exclusion in any trial shows
         suites[name] = {"score": round(mean(scores), 2), "scores": scores, "n": len(scores), **last}
     return {
         "suites": suites,
@@ -298,7 +305,7 @@ def latest_legs(docs: list[dict]) -> list[dict]:
     rows: dict[tuple, dict] = {}
     for d in docs:
         for leg in d.get("legs", []):
-            key = (leg.get("model"), leg.get("image"), leg.get("config"))
+            key = (model_of(leg), leg.get("image"), leg.get("config"))
             row = rows.setdefault(
                 key,
                 {k: leg.get(k) for k in ("id", "model", "image", "config", "label", "kv_cache_dtype", "env")}
@@ -312,16 +319,102 @@ def latest_legs(docs: list[dict]) -> list[dict]:
             if (leg.get("suites") or {}).keys() & {"tools", "hardmode"}:
                 row["categories"] = leg.get("categories") or row["categories"]
                 row["scenarios"] = leg.get("scenarios") or row["scenarios"]
-    return [r for r in rows.values() if r["suites"]]
+    groups = by_model(r for r in rows.values() if r["suites"])
+    if len(groups) > 1:  # rows of several models: each model's rows together, labels say which
+        for r in (r for g in groups.values() for r in g):
+            r["label"] = f"{model_short(r)} · {r['label']}"
+    return [r for g in groups.values() for r in g]
 
 
 def baseline(rows: list[dict]) -> dict | None:
-    """The reference row for deltas: native FlashInfer on the official image
-    when measured, else the first row."""
-    for r in rows:
-        if not _is_mjolnir(str(r.get("image", ""))) and r.get("config") == "NVFP4":
+    """The reference row for deltas among ONE model's rows: native FlashInfer
+    on the official image when measured, else the first official-image row,
+    else the first row. For mixed rows use :func:`baselines`."""
+    official = [r for r in rows if not _is_mjolnir(str(r.get("image", "")))]
+    for r in official:
+        if r.get("config") == "NVFP4":
             return r
-    return rows[0] if rows else None
+    return (official or rows or [None])[0]
+
+
+def baselines(rows: list[dict]) -> list[dict | None]:
+    """Each row's baseline, in row order: its own model's :func:`baseline`.
+    Rows of another model are never a baseline — their scores are not comparable."""
+    per_model = {m: baseline(group) for m, group in by_model(rows).items()}
+    return [per_model[model_of(r)] for r in rows]
+
+
+def _maxes(s: dict) -> list:
+    """Every trial's max for a suite (older records keep only the last trial's)."""
+    return list(s.get("maxes") or ([s["max"]] if s.get("max") else []))
+
+
+def _on_max(s: dict, m: float) -> float:
+    """The suite's score with every trial's points over ``m``: exact from the
+    per-trial points when recorded, else from the one run's points, else from
+    the mean % (an older averaged record, which keeps only the last max)."""
+    if s.get("earneds"):
+        return mean(e / m * 100 for e in s["earneds"])
+    if s.get("n", 1) == 1 and "earned" in s:
+        return s["earned"] / m * 100
+    return s["score"] * s["max"] / m
+
+
+def delta(r: dict, base: dict | None, name: str) -> tuple[float, bool] | None:
+    """Points ``r`` scored over ``base`` on suite ``name``, and whether an
+    excluded scenario was scored as zero (None when ``r`` is the baseline,
+    either lacks the suite, or the two measured different scenario sets).
+
+    tool-eval-bench drops a scenario it judged the endpoint unable to serve
+    (TC-45 when its ``tool_choice=required`` probe fails, a timeout, an error),
+    which shrinks that trial's max and lowers its completion rate, so its %
+    divides by less. Then every trial of both legs is scored on the largest
+    max, the excluded points counting as zero: an exclusion never raises a
+    delta. Different maxes at full completion mean different scenario sets
+    (another run's arguments), which no rescaling makes comparable."""
+    if base is None or r is base:
+        return None
+    v, b = r.get("suites", {}).get(name) or {}, base.get("suites", {}).get(name) or {}
+    if v.get("score") is None or b.get("score") is None:
+        return None
+    maxes = _maxes(v) + _maxes(b)
+    if len(set(maxes)) > 1:
+        if all(x.get("completion_rate", 100.0) >= 100.0 for x in (v, b)):
+            return None
+        m = max(maxes)
+        return _on_max(v, m) - _on_max(b, m), True
+    return v["score"] - b["score"], False
+
+
+def fmt_delta(d: tuple[float, bool] | None) -> str:
+    """' (+1.2)', ' (-1.4‡)' when an excluded scenario was scored as zero, '' for no delta."""
+    return f" ({d[0]:+.1f}{'‡' if d[1] else ''})" if d else ""
+
+
+def leg_excluded(r: dict, base: dict | None, names: list[str]) -> bool:
+    """Whether any of ``r``'s suite deltas scored an excluded scenario as zero."""
+    return any((d := delta(r, base, n)) and d[1] for n in names)
+
+
+def any_excluded(rows: list[dict], bases: list[dict | None], names: list[str]) -> bool:
+    return any(leg_excluded(r, b, names) for r, b in zip(rows, bases, strict=True))
+
+
+EXCLUDED_NOTE = (
+    "‡ = tool-eval-bench excluded a scenario on one of the two legs; the delta scores both on the full max, "
+    "the excluded points counting as zero"
+)
+EXCLUDED_NOTE_SHORT = "‡ = a scenario was excluded on one side, scored as zero"
+CATEGORY_EXCLUDED_NOTE = (
+    "‡ = this leg had a scenario excluded: its category % divides by the smaller max, so one category reads high"
+)
+
+
+def baseline_caption(bases: list[dict | None], item_sep: str = " · ") -> str:
+    """The distinct baseline labels, first-seen order, ' / '-joined ('' if none);
+    ``item_sep`` replaces the label's own ' · ' separator."""
+    labels = dict.fromkeys(b["label"].replace(" · ", item_sep) for b in bases if b is not None)
+    return " / ".join(labels)
 
 
 def suites_in(rows: list[dict]) -> list[str]:

{mjolnir_cli-0.2.0 → mjolnir_cli-0.2.1}/mjolnir_cli/autoresearch.py

@@ -39,7 +39,6 @@ import json
 import shutil
 import subprocess
 from pathlib import Path
-from typing import Optional
 
 import typer
 from rich import box
@@ -119,7 +118,7 @@ MAX_SUSPECT = 2  # more suspect L than this and the measurement itself is reject
 
 def _gm(xs) -> float:
     xs = list(xs)
-    return results.geomean(xs) if xs else 1.0
+    return results.geomean(xs) or 1.0  # geomean is None only when no positive value survives: neutral 1.0
 
 
 def _pct(x: float) -> str:
@@ -158,7 +157,7 @@ def judge(row: dict, best: dict | None, min_gain: float = 0.02, noise: float = N
     if best is None:
         return "baseline", "the starting kernel", {}
     new, old = row.get("per_L") or {}, best.get("per_L") or {}
-    common = [L for L in new if L in old and old[L]]
+    common = [L for L in new if old.get(L)]
     if not common:  # rows written before per-L judging: the plain geomean rule
         st = decide(score, correct, best.get("score"), min_gain)
         return st, f"geomean {_pct(score / best['score'])} (no per-L reference)", {}
@@ -172,8 +171,8 @@ def judge(row: dict, best: dict | None, min_gain: float = 0.02, noise: float = N
     gain = _gm(trusted.values())
     if gain <= 1 + min_gain:
         return "discard", f"geomean {_pct(gain)} ≤ +{min_gain * 100:g}%{note}", rel
-    top = max(trusted, key=trusted.get)
-    loo = _gm(v for L, v in trusted.items() if L != top)
+    top = max(trusted, key=lambda L: trusted[L])
+    loo = _gm(v for L, v in trusted.items() if top != L)
     if loo <= 1 + min_gain / 2:
         return (
             "discard",
@@ -207,7 +206,7 @@ def _root() -> Path:
 # ── commands ────────────────────────────────────────────────────────────────
 
 
-def _init(root: Path, tag: str, base: str, evaluate_baseline: bool, image: Optional[str]) -> int:
+def _init(root: Path, tag: str, base: str, evaluate_baseline: bool, image: str | None) -> int:
     branch = f"autoresearch/{tag}"
     if _git(root, "branch", "--list", branch):
         console.print(f"[red]{branch} already exists — pick a fresh tag (a run never reuses a branch)[/red]")
@@ -232,7 +231,7 @@ def init(
     tag: str = typer.Argument(..., help="run tag, e.g. oct11 — the branch is autoresearch/<tag>"),
     base: str = typer.Option("main", "--base", help="branch to start from"),
     evaluate_baseline: bool = typer.Option(True, "--baseline/--no-baseline", help="score the starting kernel now"),
-    image: Optional[str] = typer.Option(None, "--image"),
+    image: str | None = typer.Option(None, "--image"),
 ):
     """Create the research branch and score the starting kernel (the baseline row)."""
     raise typer.Exit(_init(_root(), tag, base, evaluate_baseline, image))
@@ -243,9 +242,9 @@ def _evaluate(
     hypothesis: str,
     agent: str,
     sources: str,
-    image: Optional[str],
+    image: str | None,
     min_gain: float = 0.02,
-    ls: Optional[str] = None,
+    ls: str | None = None,
     iters: int = 50,
     gate: bool = True,
     skip_correctness: bool = False,
@@ -301,7 +300,7 @@ def _evaluate(
             by = results.series_by_id(doc)
             if "fi" in by and "mjolnir" in by:
                 sp = results.speedup(by["fi"], by["mjolnir"])
-                want = set(int(x) for x in (ls or " ".join(map(str, ALL_LS))).replace(",", " ").split())
+                want = {int(x) for x in (ls or " ".join(map(str, ALL_LS))).replace(",", " ").split()}
                 if sp and set(sp) >= want:
                     score = results.geomean(sp.values())
                     per_L = {str(k): round(v, 4) for k, v in sp.items()}
@@ -310,7 +309,7 @@ def _evaluate(
             sweep_run = doc["run_id"]
     row = {
         "id": len(rows) + 1,
-        "ts": _dt.datetime.now(_dt.timezone.utc).isoformat(timespec="seconds"),
+        "ts": _dt.datetime.now(_dt.UTC).isoformat(timespec="seconds"),
         "branch": _git(root, "rev-parse", "--abbrev-ref", "HEAD"),
         "commit": commit,
         "status": None,
@@ -390,10 +389,10 @@ def evaluate(
     agent: str = typer.Option("autoresearch", "--agent", help="who proposed/implemented it"),
     sources: str = typer.Option("", "--sources", help="comma-separated URLs / arXiv ids behind the idea"),
     min_gain: float = typer.Option(0.02, "--min-gain", help="required geomean gain over the best kept score"),
-    ls: Optional[str] = typer.Option(None, "--ls", help="L values (default: all 8; a subset is never a keep)"),
+    ls: str | None = typer.Option(None, "--ls", help="L values (default: all 8; a subset is never a keep)"),
     iters: int = typer.Option(50, "--iters"),
     gate: bool = typer.Option(True, "--gate/--no-gate"),
-    image: Optional[str] = typer.Option(None, "--image"),
+    image: str | None = typer.Option(None, "--image"),
 ):
     """Score the committed kernel: quick correctness + FlashInfer vs Mjolnir sweep;
     append the result and the fixed keep/discard decision to the ledger."""
@@ -464,8 +463,8 @@ def chart():
 
 @app.command()
 def run(
-    tag: Optional[str] = typer.Option(None, "--tag", help="start a new run first (init <tag>)"),
-    model: Optional[str] = typer.Option(None, "--model", help="OpenCode model (provider/model) for the orchestrator"),
+    tag: str | None = typer.Option(None, "--tag", help="start a new run first (init <tag>)"),
+    model: str | None = typer.Option(None, "--model", help="OpenCode model (provider/model) for the orchestrator"),
     prompt: str = typer.Option(
         "Read research/program.md and run the loop: setup is done when the ledger has a baseline row. "
         "Never stop to ask; record every experiment with `mjolnir autoresearch evaluate`.",
@@ -494,7 +493,7 @@ def run(
 
 @app.command()
 def export(
-    tag: Optional[str] = typer.Option(None, "--tag", help="run tag (default: from the autoresearch/<tag> branch)"),
+    tag: str | None = typer.Option(None, "--tag", help="run tag (default: from the autoresearch/<tag> branch)"),
 ):
     """Snapshot the ledger into research/runs/<tag>/ (ledger.jsonl, ledger.tsv,
     SUMMARY.md) — commit that to keep the run's notebook with the repo."""

{mjolnir_cli-0.2.0 → mjolnir_cli-0.2.1}/mjolnir_cli/benchy.py

@@ -115,8 +115,8 @@ def _std(vals: list[float]) -> float:
     return (sum((v - m) ** 2 for v in vals) / (len(vals) - 1)) ** 0.5
 
 
-def _p95(vals: list[float]) -> float:
-    """Linear-interpolated 95th percentile."""
+def _p95(vals: list[float]) -> float | None:
+    """Linear-interpolated 95th percentile (``None`` on empty input)."""
     if not vals:
         return None
     s = sorted(vals)
@@ -295,7 +295,7 @@ def run_perf(
             # pooled like the totals: accepted at each position over all drafts
             "per_pos": [
                 round(sum(col) / tot["drafts"], 4)
-                for col in zip(*(x.get("per_pos_accepted") or [] for x in sweep_spec))
+                for col in zip(*(x.get("per_pos_accepted") or [] for x in sweep_spec), strict=False)
             ]
             if tot["drafts"]
             else [],
@@ -438,13 +438,15 @@ def _pipe_table(headers: list[str], rows: list[list[str]]) -> str:
 
     def data_row(cells: list[str]) -> str:
         return (
-            "| " + " | ".join(c.rjust(w) if a == ">" else c.ljust(w) for c, w, a in zip(cells, widths, aligns)) + " |"
+            "| "
+            + " | ".join(c.rjust(w) if a == ">" else c.ljust(w) for c, w, a in zip(cells, widths, aligns, strict=True))
+            + " |"
         )
 
     def sep_cell(a: str, w: int) -> str:
         return ":" + "-" * (w - 1) if a == "<" else "-" * (w - 1) + ":"
 
-    sep = "|" + "|".join(sep_cell(a, w) for w, a in zip(widths, aligns)) + "|"
+    sep = "|" + "|".join(sep_cell(a, w) for w, a in zip(widths, aligns, strict=True)) + "|"
     return "\n".join([data_row(headers), sep, *map(data_row, rows)])