evalpair 0.1.0__tar.gz → 0.1.1__tar.gz

{evalpair-0.1.0 → evalpair-0.1.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: evalpair
-Version: 0.1.0
+Version: 0.1.1
 Summary: Run an LLM eval suite against a local or hosted model, cache every call, and compare two runs item by item with bootstrap intervals and a measured noise floor. CLI and MCP server.
 Keywords: llm,evaluation,eval,benchmark,bootstrap,ollama,mcp,cli
 Author: Kameswara Sai Srikar Manda
@@ -61,8 +61,8 @@ A real comparison: Ollama's default 4-bit Llama 3.1 8B against its 2-bit build,
 
 ```
 $ evalpair compare llama-q4 llama-q2
-A  llama-q4: 84.4% [79.4%, 90.0%]
-B  llama-q2: 60.6% [53.1%, 68.1%]
+A            llama-q4: 84.4% [79.4%, 90.0%]
+B            llama-q2: 60.6% [53.1%, 68.1%]
 B − A        -23.8 pp [-31.9 pp, -16.2 pp]
 changed      40.6% [33.1%, 48.1%] of 160 items (43 worse in B, 5 better)
 floor (A)    0.0% [0.0%, 0.0%]; excess +40.6 pp [+33.1 pp, +48.1 pp]
@@ -70,7 +70,7 @@ floor (A)    0.0% [0.0%, 0.0%]; excess +40.6 pp [+33.1 pp, +48.1 pp]
 B scores lower than A: -23.8 pp [-31.9 pp, -16.2 pp], paired over 160 items. Answers changed on 40.6% of
 items (43 got worse, 5 better). The noise floor (the same model asked twice) is 0.0% for A, so the change
 is beyond noise.
-  worse in b: gsm8k-0259  '168' → '52'
+  worse in B:  gsm8k-0259  '168' → '52'
   ...
 ```
 

{evalpair-0.1.0 → evalpair-0.1.1}/README.md

@@ -34,8 +34,8 @@ A real comparison: Ollama's default 4-bit Llama 3.1 8B against its 2-bit build,
 
 ```
 $ evalpair compare llama-q4 llama-q2
-A  llama-q4: 84.4% [79.4%, 90.0%]
-B  llama-q2: 60.6% [53.1%, 68.1%]
+A            llama-q4: 84.4% [79.4%, 90.0%]
+B            llama-q2: 60.6% [53.1%, 68.1%]
 B − A        -23.8 pp [-31.9 pp, -16.2 pp]
 changed      40.6% [33.1%, 48.1%] of 160 items (43 worse in B, 5 better)
 floor (A)    0.0% [0.0%, 0.0%]; excess +40.6 pp [+33.1 pp, +48.1 pp]
@@ -43,7 +43,7 @@ floor (A)    0.0% [0.0%, 0.0%]; excess +40.6 pp [+33.1 pp, +48.1 pp]
 B scores lower than A: -23.8 pp [-31.9 pp, -16.2 pp], paired over 160 items. Answers changed on 40.6% of
 items (43 got worse, 5 better). The noise floor (the same model asked twice) is 0.0% for A, so the change
 is beyond noise.
-  worse in b: gsm8k-0259  '168' → '52'
+  worse in B:  gsm8k-0259  '168' → '52'
   ...
 ```
 

{evalpair-0.1.0 → evalpair-0.1.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "evalpair"
-version = "0.1.0"
+version = "0.1.1"
 description = "Run an LLM eval suite against a local or hosted model, cache every call, and compare two runs item by item with bootstrap intervals and a measured noise floor. CLI and MCP server."
 readme = "README.md"
 requires-python = ">=3.11"

{evalpair-0.1.0 → evalpair-0.1.1}/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "evalpair"
-version = "0.1.0"
+version = "0.1.1"
 description = "Run an LLM eval suite against a local or hosted model, cache every call, and compare two runs item by item with bootstrap intervals and a measured noise floor. CLI and MCP server."
 readme = "README.md"
 requires-python = ">=3.11"

{evalpair-0.1.0 → evalpair-0.1.1}/src/evalpair/__init__.py

@@ -1,3 +1,3 @@
 """Run an LLM eval suite, cache every call, and compare two runs item by item with intervals."""
 
-__version__ = "0.1.0"
+__version__ = "0.1.1"

{evalpair-0.1.0 → evalpair-0.1.1}/src/evalpair/cli.py

@@ -48,7 +48,7 @@ def text_summary(s: dict) -> str:
     if s["failures"]:
         lines.append("  failures (unparsed and truncated first):")
         for f in s["failures"]:
-            end = f["reply_end"].replace("\n", " ")[-160:]
+            end = " ".join(f["reply_end"].split())[-160:]  # newlines and tabs as single spaces
             lines.append(f"    {f['id']}: answer {f['answer']!r} via {f['how']}"
                          f"{', truncated' if f['truncated'] else ''}  …{end}")
     return "\n".join(lines)
@@ -57,17 +57,17 @@ def text_summary(s: dict) -> str:
 def text_compare(c: dict) -> str:
     if not c.get("items"):
         return "\n".join(c["notes"])
-    lines = [f"A  {c['a']['run']}: {c['a']['score']['text']}",
-             f"B  {c['b']['run']}: {c['b']['score']['text']}",
+    lines = [f"A            {c['a']['run']}: {c['a']['score']['text']}",
+             f"B            {c['b']['run']}: {c['b']['score']['text']}",
              f"B − A        {c['difference']['text']}",
              f"changed      {c['answers_changed']['text']} of {c['items']} items "
              f"({c['worse_in_b']} worse in B, {c['better_in_b']} better)"]
     for k, f in (c.get("noise_floor") or {}).items():
         lines.append(f"floor ({k.upper()})    {f['change_rate']['text']}; excess {f['excess_over_floor']['text']}")
     lines += ["", c["verdict"]] + [f"Note: {n}" for n in c["notes"]]
-    for kind in ("worse_in_b", "better_in_b"):
+    for kind, label in (("worse_in_b", "worse in B"), ("better_in_b", "better in B")):
         for e in c["examples"][kind]:
-            lines.append(f"  {kind.replace('_', ' ')}: {e['id']}  {e['a_answer']!r} → {e['b_answer']!r}")
+            lines.append(f"  {label + ':':<13}{e['id']}  {e['a_answer']!r} → {e['b_answer']!r}")
     return "\n".join(lines)