kipimo 0.2.2__tar.gz → 0.3.0__tar.gz

{kipimo-0.2.2/src/kipimo.egg-info → kipimo-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kipimo
-Version: 0.2.2
+Version: 0.3.0
 Summary: Swahili agent-task evaluation suite for the East Africa coordination stack — model-agnostic seed benchmark (46 tasks: server routing, term grounding, cascade routing).
 Author: AfriKaziOS maintainers
 License-Expression: MIT
@@ -65,3 +65,29 @@ MIT-licensed harness, CC BY 4.0 data. Feedback via GitHub Issues only — pull r
 
 Model-agnostic by design: closed APIs, open-weight models, and small distilled models are all first-class citizens.
 <!-- /interconnect:v1 -->
+
+## Running a scorecard (`kipimo run`)
+
+Fan out across targets in parallel, converge to one ranked scorecard. kipimo
+still never calls a model API — you supply a *generator command* per target
+(any process that reads tasks JSONL on stdin and writes predictions JSONL on
+stdout; see `examples/generate_predictions.py`).
+
+```bash
+cat > generators.json <<'JSON'
+{
+  "gemma-3-12b":    "python generate_predictions.py --model gemma-3-12b",
+  "inkubalm-0.4b":  "python generate_predictions.py --model inkubalm-0.4b",
+  "kimi-k3":        ""
+}
+JSON
+
+kipimo run generators.json --timeout 600 --workers 4
+```
+
+**An untested target is UNKNOWN, never zero.** If a generator crashes, times
+out, or is unconfigured, that target appears under `untested` with a reason —
+it is not ranked at 0.0. Conflating "the model failed the task" with "we never
+tested the model" is how an evaluation starts measuring its own assumptions
+instead of reality. Every scorecard also reports `coverage`, so a partial run
+cannot be misread as a ranking of the field.

{kipimo-0.2.2 → kipimo-0.3.0}/README.md

@@ -47,3 +47,29 @@ MIT-licensed harness, CC BY 4.0 data. Feedback via GitHub Issues only — pull r
 
 Model-agnostic by design: closed APIs, open-weight models, and small distilled models are all first-class citizens.
 <!-- /interconnect:v1 -->
+
+## Running a scorecard (`kipimo run`)
+
+Fan out across targets in parallel, converge to one ranked scorecard. kipimo
+still never calls a model API — you supply a *generator command* per target
+(any process that reads tasks JSONL on stdin and writes predictions JSONL on
+stdout; see `examples/generate_predictions.py`).
+
+```bash
+cat > generators.json <<'JSON'
+{
+  "gemma-3-12b":    "python generate_predictions.py --model gemma-3-12b",
+  "inkubalm-0.4b":  "python generate_predictions.py --model inkubalm-0.4b",
+  "kimi-k3":        ""
+}
+JSON
+
+kipimo run generators.json --timeout 600 --workers 4
+```
+
+**An untested target is UNKNOWN, never zero.** If a generator crashes, times
+out, or is unconfigured, that target appears under `untested` with a reason —
+it is not ranked at 0.0. Conflating "the model failed the task" with "we never
+tested the model" is how an evaluation starts measuring its own assumptions
+instead of reality. Every scorecard also reports `coverage`, so a partial run
+cannot be misread as a ranking of the field.

{kipimo-0.2.2 → kipimo-0.3.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "kipimo"
-version = "0.2.2"
+version = "0.3.0"
 description = "Swahili agent-task evaluation suite for the East Africa coordination stack — model-agnostic seed benchmark (46 tasks: server routing, term grounding, cascade routing)."
 readme = "README.md"
 requires-python = ">=3.10"

{kipimo-0.2.2 → kipimo-0.3.0}/src/kipimo/__init__.py

@@ -8,3 +8,5 @@ from .targets import FAMILIES as FAMILIES
 from .targets import TARGETS as TARGETS
 from .targets import get_target as get_target
 from .targets import list_targets as list_targets
+from .harness import run_scorecard as run_scorecard
+from .harness import run_target as run_target

{kipimo-0.2.2 → kipimo-0.3.0}/src/kipimo/cli.py

@@ -10,6 +10,7 @@ Usage:
     kipimo tasks                 # emit the task set (JSONL) to stdout
     kipimo template              # emit an empty predictions file to fill in
     kipimo score preds.jsonl     # score predictions against gold
+    kipimo run generators.json   # run targets in parallel -> ranked scorecard
     kipimo targets               # emit the scorecard target registry (v0.2)
 """
 
@@ -21,7 +22,7 @@ import sys
 from collections import defaultdict
 from importlib import resources
 
-__version__ = "0.2.0"
+__version__ = "0.3.0"
 
 DISCLAIMER = ("kipimo v0.1 is a SEED benchmark (46 tasks). Swahili phrasing is "
               "simple-register and pending native-speaker review (issue #1). "
@@ -86,6 +87,10 @@ def main(argv: list[str] | None = None) -> int:
     tp.add_argument("--family", choices=("closed-api", "open-weight", "small-open"),
                     default=None, help="filter by target family")
     sub.add_parser("template", help="emit empty predictions JSONL to stdout")
+    rp = sub.add_parser("run", help="run generators across targets in parallel -> scorecard")
+    rp.add_argument("generators", help='JSON file: {"target_id": "generator command", ...}')
+    rp.add_argument("--timeout", type=int, default=600, help="seconds per target (default 600)")
+    rp.add_argument("--workers", type=int, default=4, help="parallel targets (default 4)")
     sp = sub.add_parser("score", help="score a predictions file")
     sp.add_argument("predictions", help="JSONL with {id, prediction:[...]} rows")
     args = p.parse_args(argv)
@@ -100,6 +105,15 @@ def main(argv: list[str] | None = None) -> int:
     elif args.cmd == "template":
         for t in load_tasks():
             print(json.dumps({"id": t["id"], "prediction": []}))
+    elif args.cmd == "run":
+        from .harness import load_generators, run_scorecard
+        card = run_scorecard(load_generators(args.generators),
+                             timeout=args.timeout, max_workers=args.workers)
+        print(json.dumps(card, indent=2, ensure_ascii=False))
+        if card["targets_untested"]:
+            print(f"\nNote: {card['targets_untested']} target(s) UNTESTED (unknown, not zero). "
+                  f"See 'untested' in the scorecard.", file=sys.stderr)
+        print(f"\n{DISCLAIMER}", file=sys.stderr)
     else:
         rep = score_file(args.predictions)
         print(json.dumps(rep, indent=2))

kipimo-0.3.0/src/kipimo/harness.py

@@ -0,0 +1,145 @@
+"""kipimo.harness — run the benchmark across targets in parallel, converge to a scorecard.
+
+The diamond: fan out one worker per target, converge into a single ranked
+scorecard. Targets are independent (a real dependency test — no target's
+prediction informs another's), so they run concurrently; the scorecard is the
+only true convergence point.
+
+Two design rules, both deliberate:
+
+1. **kipimo still never calls a model API.** This module orchestrates *generator
+   commands you supply* (any process that reads tasks JSONL on stdin and writes
+   predictions JSONL on stdout). Model access, keys, and endpoints stay outside
+   the package, so the benchmark remains vendor-neutral and reproducible by
+   anyone. See examples/generate_predictions.py for a stdlib generator.
+
+2. **A missing run is never a zero.** The scoring path already treats an absent
+   prediction as 0.0, which is correct *within* a run. Across targets it would
+   be a lie: a target whose generator crashed, timed out, or was never
+   configured has an UNKNOWN score, not a bad one. Conflating "the model failed
+   the task" with "we never tested the model" is how an evaluation system starts
+   training on its own assumptions instead of on evidence. Every result
+   therefore carries an explicit ``status``:
+
+       ok         — generator ran, predictions scored
+       error      — generator failed (message retained)
+       timeout    — generator exceeded its deadline
+       skipped    — no generator configured for this target
+
+   Only ``ok`` results carry a score. The scorecard reports coverage alongside
+   results so a partial run can never be read as a complete one.
+"""
+from __future__ import annotations
+
+import concurrent.futures
+import json
+import shlex
+import subprocess
+import tempfile
+import time
+from pathlib import Path
+
+from .cli import load_tasks, score_file
+from .targets import get_target, list_targets
+
+DEFAULT_TIMEOUT = 600  # seconds per target; generators are network-bound
+
+
+def _tasks_jsonl() -> str:
+    return "\n".join(json.dumps(t, ensure_ascii=False) for t in load_tasks())
+
+
+def run_target(target_id: str, command: str, timeout: int = DEFAULT_TIMEOUT) -> dict:
+    """Run one generator command and score its predictions.
+
+    ``command`` is a shell-style command that reads tasks JSONL on stdin and
+    writes predictions JSONL ({"id":..., "prediction":[...]}) on stdout.
+    Returns a result dict with an explicit status; never raises for generator
+    failure (a broken target must not abort the whole scorecard).
+    """
+    started = time.time()
+    base = {"target": target_id, "command": command}
+    try:
+        get_target(target_id)  # validate against the registry
+    except Exception:
+        base["registry"] = "unregistered"
+    if not command:
+        return {**base, "status": "skipped", "score": None,
+                "reason": "no generator command configured"}
+    try:
+        proc = subprocess.run(
+            shlex.split(command), input=_tasks_jsonl(), text=True,
+            capture_output=True, timeout=timeout,
+        )
+    except subprocess.TimeoutExpired:
+        return {**base, "status": "timeout", "score": None,
+                "elapsed_s": round(time.time() - started, 1),
+                "reason": f"exceeded {timeout}s"}
+    except Exception as e:  # generator binary missing, bad command, etc.
+        return {**base, "status": "error", "score": None,
+                "reason": f"{type(e).__name__}: {e}"}
+    if proc.returncode != 0:
+        return {**base, "status": "error", "score": None,
+                "reason": f"exit {proc.returncode}: {(proc.stderr or '').strip()[:200]}"}
+    if not proc.stdout.strip():
+        return {**base, "status": "error", "score": None,
+                "reason": "generator produced no predictions"}
+    with tempfile.NamedTemporaryFile("w", suffix=".jsonl", delete=False,
+                                     encoding="utf-8") as fh:
+        fh.write(proc.stdout)
+        path = fh.name
+    try:
+        report = score_file(path)
+    except Exception as e:
+        return {**base, "status": "error", "score": None,
+                "reason": f"unscoreable predictions: {type(e).__name__}: {e}"}
+    finally:
+        Path(path).unlink(missing_ok=True)
+    return {**base, "status": "ok", "score": report["overall"], "report": report,
+            "elapsed_s": round(time.time() - started, 1)}
+
+
+def run_scorecard(generators: dict[str, str], timeout: int = DEFAULT_TIMEOUT,
+                  max_workers: int = 4) -> dict:
+    """Fan out across targets, converge to one scorecard.
+
+    ``generators`` maps target_id -> generator command. Targets run in parallel
+    because they are genuinely independent; the scorecard is the convergence.
+    """
+    results: list[dict] = []
+    with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as pool:
+        futures = {pool.submit(run_target, tid, cmd, timeout): tid
+                   for tid, cmd in generators.items()}
+        for fut in concurrent.futures.as_completed(futures):
+            results.append(fut.result())
+    ok = [r for r in results if r["status"] == "ok"]
+    ranked = sorted(ok, key=lambda r: r["score"], reverse=True)
+    untested = [r for r in results if r["status"] != "ok"]
+    registry_total = len(list_targets())
+    return {
+        "kipimo_tasks": len(load_tasks()),
+        "targets_attempted": len(results),
+        "targets_scored": len(ok),
+        "targets_untested": len(untested),
+        "registry_targets": registry_total,
+        # coverage makes an incomplete run impossible to misread as complete
+        "coverage": f"{len(ok)}/{registry_total} registry targets scored",
+        "ranked": [{"target": r["target"], "score": r["score"],
+                    "by_type": {k: v for k, v in r["report"].items()
+                                if k not in ("overall", "n_tasks", "n_missing")},
+                    "n_missing": r["report"]["n_missing"]} for r in ranked],
+        "untested": [{"target": r["target"], "status": r["status"],
+                      "reason": r.get("reason", "")} for r in untested],
+        "caveat": ("Scores are stack-routing competence, not general Swahili fluency. "
+                   "Untested targets have UNKNOWN scores — not zero. A partial "
+                   "scorecard is not a ranking of the field."),
+    }
+
+
+def load_generators(path: str) -> dict[str, str]:
+    """Load a target_id -> command mapping from a JSON file."""
+    with open(path, encoding="utf-8") as f:
+        data = json.load(f)
+    if not isinstance(data, dict):
+        raise ValueError("generators file must be a JSON object: {target_id: command}")
+    return {str(k): str(v) for k, v in data.items()}

{kipimo-0.2.2 → kipimo-0.3.0/src/kipimo.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kipimo
-Version: 0.2.2
+Version: 0.3.0
 Summary: Swahili agent-task evaluation suite for the East Africa coordination stack — model-agnostic seed benchmark (46 tasks: server routing, term grounding, cascade routing).
 Author: AfriKaziOS maintainers
 License-Expression: MIT
@@ -65,3 +65,29 @@ MIT-licensed harness, CC BY 4.0 data. Feedback via GitHub Issues only — pull r
 
 Model-agnostic by design: closed APIs, open-weight models, and small distilled models are all first-class citizens.
 <!-- /interconnect:v1 -->
+
+## Running a scorecard (`kipimo run`)
+
+Fan out across targets in parallel, converge to one ranked scorecard. kipimo
+still never calls a model API — you supply a *generator command* per target
+(any process that reads tasks JSONL on stdin and writes predictions JSONL on
+stdout; see `examples/generate_predictions.py`).
+
+```bash
+cat > generators.json <<'JSON'
+{
+  "gemma-3-12b":    "python generate_predictions.py --model gemma-3-12b",
+  "inkubalm-0.4b":  "python generate_predictions.py --model inkubalm-0.4b",
+  "kimi-k3":        ""
+}
+JSON
+
+kipimo run generators.json --timeout 600 --workers 4
+```
+
+**An untested target is UNKNOWN, never zero.** If a generator crashes, times
+out, or is unconfigured, that target appears under `untested` with a reason —
+it is not ranked at 0.0. Conflating "the model failed the task" with "we never
+tested the model" is how an evaluation starts measuring its own assumptions
+instead of reality. Every scorecard also reports `coverage`, so a partial run
+cannot be misread as a ranking of the field.

{kipimo-0.2.2 → kipimo-0.3.0}/src/kipimo.egg-info/SOURCES.txt

@@ -3,6 +3,7 @@ README.md
 pyproject.toml
 src/kipimo/__init__.py
 src/kipimo/cli.py
+src/kipimo/harness.py
 src/kipimo/targets.py
 src/kipimo.egg-info/PKG-INFO
 src/kipimo.egg-info/SOURCES.txt
@@ -10,6 +11,7 @@ src/kipimo.egg-info/dependency_links.txt
 src/kipimo.egg-info/entry_points.txt
 src/kipimo.egg-info/top_level.txt
 src/kipimo/data/kipimo_v0.1.jsonl
+tests/test_harness.py
 tests/test_kipimo.py
 tests/test_smoke.py
 tests/test_targets.py

kipimo-0.3.0/tests/test_harness.py

@@ -0,0 +1,69 @@
+"""Harness tests — the load-bearing property is that UNTESTED is never scored as zero."""
+import json
+import sys
+
+from kipimo.harness import run_target, run_scorecard
+
+GOOD = f'{sys.executable} -c "import sys,json\nfor l in sys.stdin:\n    l=l.strip()\n    if l: t=json.loads(l); print(json.dumps({{\'id\':t[\'id\'],\'prediction\':[\'mpesa-mcp\']}}))"'
+CRASH = f'{sys.executable} -c "import sys; sys.exit(3)"'
+EMPTY = f'{sys.executable} -c "pass"'
+HANG = f'{sys.executable} -c "import time; time.sleep(60)"'
+
+
+def test_working_generator_scores():
+    r = run_target("gemma-3-12b", GOOD, timeout=60)
+    assert r["status"] == "ok"
+    assert isinstance(r["score"], float)
+    assert r["report"]["n_missing"] == 0
+
+
+def test_crashed_generator_is_error_not_zero():
+    r = run_target("gemma-3-12b", CRASH, timeout=60)
+    assert r["status"] == "error"
+    assert r["score"] is None          # NOT 0.0 — we did not test it
+    assert "exit 3" in r["reason"]
+
+
+def test_empty_output_is_error_not_zero():
+    r = run_target("gemma-3-12b", EMPTY, timeout=60)
+    assert r["status"] == "error"
+    assert r["score"] is None
+    assert "no predictions" in r["reason"]
+
+
+def test_timeout_is_recorded_not_zero():
+    r = run_target("gemma-3-12b", HANG, timeout=2)
+    assert r["status"] == "timeout"
+    assert r["score"] is None
+
+
+def test_unconfigured_target_is_skipped():
+    r = run_target("kimi-k3", "", timeout=5)
+    assert r["status"] == "skipped"
+    assert r["score"] is None
+
+
+def test_scorecard_separates_scored_from_untested():
+    card = run_scorecard({"gemma-3-12b": GOOD, "inkubalm-0.4b": CRASH, "kimi-k3": ""},
+                         timeout=30, max_workers=3)
+    assert card["targets_scored"] == 1
+    assert card["targets_untested"] == 2
+    ranked_ids = [r["target"] for r in card["ranked"]]
+    assert ranked_ids == ["gemma-3-12b"]          # only real results ranked
+    untested_ids = {r["target"] for r in card["untested"]}
+    assert untested_ids == {"inkubalm-0.4b", "kimi-k3"}
+    # coverage must make a partial run unmistakable
+    assert "/" in card["coverage"]
+    assert "not zero" in card["caveat"]
+
+
+def test_one_broken_target_does_not_abort_the_run():
+    card = run_scorecard({"a": CRASH, "b": GOOD}, timeout=30, max_workers=2)
+    assert card["targets_attempted"] == 2
+    assert card["targets_scored"] == 1            # the good one still ran
+
+
+def test_ranking_is_by_score_descending():
+    card = run_scorecard({"gemma-3-12b": GOOD, "qwen3-14b": GOOD}, timeout=30, max_workers=2)
+    scores = [r["score"] for r in card["ranked"]]
+    assert scores == sorted(scores, reverse=True)