kipimo 0.2.2__tar.gz → 0.4.0__tar.gz

{kipimo-0.2.2/src/kipimo.egg-info → kipimo-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kipimo
-Version: 0.2.2
+Version: 0.4.0
 Summary: Swahili agent-task evaluation suite for the East Africa coordination stack — model-agnostic seed benchmark (46 tasks: server routing, term grounding, cascade routing).
 Author: AfriKaziOS maintainers
 License-Expression: MIT
@@ -65,3 +65,51 @@ MIT-licensed harness, CC BY 4.0 data. Feedback via GitHub Issues only — pull r
 
 Model-agnostic by design: closed APIs, open-weight models, and small distilled models are all first-class citizens.
 <!-- /interconnect:v1 -->
+
+## Running a scorecard (`kipimo run`)
+
+Fan out across targets in parallel, converge to one ranked scorecard. kipimo
+still never calls a model API — you supply a *generator command* per target
+(any process that reads tasks JSONL on stdin and writes predictions JSONL on
+stdout; see `examples/generate_predictions.py`).
+
+```bash
+cat > generators.json <<'JSON'
+{
+  "gemma-3-12b":    "python generate_predictions.py --model gemma-3-12b",
+  "inkubalm-0.4b":  "python generate_predictions.py --model inkubalm-0.4b",
+  "kimi-k3":        ""
+}
+JSON
+
+kipimo run generators.json --timeout 600 --workers 4
+```
+
+**An untested target is UNKNOWN, never zero.** If a generator crashes, times
+out, or is unconfigured, that target appears under `untested` with a reason —
+it is not ranked at 0.0. Conflating "the model failed the task" with "we never
+tested the model" is how an evaluation starts measuring its own assumptions
+instead of reality. Every scorecard also reports `coverage`, so a partial run
+cannot be misread as a ranking of the field.
+
+## From leaderboard to deployment decision (`kipimo analyze`)
+
+A ranking says which model scored highest. Institutions need a different answer:
+**what is the cheapest model we may lawfully run ourselves that clears the bar?**
+
+```bash
+kipimo run generators.json > card.json
+kipimo analyze card.json --costs costs.json --threshold 0.85
+```
+
+Targets carry a deployment profile (`license`, `self_hostable`, `hardware_tier`,
+`offline_capable`), which `analyze` joins with measured accuracy and
+operator-supplied cost to report the Pareto frontier and the cheapest
+self-hostable qualifier:
+
+> *"gemma-3-12b is self-hostable, clears 0.85, and reaches 90% of the best
+> measured target's score."*
+
+Costs are supplied per run, never stored in the registry — vendor prices move
+weekly and a stale number in a public benchmark is worse than none. A target
+with no cost supplied is excluded from the frontier, **not treated as free**.

{kipimo-0.2.2 → kipimo-0.4.0}/README.md

@@ -47,3 +47,51 @@ MIT-licensed harness, CC BY 4.0 data. Feedback via GitHub Issues only — pull r
 
 Model-agnostic by design: closed APIs, open-weight models, and small distilled models are all first-class citizens.
 <!-- /interconnect:v1 -->
+
+## Running a scorecard (`kipimo run`)
+
+Fan out across targets in parallel, converge to one ranked scorecard. kipimo
+still never calls a model API — you supply a *generator command* per target
+(any process that reads tasks JSONL on stdin and writes predictions JSONL on
+stdout; see `examples/generate_predictions.py`).
+
+```bash
+cat > generators.json <<'JSON'
+{
+  "gemma-3-12b":    "python generate_predictions.py --model gemma-3-12b",
+  "inkubalm-0.4b":  "python generate_predictions.py --model inkubalm-0.4b",
+  "kimi-k3":        ""
+}
+JSON
+
+kipimo run generators.json --timeout 600 --workers 4
+```
+
+**An untested target is UNKNOWN, never zero.** If a generator crashes, times
+out, or is unconfigured, that target appears under `untested` with a reason —
+it is not ranked at 0.0. Conflating "the model failed the task" with "we never
+tested the model" is how an evaluation starts measuring its own assumptions
+instead of reality. Every scorecard also reports `coverage`, so a partial run
+cannot be misread as a ranking of the field.
+
+## From leaderboard to deployment decision (`kipimo analyze`)
+
+A ranking says which model scored highest. Institutions need a different answer:
+**what is the cheapest model we may lawfully run ourselves that clears the bar?**
+
+```bash
+kipimo run generators.json > card.json
+kipimo analyze card.json --costs costs.json --threshold 0.85
+```
+
+Targets carry a deployment profile (`license`, `self_hostable`, `hardware_tier`,
+`offline_capable`), which `analyze` joins with measured accuracy and
+operator-supplied cost to report the Pareto frontier and the cheapest
+self-hostable qualifier:
+
+> *"gemma-3-12b is self-hostable, clears 0.85, and reaches 90% of the best
+> measured target's score."*
+
+Costs are supplied per run, never stored in the registry — vendor prices move
+weekly and a stale number in a public benchmark is worse than none. A target
+with no cost supplied is excluded from the frontier, **not treated as free**.

{kipimo-0.2.2 → kipimo-0.4.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "kipimo"
-version = "0.2.2"
+version = "0.4.0"
 description = "Swahili agent-task evaluation suite for the East Africa coordination stack — model-agnostic seed benchmark (46 tasks: server routing, term grounding, cascade routing)."
 readme = "README.md"
 requires-python = ">=3.10"

{kipimo-0.2.2 → kipimo-0.4.0}/src/kipimo/__init__.py

@@ -8,3 +8,8 @@ from .targets import FAMILIES as FAMILIES
 from .targets import TARGETS as TARGETS
 from .targets import get_target as get_target
 from .targets import list_targets as list_targets
+from .harness import run_scorecard as run_scorecard
+from .harness import run_target as run_target
+from .pareto import analyze as analyze
+from .pareto import pareto_frontier as pareto_frontier
+from .pareto import cheapest_qualifier as cheapest_qualifier

{kipimo-0.2.2 → kipimo-0.4.0}/src/kipimo/cli.py

@@ -10,6 +10,8 @@ Usage:
     kipimo tasks                 # emit the task set (JSONL) to stdout
     kipimo template              # emit an empty predictions file to fill in
     kipimo score preds.jsonl     # score predictions against gold
+    kipimo run generators.json   # run targets in parallel -> ranked scorecard
+    kipimo analyze card.json     # -> Pareto frontier + cheapest sovereign qualifier
     kipimo targets               # emit the scorecard target registry (v0.2)
 """
 
@@ -21,7 +23,7 @@ import sys
 from collections import defaultdict
 from importlib import resources
 
-__version__ = "0.2.0"
+__version__ = "0.4.0"
 
 DISCLAIMER = ("kipimo v0.1 is a SEED benchmark (46 tasks). Swahili phrasing is "
               "simple-register and pending native-speaker review (issue #1). "
@@ -86,6 +88,14 @@ def main(argv: list[str] | None = None) -> int:
     tp.add_argument("--family", choices=("closed-api", "open-weight", "small-open"),
                     default=None, help="filter by target family")
     sub.add_parser("template", help="emit empty predictions JSONL to stdout")
+    rp = sub.add_parser("run", help="run generators across targets in parallel -> scorecard")
+    rp.add_argument("generators", help='JSON file: {"target_id": "generator command", ...}')
+    rp.add_argument("--timeout", type=int, default=600, help="seconds per target (default 600)")
+    rp.add_argument("--workers", type=int, default=4, help="parallel targets (default 4)")
+    ap = sub.add_parser("analyze", help="scorecard -> deployment decision (Pareto + cheapest qualifier)")
+    ap.add_argument("scorecard", help="JSON scorecard from `kipimo run`")
+    ap.add_argument("--costs", default=None, help='JSON file: {"target_id": cost_per_1k_tasks}')
+    ap.add_argument("--threshold", type=float, default=0.8, help="competence bar (default 0.8)")
     sp = sub.add_parser("score", help="score a predictions file")
     sp.add_argument("predictions", help="JSONL with {id, prediction:[...]} rows")
     args = p.parse_args(argv)
@@ -100,6 +110,25 @@ def main(argv: list[str] | None = None) -> int:
     elif args.cmd == "template":
         for t in load_tasks():
             print(json.dumps({"id": t["id"], "prediction": []}))
+    elif args.cmd == "analyze":
+        from .pareto import analyze
+        with open(args.scorecard, encoding="utf-8") as f:
+            card = json.load(f)
+        costs = {}
+        if args.costs:
+            with open(args.costs, encoding="utf-8") as f:
+                costs = {str(k): float(v) for k, v in json.load(f).items()}
+        print(json.dumps(analyze(card, costs, args.threshold), indent=2, ensure_ascii=False))
+        print(f"\n{DISCLAIMER}", file=sys.stderr)
+    elif args.cmd == "run":
+        from .harness import load_generators, run_scorecard
+        card = run_scorecard(load_generators(args.generators),
+                             timeout=args.timeout, max_workers=args.workers)
+        print(json.dumps(card, indent=2, ensure_ascii=False))
+        if card["targets_untested"]:
+            print(f"\nNote: {card['targets_untested']} target(s) UNTESTED (unknown, not zero). "
+                  f"See 'untested' in the scorecard.", file=sys.stderr)
+        print(f"\n{DISCLAIMER}", file=sys.stderr)
     else:
         rep = score_file(args.predictions)
         print(json.dumps(rep, indent=2))

kipimo-0.4.0/src/kipimo/harness.py

@@ -0,0 +1,145 @@
+"""kipimo.harness — run the benchmark across targets in parallel, converge to a scorecard.
+
+The diamond: fan out one worker per target, converge into a single ranked
+scorecard. Targets are independent (a real dependency test — no target's
+prediction informs another's), so they run concurrently; the scorecard is the
+only true convergence point.
+
+Two design rules, both deliberate:
+
+1. **kipimo still never calls a model API.** This module orchestrates *generator
+   commands you supply* (any process that reads tasks JSONL on stdin and writes
+   predictions JSONL on stdout). Model access, keys, and endpoints stay outside
+   the package, so the benchmark remains vendor-neutral and reproducible by
+   anyone. See examples/generate_predictions.py for a stdlib generator.
+
+2. **A missing run is never a zero.** The scoring path already treats an absent
+   prediction as 0.0, which is correct *within* a run. Across targets it would
+   be a lie: a target whose generator crashed, timed out, or was never
+   configured has an UNKNOWN score, not a bad one. Conflating "the model failed
+   the task" with "we never tested the model" is how an evaluation system starts
+   training on its own assumptions instead of on evidence. Every result
+   therefore carries an explicit ``status``:
+
+       ok         — generator ran, predictions scored
+       error      — generator failed (message retained)
+       timeout    — generator exceeded its deadline
+       skipped    — no generator configured for this target
+
+   Only ``ok`` results carry a score. The scorecard reports coverage alongside
+   results so a partial run can never be read as a complete one.
+"""
+from __future__ import annotations
+
+import concurrent.futures
+import json
+import shlex
+import subprocess
+import tempfile
+import time
+from pathlib import Path
+
+from .cli import load_tasks, score_file
+from .targets import get_target, list_targets
+
+DEFAULT_TIMEOUT = 600  # seconds per target; generators are network-bound
+
+
+def _tasks_jsonl() -> str:
+    return "\n".join(json.dumps(t, ensure_ascii=False) for t in load_tasks())
+
+
+def run_target(target_id: str, command: str, timeout: int = DEFAULT_TIMEOUT) -> dict:
+    """Run one generator command and score its predictions.
+
+    ``command`` is a shell-style command that reads tasks JSONL on stdin and
+    writes predictions JSONL ({"id":..., "prediction":[...]}) on stdout.
+    Returns a result dict with an explicit status; never raises for generator
+    failure (a broken target must not abort the whole scorecard).
+    """
+    started = time.time()
+    base = {"target": target_id, "command": command}
+    try:
+        get_target(target_id)  # validate against the registry
+    except Exception:
+        base["registry"] = "unregistered"
+    if not command:
+        return {**base, "status": "skipped", "score": None,
+                "reason": "no generator command configured"}
+    try:
+        proc = subprocess.run(
+            shlex.split(command), input=_tasks_jsonl(), text=True,
+            capture_output=True, timeout=timeout,
+        )
+    except subprocess.TimeoutExpired:
+        return {**base, "status": "timeout", "score": None,
+                "elapsed_s": round(time.time() - started, 1),
+                "reason": f"exceeded {timeout}s"}
+    except Exception as e:  # generator binary missing, bad command, etc.
+        return {**base, "status": "error", "score": None,
+                "reason": f"{type(e).__name__}: {e}"}
+    if proc.returncode != 0:
+        return {**base, "status": "error", "score": None,
+                "reason": f"exit {proc.returncode}: {(proc.stderr or '').strip()[:200]}"}
+    if not proc.stdout.strip():
+        return {**base, "status": "error", "score": None,
+                "reason": "generator produced no predictions"}
+    with tempfile.NamedTemporaryFile("w", suffix=".jsonl", delete=False,
+                                     encoding="utf-8") as fh:
+        fh.write(proc.stdout)
+        path = fh.name
+    try:
+        report = score_file(path)
+    except Exception as e:
+        return {**base, "status": "error", "score": None,
+                "reason": f"unscoreable predictions: {type(e).__name__}: {e}"}
+    finally:
+        Path(path).unlink(missing_ok=True)
+    return {**base, "status": "ok", "score": report["overall"], "report": report,
+            "elapsed_s": round(time.time() - started, 1)}
+
+
+def run_scorecard(generators: dict[str, str], timeout: int = DEFAULT_TIMEOUT,
+                  max_workers: int = 4) -> dict:
+    """Fan out across targets, converge to one scorecard.
+
+    ``generators`` maps target_id -> generator command. Targets run in parallel
+    because they are genuinely independent; the scorecard is the convergence.
+    """
+    results: list[dict] = []
+    with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as pool:
+        futures = {pool.submit(run_target, tid, cmd, timeout): tid
+                   for tid, cmd in generators.items()}
+        for fut in concurrent.futures.as_completed(futures):
+            results.append(fut.result())
+    ok = [r for r in results if r["status"] == "ok"]
+    ranked = sorted(ok, key=lambda r: r["score"], reverse=True)
+    untested = [r for r in results if r["status"] != "ok"]
+    registry_total = len(list_targets())
+    return {
+        "kipimo_tasks": len(load_tasks()),
+        "targets_attempted": len(results),
+        "targets_scored": len(ok),
+        "targets_untested": len(untested),
+        "registry_targets": registry_total,
+        # coverage makes an incomplete run impossible to misread as complete
+        "coverage": f"{len(ok)}/{registry_total} registry targets scored",
+        "ranked": [{"target": r["target"], "score": r["score"],
+                    "by_type": {k: v for k, v in r["report"].items()
+                                if k not in ("overall", "n_tasks", "n_missing")},
+                    "n_missing": r["report"]["n_missing"]} for r in ranked],
+        "untested": [{"target": r["target"], "status": r["status"],
+                      "reason": r.get("reason", "")} for r in untested],
+        "caveat": ("Scores are stack-routing competence, not general Swahili fluency. "
+                   "Untested targets have UNKNOWN scores — not zero. A partial "
+                   "scorecard is not a ranking of the field."),
+    }
+
+
+def load_generators(path: str) -> dict[str, str]:
+    """Load a target_id -> command mapping from a JSON file."""
+    with open(path, encoding="utf-8") as f:
+        data = json.load(f)
+    if not isinstance(data, dict):
+        raise ValueError("generators file must be a JSON object: {target_id: command}")
+    return {str(k): str(v) for k, v in data.items()}

kipimo-0.4.0/src/kipimo/pareto.py

@@ -0,0 +1,139 @@
+"""kipimo.pareto — turn a scorecard into a deployment decision.
+
+A leaderboard answers "which model scored highest?". That is rarely the question
+an African institution faces. Theirs is:
+
+    What is the CHEAPEST model we are allowed to run OURSELVES that clears the
+    competence bar for this task?
+
+Answering it needs three things a score alone does not carry: what the model
+costs to run, whether the licence permits self-hosting, and what hardware it
+demands. This module joins measured accuracy (from the harness) with the
+registry's deployment profile and operator-supplied cost, then reports:
+
+  * the **Pareto frontier** — targets not beaten on BOTH accuracy and cost by
+    some other target. Everything off the frontier is strictly dominated and can
+    be dropped from consideration without argument.
+  * the **cheapest qualifier** at a competence threshold, optionally restricted
+    to self-hostable / offline-capable / hardware-tier-limited targets. This is
+    the sovereign-deployment answer.
+
+Cost is passed in per run, never stored in the registry: vendor prices move
+weekly and differ by region, and a stale number in a public benchmark is worse
+than no number. Targets without a cost are ranked on accuracy and reported
+separately rather than silently assigned zero — the same discipline the harness
+applies to untested targets.
+"""
+from __future__ import annotations
+
+from .targets import get_target
+
+HARDWARE_ORDER = ["edge", "workstation", "server", "datacenter", "api-only"]
+
+
+def _profile(target_id: str) -> dict:
+    try:
+        t = get_target(target_id)
+    except Exception:
+        return {}
+    return {k: t.get(k) for k in
+            ("license", "self_hostable", "hardware_tier", "offline_capable", "family")}
+
+
+def build_rows(scorecard: dict, costs: dict[str, float] | None = None) -> list[dict]:
+    """Join harness results with registry profiles and operator-supplied costs.
+
+    ``costs`` maps target_id -> cost per 1k tasks (any consistent currency).
+    """
+    costs = costs or {}
+    rows = []
+    for r in scorecard.get("ranked", []):
+        tid = r["target"]
+        rows.append({"target": tid, "score": r["score"],
+                     "cost_per_1k": costs.get(tid),  # None = not supplied, not free
+                     **_profile(tid)})
+    return rows
+
+
+def pareto_frontier(rows: list[dict]) -> list[dict]:
+    """Targets not dominated on both accuracy (higher better) and cost (lower better).
+
+    Rows without a cost cannot be compared on cost and are excluded here; they
+    are reported separately by ``analyze`` rather than dropped silently.
+    """
+    priced = [r for r in rows if r.get("cost_per_1k") is not None and r.get("score") is not None]
+    frontier = []
+    for r in priced:
+        dominated = any(
+            o is not r
+            and o["score"] >= r["score"]
+            and o["cost_per_1k"] <= r["cost_per_1k"]
+            and (o["score"] > r["score"] or o["cost_per_1k"] < r["cost_per_1k"])
+            for o in priced
+        )
+        if not dominated:
+            frontier.append(r)
+    return sorted(frontier, key=lambda r: r["cost_per_1k"])
+
+
+def cheapest_qualifier(rows: list[dict], threshold: float, *,
+                       self_hostable: bool | None = None,
+                       offline_capable: bool | None = None,
+                       max_hardware_tier: str | None = None) -> dict | None:
+    """Cheapest target clearing ``threshold``, under deployment constraints.
+
+    With ``self_hostable=True`` this answers the sovereignty question directly:
+    the cheapest model an institution may lawfully run on its own hardware while
+    still clearing the competence bar.
+    """
+    cands = [r for r in rows if (r.get("score") or 0) >= threshold]
+    if self_hostable is not None:
+        cands = [r for r in cands if r.get("self_hostable") is self_hostable]
+    if offline_capable is not None:
+        cands = [r for r in cands if r.get("offline_capable") is offline_capable]
+    if max_hardware_tier:
+        limit = HARDWARE_ORDER.index(max_hardware_tier)
+        cands = [r for r in cands
+                 if r.get("hardware_tier") in HARDWARE_ORDER
+                 and HARDWARE_ORDER.index(r["hardware_tier"]) <= limit]
+    if not cands:
+        return None
+    priced = [r for r in cands if r.get("cost_per_1k") is not None]
+    # cheapest if costs known; otherwise the qualifier on the smallest hardware
+    if priced:
+        return min(priced, key=lambda r: r["cost_per_1k"])
+    return min(cands, key=lambda r: HARDWARE_ORDER.index(r.get("hardware_tier") or "api-only"))
+
+
+def analyze(scorecard: dict, costs: dict[str, float] | None = None,
+            threshold: float = 0.8) -> dict:
+    """Full deployment analysis over a harness scorecard."""
+    rows = build_rows(scorecard, costs)
+    unpriced = [r["target"] for r in rows if r.get("cost_per_1k") is None]
+    sovereign = cheapest_qualifier(rows, threshold, self_hostable=True)
+    edge = cheapest_qualifier(rows, threshold, self_hostable=True,
+                              max_hardware_tier="workstation")
+    best = max(rows, key=lambda r: r["score"]) if rows else None
+    gap = None
+    if best and sovereign and best["score"]:
+        gap = round(sovereign["score"] / best["score"], 3)
+    return {
+        "threshold": threshold,
+        "rows": rows,
+        "pareto_frontier": [r["target"] for r in pareto_frontier(rows)],
+        "unpriced_targets": unpriced,
+        "best_overall": best["target"] if best else None,
+        "cheapest_sovereign_qualifier": sovereign["target"] if sovereign else None,
+        "cheapest_edge_qualifier": edge["target"] if edge else None,
+        "sovereign_vs_frontier_ratio": gap,
+        "reading": (
+            f"{sovereign['target']} is self-hostable, clears {threshold}, and reaches "
+            f"{int(gap*100)}% of the best measured target's score."
+            if sovereign and gap else
+            f"No self-hostable target clears {threshold} in this run."
+        ),
+        "caveat": ("Costs are operator-supplied for this run, not registry facts. "
+                   "Unpriced targets are excluded from the Pareto frontier — not "
+                   "treated as free. Licences marked '(verify)' must be confirmed "
+                   "on the model's own repository before deployment."),
+    }

{kipimo-0.2.2 → kipimo-0.4.0}/src/kipimo/targets.py

@@ -15,6 +15,24 @@ Families:
 
 `endpoint_env` names the environment variables an external generator is
 expected to read (OpenAI-compatible convention). Nothing here reads them.
+
+DEPLOYMENT PROFILE (optional, per target)
+-----------------------------------------
+Accuracy alone cannot answer the question East African institutions actually
+face, which is not "which model is best?" but "what is the cheapest model we
+can run ourselves that clears the competence bar?" Targets may therefore carry:
+
+    license        SPDX-ish string ("MIT", "Apache-2.0", "CC-BY-NC-4.0",
+                   "proprietary", "custom") — governs whether self-hosting and
+                   commercial/civic redeployment are even permitted
+    self_hostable  bool — weights obtainable and runnable outside a vendor API
+    hardware_tier  "edge" | "workstation" | "server" | "datacenter" | "api-only"
+    offline_capable bool — can serve with no internet at inference time
+
+Cost and latency are deliberately NOT stored here. Vendor prices change weekly
+and vary by region; a number frozen in a registry is a number that quietly goes
+wrong. Cost is supplied per-run by the operator (see kipimo.pareto) and latency
+is measured by the harness. The registry holds only slow-moving facts.
 """
 
 from __future__ import annotations
@@ -30,6 +48,10 @@ TARGETS: list[dict] = [
         "params_b": None,
         "endpoint_env": ["ANTHROPIC_API_KEY"],
         "notes": "reference frontier closed model",
+        "license": "proprietary",
+        "self_hostable": False,
+        "hardware_tier": "api-only",
+        "offline_capable": False,
     },
     {
         "id": "gpt-5.6",
@@ -38,6 +60,10 @@ TARGETS: list[dict] = [
         "params_b": None,
         "endpoint_env": ["OPENAI_API_KEY"],
         "notes": "reference frontier closed model",
+        "license": "proprietary",
+        "self_hostable": False,
+        "hardware_tier": "api-only",
+        "offline_capable": False,
     },
     # --- open weights (frontier-adjacent, downloadable) ---
     {
@@ -47,6 +73,10 @@ TARGETS: list[dict] = [
         "params_b": 1000,
         "endpoint_env": ["KIPIMO_BASE_URL", "KIPIMO_API_KEY"],
         "notes": "weights public (HF, ~900k downloads); 1T MoE; strong agentic/tool-use; self-hostable via vLLM/SGLang/Ollama",
+        "license": "Modified-MIT",
+        "self_hostable": True,
+        "hardware_tier": "datacenter",
+        "offline_capable": True,
     },
     {
         "id": "kimi-k3",
@@ -55,6 +85,10 @@ TARGETS: list[dict] = [
         "params_b": 2800,
         "endpoint_env": ["KIPIMO_BASE_URL", "KIPIMO_API_KEY"],
         "notes": "2.8T MoE, 104B active; weights PUBLIC on HF (moonshotai/Kimi-K3, native multimodal, 1M ctx) under the custom \"Kimi K3 License\" — read LICENSE before redistribution (NOT MIT/Apache); strong agentic/MCP scores make it a valid frontier reference",
+        "license": "custom (Kimi K3 License)",
+        "self_hostable": True,
+        "hardware_tier": "datacenter",
+        "offline_capable": True,
     },
     {
         "id": "glm-5.2",
@@ -63,6 +97,10 @@ TARGETS: list[dict] = [
         "params_b": None,
         "endpoint_env": ["KIPIMO_BASE_URL", "KIPIMO_API_KEY"],
         "notes": "leads open-weight intelligence indices at time of listing",
+        "license": "MIT",
+        "self_hostable": True,
+        "hardware_tier": "datacenter",
+        "offline_capable": True,
     },
     {
         "id": "deepseek-v4-pro",
@@ -71,6 +109,10 @@ TARGETS: list[dict] = [
         "params_b": None,
         "endpoint_env": ["KIPIMO_BASE_URL", "KIPIMO_API_KEY"],
         "notes": "measurable regional install base (11-14% share in four African markets)",
+        "license": "MIT (verify on repo)",
+        "self_hostable": True,
+        "hardware_tier": "datacenter",
+        "offline_capable": True,
     },
     {
         "id": "nemotron-3-ultra",
@@ -79,6 +121,10 @@ TARGETS: list[dict] = [
         "params_b": None,
         "endpoint_env": ["KIPIMO_BASE_URL", "KIPIMO_API_KEY"],
         "notes": "US open-weight entrant; ships with open data and recipes",
+        "license": "NVIDIA-Open-Model (verify)",
+        "self_hostable": True,
+        "hardware_tier": "datacenter",
+        "offline_capable": True,
     },
     # --- small open (the deployment tier nobody measures) ---
     {
@@ -88,6 +134,10 @@ TARGETS: list[dict] = [
         "params_b": 14,
         "endpoint_env": ["KIPIMO_BASE_URL", "KIPIMO_API_KEY"],
         "notes": "runnable on a single consumer GPU",
+        "license": "Apache-2.0",
+        "self_hostable": True,
+        "hardware_tier": "workstation",
+        "offline_capable": True,
     },
     {
         "id": "llama-4-scout",
@@ -96,6 +146,10 @@ TARGETS: list[dict] = [
         "params_b": 17,
         "endpoint_env": ["KIPIMO_BASE_URL", "KIPIMO_API_KEY"],
         "notes": "active-parameter class; edge-adjacent",
+        "license": "Llama-4-Community",
+        "self_hostable": True,
+        "hardware_tier": "server",
+        "offline_capable": True,
     },
     {
         "id": "gemma-3-12b",
@@ -104,6 +158,10 @@ TARGETS: list[dict] = [
         "params_b": 12,
         "endpoint_env": ["KIPIMO_BASE_URL", "KIPIMO_API_KEY"],
         "notes": "runnable on a single consumer GPU",
+        "license": "Gemma-Terms",
+        "self_hostable": True,
+        "hardware_tier": "workstation",
+        "offline_capable": True,
     },
     {
         "id": "tiny-aya-earth",
@@ -117,6 +175,10 @@ TARGETS: list[dict] = [
             "CC-BY-NC-4.0. Sits between InkubaLM 0.4B and the ~12B tier. Verify exact "
             "HF repo id before use."
         ),
+        "license": "CC-BY-NC-4.0",
+        "self_hostable": True,
+        "hardware_tier": "edge",
+        "offline_capable": True,
     },
     {
         "id": "inkubalm-0.4b",
@@ -131,6 +193,10 @@ TARGETS: list[dict] = [
             "non-commercial redistribution. The floor of the deployment tier this "
             "benchmark exists to measure."
         ),
+        "license": "CC-BY-NC-4.0",
+        "self_hostable": True,
+        "hardware_tier": "edge",
+        "offline_capable": True,
     },
 ]
 

{kipimo-0.2.2 → kipimo-0.4.0/src/kipimo.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kipimo
-Version: 0.2.2
+Version: 0.4.0
 Summary: Swahili agent-task evaluation suite for the East Africa coordination stack — model-agnostic seed benchmark (46 tasks: server routing, term grounding, cascade routing).
 Author: AfriKaziOS maintainers
 License-Expression: MIT
@@ -65,3 +65,51 @@ MIT-licensed harness, CC BY 4.0 data. Feedback via GitHub Issues only — pull r
 
 Model-agnostic by design: closed APIs, open-weight models, and small distilled models are all first-class citizens.
 <!-- /interconnect:v1 -->
+
+## Running a scorecard (`kipimo run`)
+
+Fan out across targets in parallel, converge to one ranked scorecard. kipimo
+still never calls a model API — you supply a *generator command* per target
+(any process that reads tasks JSONL on stdin and writes predictions JSONL on
+stdout; see `examples/generate_predictions.py`).
+
+```bash
+cat > generators.json <<'JSON'
+{
+  "gemma-3-12b":    "python generate_predictions.py --model gemma-3-12b",
+  "inkubalm-0.4b":  "python generate_predictions.py --model inkubalm-0.4b",
+  "kimi-k3":        ""
+}
+JSON
+
+kipimo run generators.json --timeout 600 --workers 4
+```
+
+**An untested target is UNKNOWN, never zero.** If a generator crashes, times
+out, or is unconfigured, that target appears under `untested` with a reason —
+it is not ranked at 0.0. Conflating "the model failed the task" with "we never
+tested the model" is how an evaluation starts measuring its own assumptions
+instead of reality. Every scorecard also reports `coverage`, so a partial run
+cannot be misread as a ranking of the field.
+
+## From leaderboard to deployment decision (`kipimo analyze`)
+
+A ranking says which model scored highest. Institutions need a different answer:
+**what is the cheapest model we may lawfully run ourselves that clears the bar?**
+
+```bash
+kipimo run generators.json > card.json
+kipimo analyze card.json --costs costs.json --threshold 0.85
+```
+
+Targets carry a deployment profile (`license`, `self_hostable`, `hardware_tier`,
+`offline_capable`), which `analyze` joins with measured accuracy and
+operator-supplied cost to report the Pareto frontier and the cheapest
+self-hostable qualifier:
+
+> *"gemma-3-12b is self-hostable, clears 0.85, and reaches 90% of the best
+> measured target's score."*
+
+Costs are supplied per run, never stored in the registry — vendor prices move
+weekly and a stale number in a public benchmark is worse than none. A target
+with no cost supplied is excluded from the frontier, **not treated as free**.

{kipimo-0.2.2 → kipimo-0.4.0}/src/kipimo.egg-info/SOURCES.txt

@@ -3,6 +3,8 @@ README.md
 pyproject.toml
 src/kipimo/__init__.py
 src/kipimo/cli.py
+src/kipimo/harness.py
+src/kipimo/pareto.py
 src/kipimo/targets.py
 src/kipimo.egg-info/PKG-INFO
 src/kipimo.egg-info/SOURCES.txt
@@ -10,6 +12,8 @@ src/kipimo.egg-info/dependency_links.txt
 src/kipimo.egg-info/entry_points.txt
 src/kipimo.egg-info/top_level.txt
 src/kipimo/data/kipimo_v0.1.jsonl
+tests/test_harness.py
 tests/test_kipimo.py
+tests/test_pareto.py
 tests/test_smoke.py
 tests/test_targets.py

kipimo-0.4.0/tests/test_harness.py

@@ -0,0 +1,69 @@
+"""Harness tests — the load-bearing property is that UNTESTED is never scored as zero."""
+import json
+import sys
+
+from kipimo.harness import run_target, run_scorecard
+
+GOOD = f'{sys.executable} -c "import sys,json\nfor l in sys.stdin:\n    l=l.strip()\n    if l: t=json.loads(l); print(json.dumps({{\'id\':t[\'id\'],\'prediction\':[\'mpesa-mcp\']}}))"'
+CRASH = f'{sys.executable} -c "import sys; sys.exit(3)"'
+EMPTY = f'{sys.executable} -c "pass"'
+HANG = f'{sys.executable} -c "import time; time.sleep(60)"'
+
+
+def test_working_generator_scores():
+    r = run_target("gemma-3-12b", GOOD, timeout=60)
+    assert r["status"] == "ok"
+    assert isinstance(r["score"], float)
+    assert r["report"]["n_missing"] == 0
+
+
+def test_crashed_generator_is_error_not_zero():
+    r = run_target("gemma-3-12b", CRASH, timeout=60)
+    assert r["status"] == "error"
+    assert r["score"] is None          # NOT 0.0 — we did not test it
+    assert "exit 3" in r["reason"]
+
+
+def test_empty_output_is_error_not_zero():
+    r = run_target("gemma-3-12b", EMPTY, timeout=60)
+    assert r["status"] == "error"
+    assert r["score"] is None
+    assert "no predictions" in r["reason"]
+
+
+def test_timeout_is_recorded_not_zero():
+    r = run_target("gemma-3-12b", HANG, timeout=2)
+    assert r["status"] == "timeout"
+    assert r["score"] is None
+
+
+def test_unconfigured_target_is_skipped():
+    r = run_target("kimi-k3", "", timeout=5)
+    assert r["status"] == "skipped"
+    assert r["score"] is None
+
+
+def test_scorecard_separates_scored_from_untested():
+    card = run_scorecard({"gemma-3-12b": GOOD, "inkubalm-0.4b": CRASH, "kimi-k3": ""},
+                         timeout=30, max_workers=3)
+    assert card["targets_scored"] == 1
+    assert card["targets_untested"] == 2
+    ranked_ids = [r["target"] for r in card["ranked"]]
+    assert ranked_ids == ["gemma-3-12b"]          # only real results ranked
+    untested_ids = {r["target"] for r in card["untested"]}
+    assert untested_ids == {"inkubalm-0.4b", "kimi-k3"}
+    # coverage must make a partial run unmistakable
+    assert "/" in card["coverage"]
+    assert "not zero" in card["caveat"]
+
+
+def test_one_broken_target_does_not_abort_the_run():
+    card = run_scorecard({"a": CRASH, "b": GOOD}, timeout=30, max_workers=2)
+    assert card["targets_attempted"] == 2
+    assert card["targets_scored"] == 1            # the good one still ran
+
+
+def test_ranking_is_by_score_descending():
+    card = run_scorecard({"gemma-3-12b": GOOD, "qwen3-14b": GOOD}, timeout=30, max_workers=2)
+    scores = [r["score"] for r in card["ranked"]]
+    assert scores == sorted(scores, reverse=True)

kipimo-0.4.0/tests/test_pareto.py

@@ -0,0 +1,55 @@
+"""Pareto/deployment tests — the sovereignty question, and honesty about unknowns."""
+from kipimo.pareto import analyze, build_rows, cheapest_qualifier, pareto_frontier
+
+CARD = {"ranked": [
+    {"target": "gpt-5.6", "score": 0.95}, {"target": "glm-5.2", "score": 0.93},
+    {"target": "qwen3-14b", "score": 0.88}, {"target": "gemma-3-12b", "score": 0.86},
+    {"target": "inkubalm-0.4b", "score": 0.61}], "untested": []}
+COSTS = {"gpt-5.6": 12.5, "glm-5.2": 1.8, "qwen3-14b": 0.09,
+         "gemma-3-12b": 0.05, "inkubalm-0.4b": 0.01}
+
+
+def test_rows_join_registry_profile():
+    rows = build_rows(CARD, COSTS)
+    g = next(r for r in rows if r["target"] == "glm-5.2")
+    assert g["license"] == "MIT" and g["self_hostable"] is True
+
+
+def test_dominated_target_is_off_the_frontier():
+    # a target both worse AND pricier than another must be excluded
+    card = {"ranked": [{"target": "qwen3-14b", "score": 0.88},
+                       {"target": "gemma-3-12b", "score": 0.70}]}
+    front = pareto_frontier(build_rows(card, {"qwen3-14b": 0.09, "gemma-3-12b": 0.50}))
+    assert [r["target"] for r in front] == ["qwen3-14b"]
+
+
+def test_cheapest_sovereign_qualifier_excludes_api_only():
+    rows = build_rows(CARD, COSTS)
+    pick = cheapest_qualifier(rows, 0.85, self_hostable=True)
+    assert pick["target"] == "gemma-3-12b"       # cheapest self-hostable clearing 0.85
+    assert pick["self_hostable"] is True
+
+
+def test_proprietary_wins_overall_but_not_sovereignty():
+    a = analyze(CARD, COSTS, threshold=0.85)
+    assert a["best_overall"] == "gpt-5.6"                    # highest score
+    assert a["cheapest_sovereign_qualifier"] == "gemma-3-12b"  # but not deployable
+    assert 0.8 < a["sovereign_vs_frontier_ratio"] < 1.0
+
+
+def test_hardware_tier_constraint_respected():
+    rows = build_rows(CARD, COSTS)
+    pick = cheapest_qualifier(rows, 0.85, self_hostable=True, max_hardware_tier="workstation")
+    assert pick["hardware_tier"] in ("edge", "workstation")   # datacenter excluded
+
+
+def test_threshold_with_no_qualifier_reports_honestly():
+    a = analyze(CARD, COSTS, threshold=0.99)
+    assert a["cheapest_sovereign_qualifier"] is None
+    assert "No self-hostable target clears" in a["reading"]
+
+
+def test_unpriced_target_is_not_treated_as_free():
+    a = analyze(CARD, {"gpt-5.6": 12.5}, threshold=0.85)   # only one cost supplied
+    assert "qwen3-14b" in a["unpriced_targets"]
+    assert "qwen3-14b" not in a["pareto_frontier"]         # excluded, not ranked cheapest

{kipimo-0.2.2 → kipimo-0.4.0}/tests/test_targets.py

@@ -6,15 +6,34 @@ from kipimo import FAMILIES, TARGETS, get_target, list_targets
 from kipimo.cli import main
 
 
+CORE = {"id", "label", "family", "params_b", "endpoint_env", "notes"}
+PROFILE = {"license", "self_hostable", "hardware_tier", "offline_capable"}
+HARDWARE = {"edge", "workstation", "server", "datacenter", "api-only"}
+
+
 def test_schema_and_uniqueness():
     ids = [t["id"] for t in TARGETS]
     assert len(ids) == len(set(ids))
     for t in TARGETS:
-        assert set(t) == {"id", "label", "family", "params_b", "endpoint_env", "notes"}
+        assert CORE <= set(t), f"{t['id']} missing core keys"
+        assert set(t) <= CORE | PROFILE, f"{t['id']} has unknown keys"
         assert t["family"] in FAMILIES
         assert isinstance(t["endpoint_env"], list) and t["endpoint_env"]
 
 
+def test_deployment_profile_is_valid_where_present():
+    for t in TARGETS:
+        if "hardware_tier" in t:
+            assert t["hardware_tier"] in HARDWARE, t["id"]
+        if "self_hostable" in t:
+            assert isinstance(t["self_hostable"], bool)
+        if "offline_capable" in t:
+            assert isinstance(t["offline_capable"], bool)
+        # an api-only target can be neither self-hostable nor offline
+        if t.get("hardware_tier") == "api-only":
+            assert t.get("self_hostable") is False and t.get("offline_capable") is False, t["id"]
+
+
 def test_every_family_represented():
     for fam in FAMILIES:
         assert list_targets(fam), f"no targets in family {fam}"