june-bench 0.0.1__tar.gz

june_bench-0.0.1/PKG-INFO

@@ -0,0 +1,58 @@
+Metadata-Version: 2.4
+Name: june-bench
+Version: 0.0.1
+Summary: Reproducible benchmark suite for memory/QA systems — June + pluggable competitors.
+Author: Junemind
+License: Proprietary
+Project-URL: Homepage, https://github.com/Junemind
+Keywords: benchmark,rag,memory,qa,retrieval,evaluation,llm,june
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Operating System :: OS Independent
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: License :: Other/Proprietary License
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Provides-Extra: june-api
+Requires-Dist: httpx>=0.24; extra == "june-api"
+Provides-Extra: june-local
+Requires-Dist: june>=0.1.0; extra == "june-local"
+Provides-Extra: cognee
+Requires-Dist: cognee[evals]; extra == "cognee"
+Provides-Extra: all
+Requires-Dist: httpx>=0.24; extra == "all"
+Provides-Extra: dev
+Requires-Dist: pytest>=8; extra == "dev"
+
+# june-bench
+
+A pip-installable, **reproducible** benchmark suite for memory / QA systems — **June + pluggable
+competitors** — over LoCoMo, LongMemEval, HotpotQA/2Wiki/MuSiQue, and FinanceBench, with the same
+data and the same scorer.
+
+```bash
+pip install june-bench
+june-bench list
+june-bench run --system echo --dataset smoke --split smoke    # offline, no key, no download
+```
+
+A benchmark is `run(system, dataset) → records → score`. Two typed ports are the only extension
+points:
+
+* **`System`** — the thing benchmarked. `JuneApiSystem` (default; a thin HTTP client to June's
+  `/v1/answer`, so **no June source is shipped**), `JuneLocalSystem` (`[june-local]` extra; a
+  source-protected compiled wheel), `CogneeSystem` (`[cognee]` extra), or any future system as one
+  adapter.
+* **`Dataset`** — what it runs on. The four benchmarks behind a registry.
+
+The scorer is the canonical SQuAD/HotpotQA EM/F1 + selective-accuracy/coverage/cost — Cognee-comparable.
+Tiny **smoke fixtures ship in the wheel** (offline wiring proof); full splits are **fetched, sha-verified,
+from a pinned release**. No score is ever baked into the package — every result row records
+dataset + scorer + system + model + cost, so a published number is reproducible by a stranger.
+
+Status: **SB0** (contracts + no-deps smoke + skeleton). Datasets, June/Cognee systems, and the full CLI
+land in SB1–SB6.

june_bench-0.0.1/README.md

@@ -0,0 +1,28 @@
+# june-bench
+
+A pip-installable, **reproducible** benchmark suite for memory / QA systems — **June + pluggable
+competitors** — over LoCoMo, LongMemEval, HotpotQA/2Wiki/MuSiQue, and FinanceBench, with the same
+data and the same scorer.
+
+```bash
+pip install june-bench
+june-bench list
+june-bench run --system echo --dataset smoke --split smoke    # offline, no key, no download
+```
+
+A benchmark is `run(system, dataset) → records → score`. Two typed ports are the only extension
+points:
+
+* **`System`** — the thing benchmarked. `JuneApiSystem` (default; a thin HTTP client to June's
+  `/v1/answer`, so **no June source is shipped**), `JuneLocalSystem` (`[june-local]` extra; a
+  source-protected compiled wheel), `CogneeSystem` (`[cognee]` extra), or any future system as one
+  adapter.
+* **`Dataset`** — what it runs on. The four benchmarks behind a registry.
+
+The scorer is the canonical SQuAD/HotpotQA EM/F1 + selective-accuracy/coverage/cost — Cognee-comparable.
+Tiny **smoke fixtures ship in the wheel** (offline wiring proof); full splits are **fetched, sha-verified,
+from a pinned release**. No score is ever baked into the package — every result row records
+dataset + scorer + system + model + cost, so a published number is reproducible by a stranger.
+
+Status: **SB0** (contracts + no-deps smoke + skeleton). Datasets, June/Cognee systems, and the full CLI
+land in SB1–SB6.

june_bench-0.0.1/june_bench/__init__.py

@@ -0,0 +1,19 @@
+"""june_bench — a pip-installable, reproducible benchmark suite for memory/QA systems.
+
+`run(system, dataset) → records → score`. Two typed ports (`System`, `Dataset`) are the only
+extension points; June and pluggable competitors plug in behind `System`, the four benchmarks behind
+`Dataset`. The harness core is pure (no model, no network) and unit-tests with the bundled `smoke`
+fixtures; June is reached over its REST API by default (no June source shipped).
+"""
+from __future__ import annotations
+
+from june_bench.ports import Dataset, Example, Prediction, Record, System
+from june_bench.runner import run, run_async
+from june_bench.score import score
+
+__version__ = "0.0.1"
+
+__all__ = [
+    "Example", "Prediction", "Record", "System", "Dataset",
+    "run", "run_async", "score", "__version__",
+]

june_bench-0.0.1/june_bench/cli.py

@@ -0,0 +1,104 @@
+"""june-bench CLI (SB0) — `run`, `list`. SB5 adds `suite`/`score`/`report`.
+
+    june-bench list
+    june-bench run --system echo --dataset smoke --split smoke
+    june-bench run --system null --dataset smoke --json
+
+The `echo`/`null` systems + the bundled `smoke` dataset need no deps, key, or network, so this runs
+offline immediately after `pip install june-bench`.
+"""
+from __future__ import annotations
+
+import argparse
+import sys
+
+import pathlib
+
+from june_bench import datasets, systems
+from june_bench.report import suite_json, suite_markdown, to_json, to_markdown
+from june_bench.runner import run
+from june_bench.score import score
+
+
+def _cmd_list(_args) -> int:  # noqa: ANN001
+    print("systems: ", ", ".join(systems.names()))
+    print("datasets:", ", ".join(datasets.registry.names()))
+    return 0
+
+
+def _cmd_run(args) -> int:  # noqa: ANN001
+    try:
+        system = systems.get(args.system)
+        dataset = datasets.registry.get(args.dataset)
+    except KeyError as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        return 2
+    records = run(system, dataset, split=args.split, limit=args.limit)
+    summary = score(records)
+    if args.json:
+        print(to_json(summary, system=args.system, dataset=args.dataset,
+                      split=args.split, model=args.model))
+    else:
+        print(to_markdown(summary, system=args.system, dataset=args.dataset,
+                          split=args.split, model=args.model))
+    return 0
+
+
+def _cmd_suite(args) -> int:  # noqa: ANN001
+    sysnames = [s.strip() for s in args.systems.split(",") if s.strip()]
+    dsnames = (datasets.registry.names() if args.datasets == "all"
+               else [d.strip() for d in args.datasets.split(",") if d.strip()])
+    rows: list[dict] = []
+    for sn in sysnames:
+        for dn in dsnames:
+            try:                                   # fail-soft per cell — one bad cell never kills the matrix
+                recs = run(systems.get(sn), datasets.registry.get(dn),
+                           split=args.split, limit=args.limit)
+                rows.append({"system": sn, "dataset": dn, "summary": score(recs)})
+            except Exception as exc:               # noqa: BLE001
+                rows.append({"system": sn, "dataset": dn, "error": str(exc)})
+    if args.json:
+        out = suite_json(rows, split=args.split, model=args.model)
+    else:
+        out = suite_markdown(rows, split=args.split, model=args.model)
+    if args.out:
+        pathlib.Path(args.out).write_text(out, encoding="utf-8")
+        print(f"wrote {args.out} ({len(rows)} cells)")
+    else:
+        print(out)
+    return 0
+
+
+def build_parser() -> argparse.ArgumentParser:
+    ap = argparse.ArgumentParser(prog="june-bench",
+                                 description="Reproducible benchmark suite (June + pluggable competitors)")
+    sub = ap.add_subparsers(dest="cmd", required=True)
+    sub.add_parser("list", help="list registered systems + datasets").set_defaults(func=_cmd_list)
+    r = sub.add_parser("run", help="run one system on one dataset split")
+    r.add_argument("--system", default="echo")
+    r.add_argument("--dataset", default="smoke")
+    r.add_argument("--split", default="smoke")
+    r.add_argument("--limit", type=int, default=None)
+    r.add_argument("--model", default="", help="model id, recorded in the result header")
+    r.add_argument("--json", action="store_true")
+    r.set_defaults(func=_cmd_run)
+
+    su = sub.add_parser("suite", help="run a systems × datasets matrix → a RESULTS.md table")
+    su.add_argument("--systems", default="echo", help="comma-separated system names")
+    su.add_argument("--datasets", default="smoke", help="comma-separated names, or 'all'")
+    su.add_argument("--split", default="smoke")
+    su.add_argument("--limit", type=int, default=None)
+    su.add_argument("--model", default="")
+    su.add_argument("--out", default="", help="write the table to this path (else stdout)")
+    su.add_argument("--json", action="store_true")
+    su.set_defaults(func=_cmd_suite)
+    return ap
+
+
+def main(argv=None) -> int:
+    args = build_parser().parse_args(argv)
+    return args.func(args)
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

june_bench-0.0.1/june_bench/datasets/__init__.py

@@ -0,0 +1,7 @@
+"""Datasets. SB0 ships the bundled `smoke` set; SB2 registers the four real benchmarks."""
+from __future__ import annotations
+
+from june_bench.datasets import registry
+from june_bench.datasets.loaders import SmokeDataset
+
+__all__ = ["registry", "SmokeDataset"]

june_bench-0.0.1/june_bench/datasets/fixtures/hotpot.smoke.json

@@ -0,0 +1,20 @@
+[
+  {
+    "_id": "hp-smoke-1",
+    "question": "What nationality was the director of the 1994 film Ed Wood?",
+    "answer": "American",
+    "context": [
+      ["Ed Wood (film)", ["Ed Wood is a 1994 American biographical period comedy-drama film directed by Tim Burton.", " It stars Johnny Depp as the cult filmmaker Ed Wood."]],
+      ["Tim Burton", ["Timothy Walter Burton is an American filmmaker, born in Burbank, California."]]
+    ]
+  },
+  {
+    "_id": "hp-smoke-2",
+    "question": "In which city is the Eiffel Tower located?",
+    "answer": "Paris",
+    "context": [
+      ["Eiffel Tower", ["The Eiffel Tower is a wrought-iron lattice tower on the Champ de Mars in Paris, France."]],
+      ["Paris", ["Paris is the capital and most populous city of France."]]
+    ]
+  }
+]

june_bench-0.0.1/june_bench/datasets/fixtures/june.smoke.json

@@ -0,0 +1,13 @@
+{
+  "meta": {"source": "june_bench smoke fixture", "format": "june"},
+  "documents": [
+    {"id": "conv-smoke#0::session_1", "text": "Caroline mentioned she joined an LGBTQ support group on 7 May 2023."},
+    {"id": "conv-smoke#0::session_2", "text": "They talked about the weather and weekend plans."},
+    {"id": "conv-smoke#1::session_1", "text": "I graduated with a degree in Business Administration in 2018."},
+    {"id": "conv-smoke#1::session_2", "text": "Later we discussed my first job after college."}
+  ],
+  "queries": [
+    {"id": "conv-smoke#0", "query": "When did Caroline go to the LGBTQ support group?", "gold": ["conv-smoke#0::session_1"], "haystack": ["conv-smoke#0::session_1", "conv-smoke#0::session_2"], "question_type": "cat2", "answer": "7 May 2023"},
+    {"id": "conv-smoke#1", "query": "What degree did I graduate with?", "gold": ["conv-smoke#1::session_1"], "haystack": ["conv-smoke#1::session_1", "conv-smoke#1::session_2"], "question_type": "single-session-user", "answer": "Business Administration"}
+  ]
+}

june_bench-0.0.1/june_bench/datasets/fixtures/smoke.jsonl

@@ -0,0 +1,4 @@
+{"qid": "s1", "question": "Who wrote the first computer algorithm?", "golds": ["Ada Lovelace"], "context": ["Ada Lovelace is regarded as the first computer programmer; she wrote the first algorithm intended for a machine."]}
+{"qid": "s2", "question": "What city is the Eiffel Tower in?", "golds": ["Paris"], "context": ["The Eiffel Tower is a wrought-iron lattice tower on the Champ de Mars in Paris, France."]}
+{"qid": "s3", "question": "Which planet is known as the Red Planet?", "golds": ["Mars"], "context": ["Mars is often called the Red Planet because of its reddish appearance."]}
+{"qid": "s4", "question": "Who painted the Mona Lisa?", "golds": ["Leonardo da Vinci", "da Vinci"], "context": ["The Mona Lisa is a portrait painted by the Italian artist Leonardo da Vinci."]}

june_bench-0.0.1/june_bench/datasets/loaders.py

@@ -0,0 +1,138 @@
+"""Dataset loaders (SB0 smoke + SB2 the four real benchmarks).
+
+Two formats cover all six datasets, so each is one small adapter behind the `Dataset` port:
+
+* **HotpotQA format** (`hotpot` / `2wiki` / `musique`) — `[{_id, question, answer, context:[[title,
+  [sents]]]}]`. Context-QA: the distractor passages ride in `Example.context`.
+* **June format** (`locomo` / `longmemeval` / `financebench`) — `{documents:[{id,text}], queries:[{query,
+  answer, gold, haystack, question_type}]}` with **conversation-scoped** doc ids (`<conv>::<chunk>`).
+  Memory-QA: the conversation's documents ride in `Example.corpus` (a memory system ingests them, then
+  answers); `gold`/`question_type` ride in `meta`.
+
+Reproducibility: the bundled few-KB fixtures (`fixtures/*.smoke.json`) make `--split smoke` run offline.
+The full splits are read from the data dir (`JUNE_BENCH_DATA`, else the repo `data/` / `benchmarks/apex_qa/`
+for in-repo dev); if absent, the loader raises with a `june-bench fetch` hint (the fetch lands SB-later).
+"""
+from __future__ import annotations
+
+import json
+import os
+import pathlib
+from collections.abc import Sequence
+
+from june_bench.ports import Example
+
+_HERE = pathlib.Path(__file__).resolve().parent
+_FIXTURES = _HERE / "fixtures"
+# In-repo dev locations (a standalone install relies on JUNE_BENCH_DATA / the fetch cache instead).
+# loaders.py lives at <repo>/june_bench/june_bench/datasets/loaders.py → repo root is parents[2].
+_REPO = _HERE.parents[2] if len(_HERE.parents) >= 3 else _HERE
+_REPO_DATA = _REPO / "data"
+_REPO_APEX = _REPO / "benchmarks" / "apex_qa"
+
+
+def _read_json(path: pathlib.Path):
+    return json.loads(path.read_text(encoding="utf-8"))
+
+
+def _resolve(filename: str) -> pathlib.Path | None:
+    for base in (os.environ.get("JUNE_BENCH_DATA"), _REPO_DATA, _REPO_APEX):
+        if not base:
+            continue
+        p = pathlib.Path(base) / filename
+        if p.exists():
+            return p
+    return None
+
+
+def _need(filename: str) -> pathlib.Path:
+    p = _resolve(filename)
+    if p is None:
+        raise FileNotFoundError(
+            f"dataset file {filename!r} not found. Set JUNE_BENCH_DATA to the data dir, "
+            f"or run `june-bench fetch` to download the full splits (the bundled `smoke` split "
+            f"needs no data).")
+    return p
+
+
+# ── HotpotQA format (hotpot / 2wiki / musique) ──────────────────────────────────────────────
+def _hotpot_context(ctx) -> tuple[str, ...]:
+    out: list[str] = []
+    for entry in ctx or []:
+        # entry is [title, [sentences]]
+        if isinstance(entry, (list, tuple)) and len(entry) == 2 and isinstance(entry[1], (list, tuple)):
+            out.append(" ".join(str(s) for s in entry[1]).strip())
+        elif isinstance(entry, str):
+            out.append(entry)
+    return tuple(p for p in out if p)
+
+
+def _hotpot_examples(raw: Sequence[dict]) -> list[Example]:
+    out = []
+    for i, r in enumerate(raw):
+        ans = r.get("answer", "")
+        out.append(Example(
+            qid=str(r.get("_id", i)), question=str(r.get("question", "")),
+            golds=tuple([ans]) if ans else (), context=_hotpot_context(r.get("context")),
+            meta={"question_type": r.get("type", ""), "level": r.get("level", "")}))
+    return out
+
+
+class HotpotFormatDataset:
+    def __init__(self, name: str, full_file: str, *, fixture: str = "hotpot.smoke.json") -> None:
+        self.name = name
+        self._full = full_file
+        self._fixture = fixture
+
+    def load(self, split: str = "smoke") -> Sequence[Example]:
+        path = _FIXTURES / self._fixture if split == "smoke" else _need(self._full)
+        return _hotpot_examples(_read_json(path))
+
+
+# ── June format (locomo / longmemeval / financebench) ───────────────────────────────────────
+def _conv_key(doc_or_query_id: str) -> str:
+    return str(doc_or_query_id).split("::", 1)[0]
+
+
+def _june_examples(blob: dict, *, max_corpus: int = 200) -> list[Example]:
+    docs_by_conv: dict[str, list[str]] = {}
+    for d in blob.get("documents", []):
+        docs_by_conv.setdefault(_conv_key(d.get("id", "")), []).append(str(d.get("text", "")))
+    out = []
+    for q in blob.get("queries", []):
+        conv = _conv_key(q.get("id", ""))
+        corpus = tuple(docs_by_conv.get(conv, [])[:max_corpus])
+        ans = q.get("answer")
+        out.append(Example(
+            qid=str(q.get("id", "")), question=str(q.get("query", "")),
+            golds=tuple([str(ans)]) if ans not in (None, "") else (),
+            corpus=corpus,
+            meta={"question_type": q.get("question_type", ""),
+                  "gold_ids": list(q.get("gold", []) or [])}))
+    return out
+
+
+class JuneFormatDataset:
+    def __init__(self, name: str, full_file: str, *, fixture: str = "june.smoke.json") -> None:
+        self.name = name
+        self._full = full_file
+        self._fixture = fixture
+
+    def load(self, split: str = "smoke") -> Sequence[Example]:
+        path = _FIXTURES / self._fixture if split == "smoke" else _need(self._full)
+        return _june_examples(_read_json(path))
+
+
+# ── the bundled smoke trivia set (SB0) ──────────────────────────────────────────────────────
+class SmokeDataset:
+    name = "smoke"
+
+    def load(self, split: str = "smoke") -> Sequence[Example]:
+        raws = [json.loads(ln) for ln in (_FIXTURES / "smoke.jsonl").read_text(
+            encoding="utf-8").splitlines() if ln.strip()]
+        return [Example(qid=str(r.get("qid", i)), question=str(r.get("question", "")),
+                        golds=tuple(r.get("golds", [])), context=tuple(r.get("context", [])))
+                for i, r in enumerate(raws)]
+
+
+__all__ = ["SmokeDataset", "HotpotFormatDataset", "JuneFormatDataset"]

june_bench-0.0.1/june_bench/datasets/registry.py

@@ -0,0 +1,40 @@
+"""Dataset registry (SB0) — name → Dataset. Adding a benchmark is one entry, never a runner edit."""
+from __future__ import annotations
+
+from june_bench.ports import Dataset
+from june_bench.datasets.loaders import (
+    HotpotFormatDataset,
+    JuneFormatDataset,
+    SmokeDataset,
+)
+
+_DATASETS: dict[str, Dataset] = {}
+
+
+def register(dataset: Dataset) -> None:
+    if not getattr(dataset, "name", ""):
+        raise ValueError("dataset must have a non-empty name")
+    _DATASETS[dataset.name] = dataset
+
+
+def get(name: str) -> Dataset:
+    if name not in _DATASETS:
+        raise KeyError(f"unknown dataset {name!r}; registered: {names()}")
+    return _DATASETS[name]
+
+
+def names() -> list[str]:
+    return sorted(_DATASETS)
+
+
+register(SmokeDataset())
+# SB2 — the four benchmarks (six datasets, two formats). Full files resolve from the data dir
+# (JUNE_BENCH_DATA / repo); the bundled `smoke` split needs no data.
+register(HotpotFormatDataset("hotpot", "hotpot_dev.json"))
+register(HotpotFormatDataset("2wiki", "2wiki_dev.json"))
+register(HotpotFormatDataset("musique", "musique_dev.json"))
+register(JuneFormatDataset("locomo", "locomo.june.json"))
+register(JuneFormatDataset("longmemeval", "lme.june.json"))
+register(JuneFormatDataset("financebench", "financebench.june.json"))
+
+__all__ = ["register", "get", "names"]

june_bench-0.0.1/june_bench/ports.py

@@ -0,0 +1,73 @@
+"""june_bench ports — the two typed contracts the whole suite extends through (SB0).
+
+A benchmark is ``run(system, dataset) → records → score``. Only two things vary: the **System**
+being benchmarked and the **Dataset** it runs on. Everything else (runner, scorer, reporter) is
+fixed. So a new competitor or a new dataset is exactly one adapter behind one of these Protocols —
+never an edit to the harness. Pure: stdlib only, so the core installs and unit-tests with no model,
+no datasets, no network (the ``NullSystem``/``EchoSystem`` smoke).
+"""
+from __future__ import annotations
+
+from collections.abc import Sequence
+from dataclasses import dataclass, field
+from typing import Protocol, runtime_checkable
+
+
+@dataclass(frozen=True, slots=True)
+class Example:
+    """One benchmark item. ``context`` is in-prompt passages (context-QA like HotpotQA);
+    ``corpus`` is documents a *memory* system ingests before answering (LoCoMo/LongMemEval).
+    A system uses whichever its modality needs. ``meta`` carries dataset-specific fields
+    (category, session id, …) so per-category scoring needs no schema change."""
+    qid: str
+    question: str
+    golds: tuple[str, ...]
+    context: tuple[str, ...] = ()
+    corpus: tuple[str, ...] = ()
+    meta: dict = field(default_factory=dict)
+
+
+@dataclass(frozen=True, slots=True)
+class Prediction:
+    """A system's answer + optional provenance (``calls``/``cost``/``abstained`` in ``meta``)
+    so the cost axis is measured, not guessed."""
+    text: str
+    meta: dict = field(default_factory=dict)
+
+
+@dataclass(frozen=True, slots=True)
+class Record:
+    """One scored row: the prediction joined to its gold + provenance. The scorer reads only
+    these, so scoring is decoupled from how the answer was produced (any System, any Dataset)."""
+    qid: str
+    question: str
+    golds: tuple[str, ...]
+    prediction: str
+    context: tuple[str, ...] = ()
+    calls: int = 0
+    cost: float = 0.0
+    abstained: bool = False
+    meta: dict = field(default_factory=dict)
+
+
+@runtime_checkable
+class System(Protocol):
+    """A thing being benchmarked. ``answer`` is the one required method; a memory system may also
+    expose ``ingest(corpus)`` (the runner calls it when present). Async so real adapters (June over
+    HTTP, an LLM) fit without blocking; sync logic just returns. The model/HTTP/heavy deps live in
+    the adapter, never in this contract."""
+    name: str
+
+    async def answer(self, example: Example) -> Prediction: ...
+
+
+@runtime_checkable
+class Dataset(Protocol):
+    """A benchmark's data. ``load(split)`` returns the examples for a named split (e.g. ``smoke``,
+    ``dev100``, ``full``). Loaders are pure readers over the bundled fixtures or the fetched dumps."""
+    name: str
+
+    def load(self, split: str) -> Sequence[Example]: ...
+
+
+__all__ = ["Example", "Prediction", "Record", "System", "Dataset"]

june_bench-0.0.1/june_bench/profiles.py

@@ -0,0 +1,37 @@
+"""Per-dataset scoring profiles (SB1) — which axes are the *headline* for each benchmark.
+
+All benchmarks are scored on the same three axes (accuracy/coverage/cost, `score.py`), but they don't
+all *report* the same headline: HotpotQA/2Wiki/MuSiQue/FinanceBench are EM/F1; the memory benchmarks
+(LoCoMo, LongMemEval) pair accuracy with coverage / retrieval-recall. A profile is just the ordered
+metric keys a reporter surfaces — adding a dataset's profile is one entry, never a scorer change."""
+from __future__ import annotations
+
+_PROFILES: dict[str, tuple[str, ...]] = {
+    "default": ("em", "f1", "coverage"),
+    "hotpot": ("em", "f1"),
+    "2wiki": ("em", "f1"),
+    "musique": ("em", "f1"),
+    "locomo": ("em", "f1", "coverage"),
+    "longmemeval": ("em", "f1", "context_recall"),
+    "financebench": ("em", "f1"),
+    "smoke": ("em", "f1"),
+}
+
+
+def headline_metrics(dataset: str) -> tuple[str, ...]:
+    """The ordered headline metric keys for a dataset (falls back to the default profile)."""
+    return _PROFILES.get(dataset, _PROFILES["default"])
+
+
+def headline(summary: dict, dataset: str) -> dict:
+    """Project a full `score()` summary down to a dataset's headline metrics (order preserved)."""
+    return {k: summary[k] for k in headline_metrics(dataset) if k in summary}
+
+
+def register_profile(dataset: str, metrics: tuple[str, ...]) -> None:
+    if not metrics:
+        raise ValueError("a profile needs at least one metric")
+    _PROFILES[dataset] = tuple(metrics)
+
+
+__all__ = ["headline_metrics", "headline", "register_profile"]

june_bench-0.0.1/june_bench/report.py

@@ -0,0 +1,60 @@
+"""Reporter (SB0) — a scored summary → a self-describing markdown row + JSON.
+
+Every row records *system · dataset · split · model + the three axes*, so a published number is
+reproducible and self-describing (the honesty discipline). SB5 extends this to a full multi-arm
+`RESULTS.md` table; SB0 ships the single-run row.
+"""
+from __future__ import annotations
+
+import json
+
+
+def to_json(summary: dict, *, system: str, dataset: str, split: str, model: str = "") -> str:
+    head = {"system": system, "dataset": dataset, "split": split, "model": model}
+    return json.dumps({**head, **summary}, indent=2)
+
+
+def to_markdown(summary: dict, *, system: str, dataset: str, split: str, model: str = "") -> str:
+    s = summary
+    return (
+        f"### {system} · {dataset}/{split}" + (f" · {model}" if model else "") + "\n\n"
+        "| n | answered | EM | F1 | coverage | ctx-recall | calls/ans | cost/ans |\n"
+        "|---|---|---|---|---|---|---|---|\n"
+        f"| {s['n']} | {s['answered']} | {s['em']} | {s['f1']} | {s['coverage']} | "
+        f"{s['context_recall']} | {s['calls_per_answer']} | {s['cost_per_answer']} |\n"
+    )
+
+
+def suite_markdown(rows: list[dict], *, split: str = "", model: str = "") -> str:
+    """A systems × datasets matrix → one `RESULTS.md`-ready table. Each row is
+    ``{system, dataset, summary}`` or ``{system, dataset, error}`` (fail-soft per cell). The
+    per-dataset *headline* metrics (profiles) are noted, but every axis is shown for honesty."""
+    from june_bench.profiles import headline_metrics
+    head = f"# june-bench results"
+    if split:
+        head += f" · split `{split}`"
+    if model:
+        head += f" · model `{model}`"
+    lines = [head, "",
+             "| system | dataset | headline | n | EM | F1 | coverage | ctx-recall | calls/ans | cost/ans |",
+             "|---|---|---|---|---|---|---|---|---|---|"]
+    for r in rows:
+        sysn, dsn = r.get("system", "?"), r.get("dataset", "?")
+        if "error" in r:
+            lines.append(f"| {sysn} | {dsn} | — | — | _error_ | {r['error'][:60]} | | | | |")
+            continue
+        s = r["summary"]
+        hl = "/".join(headline_metrics(dsn))
+        lines.append(f"| {sysn} | {dsn} | {hl} | {s['n']} | {s['em']} | {s['f1']} | "
+                     f"{s['coverage']} | {s['context_recall']} | {s['calls_per_answer']} | "
+                     f"{s['cost_per_answer']} |")
+    lines += ["", "_Same data, same scorer; every cell records the system, dataset, split and model. "
+              "Headline = the dataset's reported metric(s); all axes shown for honesty._"]
+    return "\n".join(lines) + "\n"
+
+
+def suite_json(rows: list[dict], *, split: str = "", model: str = "") -> str:
+    return json.dumps({"split": split, "model": model, "rows": rows}, indent=2)
+
+
+__all__ = ["to_json", "to_markdown", "suite_markdown", "suite_json"]

june_bench-0.0.1/june_bench/runner.py

@@ -0,0 +1,46 @@
+"""june_bench runner — the fixed orchestration: ask the System each Example → Records (SB0).
+
+Pure: depends only on the two ports, so it's unit-tested with a fake System + fixture Dataset
+(no model, no network). If a System exposes ``ingest`` (a memory system), the runner calls it with
+each example's ``corpus`` before asking — so context-QA and memory systems share one path.
+"""
+from __future__ import annotations
+
+import asyncio
+import inspect
+from collections.abc import Sequence
+
+from june_bench.ports import Dataset, Example, Record, System
+
+
+async def _maybe_await(value):
+    return await value if inspect.isawaitable(value) else value
+
+
+async def _answer_one(system: System, ex: Example) -> Record:
+    ingest = getattr(system, "ingest", None)
+    if ingest is not None and ex.corpus:
+        await _maybe_await(ingest(ex.corpus))
+    pred = await system.answer(ex)
+    m = pred.meta or {}
+    return Record(
+        qid=ex.qid, question=ex.question, golds=ex.golds, prediction=pred.text,
+        context=ex.context, calls=int(m.get("calls", 0)), cost=float(m.get("cost", 0.0)),
+        abstained=bool(m.get("abstained", False)), meta=dict(m))
+
+
+async def run_async(system: System, examples: Sequence[Example]) -> list[Record]:
+    return [await _answer_one(system, ex) for ex in examples]
+
+
+def run(system: System, dataset: Dataset, *, split: str = "smoke",
+        limit: int | None = None) -> list[Record]:
+    """Load the split, ask the system each example, return scored-ready Records. Sync entry point
+    (wraps the async path) so the CLI and tests call one function."""
+    examples = list(dataset.load(split))
+    if limit is not None:
+        examples = examples[:limit]
+    return asyncio.run(run_async(system, examples))
+
+
+__all__ = ["run", "run_async"]