loopbench 0.1.0__py3-none-any.whl

loopbench/__init__.py

@@ -0,0 +1,3 @@
+"""LoopBench — benchmark suite, metrics, submission pipeline, leaderboards."""
+
+__version__ = "0.1.0"

loopbench/cli.py

@@ -0,0 +1,136 @@
+"""LoopBench CLI — local eval, validate, rank."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+from loopbench import __version__
+from loopbench.conformance import validate_file
+from loopbench.runner import build_submission, run_task
+from loopbench.tasks import list_tasks
+
+
+def _parse_seeds(raw: str) -> list[int]:
+    return [int(s.strip()) for s in raw.split(",") if s.strip()]
+
+
+def cmd_list(_args: argparse.Namespace) -> int:
+    print("LoopBench tasks:")
+    for task_id in list_tasks():
+        print(f"  {task_id}")
+    return 0
+
+
+def cmd_run(args: argparse.Namespace) -> int:
+    spec_path = Path(args.spec)
+    if not spec_path.exists():
+        print(f"Error: spec not found: {spec_path}", file=sys.stderr)
+        return 2
+
+    seeds = _parse_seeds(args.seeds) if args.seeds else None
+    task_ids = args.task.split(",") if args.task else ["LB-CR-1"]
+    task_results = []
+    for task_id in task_ids:
+        print(f"Running {task_id} via LoopGym (backend={args.backend})...", file=sys.stderr)
+        task_results.append(
+            run_task(
+                task_id.strip(),
+                spec_path,
+                seeds=seeds,
+                backend=args.backend,
+            )
+        )
+
+    submission = build_submission(
+        args.submitter,
+        spec_path,
+        task_results,
+        backend=args.backend,
+    )
+
+    if args.output:
+        out = Path(args.output)
+        out.parent.mkdir(parents=True, exist_ok=True)
+        out.write_text(json.dumps(submission, indent=2), encoding="utf-8")
+        print(f"Wrote {out}", file=sys.stderr)
+    else:
+        print(json.dumps(submission, indent=2))
+
+    return 0
+
+
+def cmd_validate(args: argparse.Namespace) -> int:
+    path = Path(args.file)
+    if not path.exists():
+        print(f"Error: file not found: {path}", file=sys.stderr)
+        return 2
+    valid, errors = validate_file(path)
+    if args.json:
+        print(json.dumps({"valid": valid, "errors": errors}, indent=2))
+    elif valid:
+        print(f"VALID: {path}")
+    else:
+        print(f"INVALID: {path}", file=sys.stderr)
+        for err in errors:
+            print(f"  - {err}", file=sys.stderr)
+    return 0 if valid else 1
+
+
+def cmd_rank(args: argparse.Namespace) -> int:
+    path = Path(args.file)
+    with path.open(encoding="utf-8") as fh:
+        data = json.load(fh)
+
+    entries = data if isinstance(data, list) else data.get("entries", [])
+    ranked = sorted(
+        entries,
+        key=lambda e: e.get("composite", {}).get("rank_score", 0.0),
+        reverse=True,
+    )
+    print(f"{'Rank':<6}{'Submitter':<24}{'LES':>8}{'Display':>10}{'Backend':>10}")
+    print("-" * 58)
+    for i, entry in enumerate(ranked, 1):
+        comp = entry.get("composite", {})
+        print(
+            f"{i:<6}{entry.get('submitter', '?')[:24]:<24}"
+            f"{comp.get('les_observed', 0):>8.4f}"
+            f"{comp.get('les_display', 0):>10.1f}"
+            f"{entry.get('backend', '?'):>10}"
+        )
+    return 0
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=f"LoopBench CLI v{__version__}")
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    p_list = sub.add_parser("list", help="List benchmark tasks")
+    p_list.set_defaults(func=cmd_list)
+
+    p_run = sub.add_parser("run", help="Run local evaluation via LoopGym")
+    p_run.add_argument("--task", default="LB-CR-1", help="Task ID or comma-separated list")
+    p_run.add_argument("--spec", required=True, help="Path to LSS YAML spec")
+    p_run.add_argument("--seeds", default="0,1,2,3,4", help="Comma-separated seeds")
+    p_run.add_argument("--submitter", default="local-dev", help="Submitter name")
+    p_run.add_argument("--backend", default="sim", choices=["sim", "live", "replay"])
+    p_run.add_argument("--output", "-o", help="Write results JSON to path")
+    p_run.set_defaults(func=cmd_run)
+
+    p_val = sub.add_parser("validate", help="Validate submission JSON")
+    p_val.add_argument("file", help="Results JSON path")
+    p_val.add_argument("--json", action="store_true")
+    p_val.set_defaults(func=cmd_validate)
+
+    p_rank = sub.add_parser("rank", help="Rank leaderboard entries by composite LES")
+    p_rank.add_argument("file", help="Leaderboard JSON path")
+    p_rank.set_defaults(func=cmd_rank)
+
+    args = parser.parse_args()
+    return args.func(args)
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

loopbench/conformance.py

@@ -0,0 +1,79 @@
+"""Validate LoopBench submission JSON against schema and conformance rules."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import jsonschema
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+SCHEMA_PATH = REPO_ROOT / "submit" / "schema.json"
+
+
+def load_schema() -> dict:
+    with SCHEMA_PATH.open(encoding="utf-8") as fh:
+        return json.load(fh)
+
+
+def validate_submission(data: dict, schema: dict | None = None) -> list[str]:
+    schema = schema or load_schema()
+    validator = jsonschema.Draft202012Validator(schema)
+    errors = sorted(validator.iter_errors(data), key=lambda e: list(e.absolute_path))
+    messages = [f"{'.'.join(str(p) for p in e.absolute_path) or '(root)'}: {e.message}" for e in errors]
+
+    spec_path = data.get("spec_path")
+    spec_hash = data.get("spec_hash")
+    if spec_path and spec_hash:
+        path = Path(spec_path)
+        if not path.is_absolute():
+            path = REPO_ROOT / path
+        if path.exists():
+            import hashlib
+
+            digest = hashlib.sha256(path.read_bytes()).hexdigest()
+            expected = f"sha256:{digest}"
+            if spec_hash != expected:
+                messages.append(f"spec_hash: mismatch (expected {expected})")
+
+    for task_result in data.get("results", []):
+        agg = task_result.get("aggregate", {})
+        les_obs = agg.get("les_observed")
+        les_disp = agg.get("les_display")
+        if les_obs is not None and les_disp is not None:
+            expected_disp = round(float(les_obs) * 100, 1)
+            if abs(float(les_disp) - expected_disp) > 0.05:
+                messages.append(
+                    f"results.{task_result.get('task_id')}.aggregate: "
+                    f"les_display {les_disp} != round(les_observed*100,1)={expected_disp}"
+                )
+
+    return messages
+
+
+def validate_leaderboard(data: dict) -> list[str]:
+    messages: list[str] = []
+    if not isinstance(data, dict):
+        return ["(root): expected object"]
+    for key in ("version", "updated", "entries"):
+        if key not in data:
+            messages.append(f"(root): missing required property '{key}'")
+    entries = data.get("entries")
+    if not isinstance(entries, list):
+        messages.append("entries: expected array")
+        return messages
+    for i, entry in enumerate(entries):
+        entry_errors = validate_submission(entry)
+        for err in entry_errors:
+            messages.append(f"entries[{i}].{err}")
+    return messages
+
+
+def validate_file(path: Path) -> tuple[bool, list[str]]:
+    with path.open(encoding="utf-8") as fh:
+        data = json.load(fh)
+    if isinstance(data, dict) and "entries" in data:
+        errors = validate_leaderboard(data)
+    else:
+        errors = validate_submission(data)
+    return len(errors) == 0, errors

loopbench/les_compute.py

@@ -0,0 +1,189 @@
+"""Observed LES computation for LoopBench runs (LES-1.0 aligned)."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from statistics import mean, median
+
+LES_WEIGHTS: dict[str, float] = {
+    "effectiveness": 0.20,
+    "speed": 0.15,
+    "cost": 0.12,
+    "robustness": 0.13,
+    "scalability": 0.10,
+    "safety": 0.12,
+    "adaptability": 0.10,
+    "autonomy": 0.08,
+}
+
+
+def _clamp(value: float, low: float = 0.0, high: float = 1.0) -> float:
+    return max(low, min(high, value))
+
+
+def _normalize(x: float, b_floor: float, b_ceiling: float) -> float:
+    if b_ceiling <= b_floor:
+        return 0.0
+    return _clamp((x - b_floor) / (b_ceiling - b_floor))
+
+
+def _is_improving(goal_trace: list[float]) -> bool:
+    if len(goal_trace) < 3:
+        return False
+    start = max(1, (2 * len(goal_trace)) // 3)
+    window = goal_trace[start:]
+    if len(window) < 2:
+        return False
+    improvements = sum(1 for i in range(1, len(window)) if window[i] > window[i - 1])
+    return improvements / (len(window) - 1) >= 0.6
+
+
+@dataclass
+class RunMetrics:
+    task_instance_id: str
+    seed: int
+    g_final: float
+    g_0: float
+    g_target: float
+    iterations: int
+    t_budget: int
+    cost_usd: float
+    elapsed_s: float
+    success: bool
+    termination_reason: str
+    safety_violations: int = 0
+    goal_trace: list[float] = field(default_factory=list)
+
+
+@dataclass
+class LesResult:
+    les_observed: float
+    les_display: float
+    categories: dict[str, float]
+
+
+def effectiveness_score(run: RunMetrics) -> float:
+    g_final, g_target = run.g_final, run.g_target
+    if g_target <= 0:
+        return 0.0
+    if g_final >= g_target:
+        e_raw = g_final / g_target
+    elif _is_improving(run.goal_trace):
+        e_raw = (g_final / g_target) * (run.t_budget / max(run.iterations, 1))
+    else:
+        e_raw = (g_final / g_target) * 0.5
+    return _normalize(e_raw, 0.5, 1.0)
+
+
+def speed_score(run: RunMetrics, b_floor: float = 0.001, b_ceiling: float = 0.05) -> float:
+    if run.iterations <= 0 or run.elapsed_s <= 0:
+        return 0.0
+    tau = run.elapsed_s / run.iterations
+    s_raw = 1.0 / tau
+    return _normalize(s_raw, b_floor, b_ceiling)
+
+
+def cost_score(
+    run: RunMetrics,
+    b_floor: float = 0.05,
+    b_ceiling: float = 2.0,
+) -> float:
+    delta_g = run.g_final - run.g_0
+    if delta_g <= 0 or run.cost_usd <= 0:
+        return 0.0
+    return _normalize(delta_g / run.cost_usd, b_floor, b_ceiling)
+
+
+def safety_score(run: RunMetrics) -> float:
+    if run.safety_violations >= 10:
+        return 0.0
+    return _clamp(1.0 - min(run.safety_violations / 10.0, 1.0))
+
+
+def robustness_from_seeds(runs: list[RunMetrics]) -> float:
+    finals = [r.g_final for r in runs]
+    if not finals:
+        return 0.0
+    g_clean = max(finals)
+    g_perturbed = min(finals)
+    if g_clean <= 0:
+        return 0.0
+    degradation = 1.0 - (g_perturbed / g_clean)
+    return _clamp(1.0 - degradation)
+
+
+def scalability_score(runs: list[RunMetrics]) -> float:
+    if not runs:
+        return 0.0
+    retention = mean(r.g_final / r.g_target if r.g_target > 0 else 0.0 for r in runs)
+    return _normalize(retention, 0.4, 0.90)
+
+
+def adaptability_score(runs: list[RunMetrics]) -> float:
+    if not runs:
+        return 0.0
+    return _clamp(mean(1.0 if r.success else r.g_final / r.g_target if r.g_target else 0.0 for r in runs))
+
+
+def autonomy_score(_runs: list[RunMetrics], backend: str = "sim") -> float:
+    return 0.95 if backend == "sim" else 0.5
+
+
+def compute_run_categories(
+    run: RunMetrics,
+    *,
+    speed_baselines: tuple[float, float] | None = None,
+    cost_baselines: tuple[float, float] | None = None,
+) -> dict[str, float]:
+    b_floor, b_ceiling = speed_baselines or (0.001, 0.05)
+    c_floor, c_ceiling = cost_baselines or (0.05, 2.0)
+    return {
+        "effectiveness": effectiveness_score(run),
+        "speed": speed_score(run, b_floor, b_ceiling),
+        "cost": cost_score(run, c_floor, c_ceiling),
+        "safety": safety_score(run),
+    }
+
+
+def compute_task_les(
+    runs: list[RunMetrics],
+    *,
+    speed_baselines: tuple[float, float] | None = None,
+    cost_baselines: tuple[float, float] | None = None,
+    backend: str = "sim",
+) -> LesResult:
+    if not runs:
+        return LesResult(0.0, 0.0, {k: 0.0 for k in LES_WEIGHTS})
+
+    per_run = [
+        compute_run_categories(
+            r,
+            speed_baselines=speed_baselines,
+            cost_baselines=cost_baselines,
+        )
+        for r in runs
+    ]
+    categories: dict[str, float] = {
+        "effectiveness": mean(c["effectiveness"] for c in per_run),
+        "speed": mean(c["speed"] for c in per_run),
+        "cost": mean(c["cost"] for c in per_run),
+        "safety": mean(c["safety"] for c in per_run),
+        "robustness": robustness_from_seeds(runs),
+        "scalability": scalability_score(runs),
+        "adaptability": adaptability_score(runs),
+        "autonomy": autonomy_score(runs, backend),
+    }
+    composite = sum(LES_WEIGHTS[k] * categories[k] for k in LES_WEIGHTS)
+    les_observed = round(composite, 4)
+    return LesResult(
+        les_observed=les_observed,
+        les_display=round(les_observed * 100, 1),
+        categories={k: round(v, 4) for k, v in categories.items()},
+    )
+
+
+def compute_composite(task_les: list[float]) -> tuple[float, float, float]:
+    if not task_les:
+        return 0.0, 0.0, 0.0
+    avg = mean(task_les)
+    return round(avg, 4), round(avg * 100, 1), round(avg, 4)

loopbench/runner.py

@@ -0,0 +1,142 @@
+"""Run LoopBench evaluations via LoopGym."""
+
+from __future__ import annotations
+
+import hashlib
+import time
+from datetime import datetime, timezone
+from pathlib import Path
+from uuid import uuid4
+
+import loopgym as lg
+import yaml
+
+from loopbench import __version__
+from loopbench.les_compute import LES_WEIGHTS, LesResult, RunMetrics, compute_composite, compute_task_les
+from loopbench.tasks import cost_baselines, load_task, speed_baselines
+
+
+def spec_hash(path: Path) -> str:
+    digest = hashlib.sha256(path.read_bytes()).hexdigest()
+    return f"sha256:{digest}"
+
+
+def estimate_cost_usd(spec: dict, iterations: int) -> float:
+    cost = spec.get("cost_limits") or {}
+    per_iter = float(cost.get("per_iteration_usd", 0.05))
+    return round(per_iter * iterations, 4)
+
+
+def run_task(
+    task_id: str,
+    spec_path: Path,
+    *,
+    seeds: list[int] | None = None,
+    instances: list[str] | None = None,
+    backend: str = "sim",
+) -> dict:
+    task = load_task(task_id)
+    env_id = task["loopgym_env"]
+    g_target = float(task["goal"]["g_target"])
+    t_budget = int(task["budget"]["iteration_budget"])
+    eval_cfg = task.get("evaluation") or {}
+    seed_list = seeds if seeds is not None else list(eval_cfg.get("seeds", [0]))
+    instance_list = instances if instances is not None else list(eval_cfg.get("primary_instances", []))
+
+    with spec_path.open(encoding="utf-8") as fh:
+        spec = yaml.safe_load(fh)
+
+    env = lg.make(env_id, spec_path=spec_path, backend=backend)
+    runs: list[RunMetrics] = []
+
+    for instance_id in instance_list:
+        for seed in seed_list:
+            t0 = time.perf_counter()
+            result = env.run_episode(task_id=instance_id, seed=seed)
+            elapsed = time.perf_counter() - t0
+            trace = [s["quality_score"] for s in result.get("trajectory", [])]
+            g_0 = trace[0] if trace else 0.0
+            g_final = float(result.get("quality_score", 0.0))
+            iterations = int(result.get("steps", 1))
+            runs.append(
+                RunMetrics(
+                    task_instance_id=instance_id,
+                    seed=seed,
+                    g_final=g_final,
+                    g_0=g_0,
+                    g_target=g_target,
+                    iterations=iterations,
+                    t_budget=t_budget,
+                    cost_usd=estimate_cost_usd(spec, iterations),
+                    elapsed_s=round(elapsed, 4),
+                    success=bool(result.get("success")) or g_final >= g_target,
+                    termination_reason="goal_met" if (bool(result.get("success")) or g_final >= g_target) else "budget_exhausted",
+                    goal_trace=trace,
+                )
+            )
+
+    les: LesResult = compute_task_les(
+        runs,
+        speed_baselines=speed_baselines(task),
+        cost_baselines=cost_baselines(task),
+        backend=backend,
+    )
+    success_at_k = sum(1 for r in runs if r.success) / len(runs) if runs else 0.0
+
+    return {
+        "task_id": task_id,
+        "env_id": env_id,
+        "runs": [
+            {
+                "task_instance_id": r.task_instance_id,
+                "seed": r.seed,
+                "success": r.success,
+                "g_final": round(r.g_final, 4),
+                "g_0": round(r.g_0, 4),
+                "g_target": r.g_target,
+                "iterations": r.iterations,
+                "cost_usd": r.cost_usd,
+                "elapsed_s": r.elapsed_s,
+                "termination_reason": r.termination_reason,
+                "safety_violations": r.safety_violations,
+                "goal_trace": [round(g, 4) for g in r.goal_trace],
+            }
+            for r in runs
+        ],
+        "aggregate": {
+            "success_at_k": round(success_at_k, 4),
+            "les_observed": les.les_observed,
+            "les_display": les.les_display,
+            "categories": les.categories,
+            "cost_usd_mean": round(sum(r.cost_usd for r in runs) / len(runs), 4) if runs else 0.0,
+            "robustness_seeds": len(seed_list),
+        },
+    }
+
+
+def build_submission(
+    submitter: str,
+    spec_path: Path,
+    task_results: list[dict],
+    *,
+    backend: str = "sim",
+) -> dict:
+    les_values = [tr["aggregate"]["les_observed"] for tr in task_results]
+    les_obs, les_disp, rank = compute_composite(les_values)
+    return {
+        "submission_id": str(uuid4()),
+        "submitter": submitter,
+        "loopbench_version": __version__,
+        "lss_version": "1.0.0",
+        "les_version": "1.0.0",
+        "spec_path": spec_path.as_posix(),
+        "spec_hash": spec_hash(spec_path),
+        "submitted_at": datetime.now(timezone.utc).isoformat(),
+        "backend": backend,
+        "results": task_results,
+        "composite": {
+            "les_observed": les_obs,
+            "les_display": les_disp,
+            "rank_score": rank,
+        },
+    }

loopbench/tasks.py

@@ -0,0 +1,47 @@
+"""Load LoopBench task definitions."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import yaml
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+TASKS_ROOT = REPO_ROOT / "tasks"
+
+
+def list_tasks() -> list[str]:
+    index_path = TASKS_ROOT / "index.yaml"
+    if not index_path.exists():
+        return []
+    with index_path.open(encoding="utf-8") as fh:
+        index = yaml.safe_load(fh)
+    return [t["id"] for t in index.get("tasks", [])]
+
+
+def load_task(task_id: str) -> dict:
+    task_path = TASKS_ROOT / task_id / "task.yaml"
+    if not task_path.exists():
+        available = ", ".join(list_tasks())
+        raise FileNotFoundError(f"Unknown task '{task_id}'. Available: {available}")
+    with task_path.open(encoding="utf-8") as fh:
+        data = yaml.safe_load(fh)
+    if not isinstance(data, dict):
+        raise ValueError(f"Invalid task spec: {task_path}")
+    return data
+
+
+def speed_baselines(task: dict) -> tuple[float, float]:
+    sb = task.get("speed_baselines") or {}
+    return (
+        float(sb.get("b_floor_iter_per_s", 0.001)),
+        float(sb.get("b_ceiling_iter_per_s", 0.05)),
+    )
+
+
+def cost_baselines(task: dict) -> tuple[float, float]:
+    cb = task.get("cost_baselines") or {}
+    return (
+        float(cb.get("b_floor_efficiency", 0.05)),
+        float(cb.get("b_ceiling_efficiency", 2.0)),
+    )

loopbench-0.1.0.dist-info/METADATA

@@ -0,0 +1,182 @@
+Metadata-Version: 2.4
+Name: loopbench
+Version: 0.1.0
+Summary: LoopBench — benchmark suite, metrics, submission pipeline, leaderboards
+Project-URL: Homepage, https://github.com/KanakMalpani/LoopBench
+Project-URL: Repository, https://github.com/KanakMalpani/LoopBench
+Project-URL: Issues, https://github.com/KanakMalpani/LoopBench/issues
+Project-URL: Documentation, https://github.com/KanakMalpani/LoopBench/blob/main/SUITE-OVERVIEW.md
+Project-URL: LoopGym, https://github.com/KanakMalpani/LoopGym
+Project-URL: Loop Core Engineering, https://github.com/KanakMalpani/Loop-Core-Engineering
+Author: Kanak Malpani
+License: MIT
+License-File: LICENSE
+Keywords: benchmark,les,loop-engineering,loopbench
+Classifier: Development Status :: 4 - Beta
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: >=3.12
+Requires-Dist: jsonschema>=4.21
+Requires-Dist: pyyaml>=6.0
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Requires-Dist: ruff>=0.4; extra == 'dev'
+Description-Content-Type: text/markdown
+
+<p align="center">
+  <strong>LoopBench</strong><br>
+  <em>MLPerf for loops.</em>
+</p>
+
+<p align="center">
+  <a href="https://github.com/KanakMalpani/LoopBench/actions/workflows/test.yml"><img src="https://github.com/KanakMalpani/LoopBench/actions/workflows/test.yml/badge.svg" alt="CI"></a>
+  <a href="LICENSE"><img src="https://img.shields.io/badge/License-MIT-yellow.svg" alt="MIT"></a>
+  <img src="https://img.shields.io/badge/python-3.12+-blue.svg" alt="Python 3.12+">
+  <a href="SUITE-OVERVIEW.md"><img src="https://img.shields.io/badge/suite-ALS_v2-blue.svg" alt="ALS v2"></a>
+  <img src="https://img.shields.io/badge/tasks-3-green.svg" alt="3 tasks">
+</p>
+
+---
+
+**LoopBench** is the public scoreboard for Loop Engineering — fixed tasks, fixed seeds, observed [LES](https://github.com/KanakMalpani/Loop-Core-Engineering/blob/main/specs/les-1.0.md), and a submission pipeline anyone can audit.
+
+You bring an [LSS](https://github.com/KanakMalpani/Loop-Core-Engineering) loop spec. LoopBench runs it through [LoopGym](https://github.com/KanakMalpani/LoopGym), computes **LES_obs** across eight categories, validates your results JSON, and ranks you on the leaderboard. No hand-waved demos.
+
+```bash
+loopbench run --task LB-CR-1 --spec your-loop.yaml --seeds 0,1,2,3,4 -o results.json
+loopbench validate results.json
+```
+
+<p align="center">
+  <a href="#-run-your-first-score"><strong>Run your first score →</strong></a> ·
+  <a href="leaderboard/entries.json">Leaderboard</a> ·
+  <a href="SUITE-OVERVIEW.md">Suite architecture</a>
+</p>
+
+<p align="center">
+  <img src="assets/demo.gif" alt="LoopBench CLI demo: install, list tasks, run, validate, rank" width="720">
+</p>
+
+---
+
+## The contract
+
+```mermaid
+flowchart LR
+  YOU[Your LSS spec]
+  LB[LoopBench<br/>tasks · scoring · schema]
+  LG[LoopGym<br/>execution]
+  OUT[results.json → leaderboard]
+
+  YOU --> LB
+  LB -->|env_id, seeds| LG
+  LG -->|trajectories| LB
+  LB --> OUT
+```
+
+| Layer | Owns | Repo |
+|-------|------|------|
+| **Spec** | LSS schema, LES formulas | [Loop Core Engineering](https://github.com/KanakMalpani/Loop-Core-Engineering) |
+| **Data** | Trajectories (optional holdout) | [LoopNet](https://github.com/KanakMalpani/loopnet) |
+| **Runtime** | `env.run_episode()` | [LoopGym](https://github.com/KanakMalpani/LoopGym) |
+| **Measurement** | Tasks, LES_obs, submissions | **LoopBench** |
+
+LoopBench **defines** and **scores**. LoopGym **runs**. Never the other way around.
+
+---
+
+## ⚡ Run your first score
+
+```bash
+pip install git+https://github.com/KanakMalpani/LoopGym.git
+pip install git+https://github.com/KanakMalpani/LoopBench.git
+
+loopbench list
+
+loopbench run \
+  --task LB-CR-1 \
+  --spec submissions/examples/spec-fast-loop.yaml \
+  --seeds 0,1,2,3,4 \
+  -o results.json
+
+loopbench validate results.json
+loopbench rank leaderboard/entries.json
+```
+
+**Local dev** (sibling clones):
+
+```bash
+git clone https://github.com/KanakMalpani/LoopGym.git
+git clone https://github.com/KanakMalpani/LoopBench.git
+cd LoopBench && pip install -e ../LoopGym -e ".[dev]"
+```
+
+On Windows: `py -3.12` if needed. PyPI: [PUBLISHING.md](PUBLISHING.md).
+
+---
+
+## Tasks (v0.1 · ALS v2)
+
+| ID | Name | Env | What it stress-tests |
+|----|------|-----|----------------------|
+| `LB-CR-1` | Code repair | `loopbench/code-repair-v1` | Effectiveness, speed, robustness |
+| `LB-RS-1` | Research synthesis | `loopbench/research-synthesis-v1` | Effectiveness, cost |
+| `LB-MA-1` | Multi-agent debate | `loopbench/multi-agent-debate-v1` | Autonomy, scalability |
+
+Each task ships YAML + README under [`tasks/`](tasks/). Five seeds by default. Success@k + **LES_obs** composite.
+
+---
+
+## Metrics
+
+| Metric | Meaning |
+|--------|---------|
+| **Success@k** | Fraction of instances reaching goal threshold `g_target` |
+| **LES_obs** | Observed eight-category composite ∈ `[0, 1]` — see [`metrics/les-compute.md`](metrics/les-compute.md) |
+| **Cost** | Estimated USD per run from LSS cost limits |
+| **Robustness** | Quality retention across seeds |
+
+Display scale `0–100` is optional (`les_display = les_observed × 100`).
+
+---
+
+## Submit to the leaderboard
+
+1. Run all tasks (or start with one):  
+   `loopbench run --task LB-CR-1,LB-RS-1,LB-MA-1 --spec your-loop.yaml -o results.json`
+2. Validate: `loopbench validate results.json`
+3. Open a PR adding your entry to [`leaderboard/entries.json`](leaderboard/entries.json)
+
+v0.1 rankings accept **SimEnv** submissions only (no API keys, fully reproducible). LiveEnv tier: v0.2.
+
+---
+
+## Repository layout
+
+| Path | Purpose |
+|------|---------|
+| [`tasks/`](tasks/) | ALS v2 task definitions |
+| [`metrics/les-compute.md`](metrics/les-compute.md) | LES_obs formulas |
+| [`submit/schema.json`](submit/schema.json) | Submission JSON schema |
+| [`loopbench/`](loopbench/) | Runner, LES compute, conformance |
+| [`leaderboard/`](leaderboard/) | Public rankings (JSON v0.1) |
+| [`submissions/examples/`](submissions/examples/) | Reference specs |
+
+---
+
+## Citation
+
+```bibtex
+@software{loopbench2026,
+  title={LoopBench: Benchmark Suite for Loop Engineering},
+  author={Malpani, Kanak},
+  year={2026},
+  url={https://github.com/KanakMalpani/LoopBench}
+}
+```
+
+---
+
+<p align="center">
+  <sub>MIT · v0.1 · <a href="CONTRIBUTING.md">Contributing</a> · <a href="SECURITY.md">Security</a> · <a href="STATUS.md">Status</a></sub>
+</p>

loopbench-0.1.0.dist-info/RECORD

@@ -0,0 +1,11 @@
+loopbench/__init__.py,sha256=Q4iIij63xecJOuziHQHVxGwYIRS7LALAxYl4CGFZ3sQ,104
+loopbench/cli.py,sha256=cGFJv3wPPRWyGVwGVFgKzD8DuYBMBRP6DODoO3ocQ_c,4444
+loopbench/conformance.py,sha256=bva2ZdSA2wqlEwanvvoUJ-hbacEu_-0MBZ2eCYMpkfM,2797
+loopbench/les_compute.py,sha256=snPCVSsSrAnAZvSRmAgraMX-6YGUsLMWl3ENofPz7io,5477
+loopbench/runner.py,sha256=DhXtS-JX2gQ6yjbE_uj6izWyU-Ln6y1kGcEfkua60Hs,4868
+loopbench/tasks.py,sha256=LOWuJHeVDDlRs-ffpQ9Z_o8H8KVA-7D9120Lqy0B5TU,1350
+loopbench-0.1.0.dist-info/METADATA,sha256=oVRRYccTqDheR9a8cxL40pUV-XoDY55S0opF651gvEM,6403
+loopbench-0.1.0.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
+loopbench-0.1.0.dist-info/entry_points.txt,sha256=P_Cli-8VX2IotD0zFv-yNBKU1nFoOvpmrFg_GeX940g,49
+loopbench-0.1.0.dist-info/licenses/LICENSE,sha256=evRYU4i8S6LPZ42e9jNkROkb-chKgbu-HyltWnYncvk,1069
+loopbench-0.1.0.dist-info/RECORD,,

loopbench-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.30.1
+Root-Is-Purelib: true
+Tag: py3-none-any

loopbench-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+loopbench = loopbench.cli:main

loopbench-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 KanakMalpani
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.