inferencebench-leaderboard 0.0.2__tar.gz

inferencebench_leaderboard-0.0.2/.gitignore

@@ -0,0 +1,137 @@
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# uv / virtualenv
+.venv/
+venv/
+env/
+ENV/
+uv.lock.tmp
+.python-version
+
+# Testing / coverage
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+htmlcov/
+
+# Type checking
+.mypy_cache/
+.dmypy.json
+dmypy.json
+.pyre/
+.pytype/
+
+# Ruff
+.ruff_cache/
+
+# IDE / editor
+.idea/
+.vscode/
+*.swp
+*.swo
+*~
+.DS_Store
+
+# OS
+Thumbs.db
+desktop.ini
+
+# Secrets / env
+.env
+.env.*
+!.env.example
+.envrc
+
+# Bench-specific local caches
+~/.cache/inferencebench/
+.cache/inferencebench/
+.inferencebench/
+
+# Sigstore dev keys (never commit private keys)
+cosign.key
+cosign-*.key
+cosign-*.pub
+.bench/*.key
+# Local benchmark working dirs (kept local; published outputs land under validation-runs/)
+envelopes-voice/
+envelopes-*/
+*.pem
+!tests/fixtures/**/*.pem
+
+# Real-GPU validation artifacts (kept locally, never pushed)
+# Use slash-star (not trailing slash) so individual subpaths can be re-included below.
+validation-runs/*
+# ...except the canonical published marathon corpus — small, public, used by docs + CI
+!validation-runs/2026-05-18-multi-vendor-marathon
+validation-runs/2026-05-18-multi-vendor-marathon/*
+!validation-runs/2026-05-18-multi-vendor-marathon/marathon
+validation-runs/2026-05-18-multi-vendor-marathon/marathon/*
+!validation-runs/2026-05-18-multi-vendor-marathon/marathon/all
+!validation-runs/2026-05-18-multi-vendor-marathon/marathon/all/*.json
+# Voice ASR validation envelopes (small, public, used by leaderboard build)
+!validation-runs/2026-05-25-voice-rtx4000ada
+!validation-runs/2026-05-25-voice-rtx4000ada/*.json
+!validation-runs/2026-05-29-voice-testbm-h100
+!validation-runs/2026-05-29-voice-testbm-h100/*.json
+
+# Model weights / datasets (use Git LFS or S3)
+*.bin
+*.safetensors
+*.pt
+*.pth
+*.gguf
+*.onnx
+*.parquet
+!tests/fixtures/**/*.parquet
+
+# Logs
+*.log
+logs/
+
+# Documentation build
+docs/_build/
+site/
+
+# Internal-only files (Claude Code context + planning) — kept locally, not pushed
+/CLAUDE.md
+/INDEX.md
+/PROJECT_PLAN.md
+/CONVENTIONS.md
+/HUMAN_REVIEW_GATES.md
+**/CLAUDE.md
+memory/
+skills/
+agents/
+.claude/
+TICKETS/

inferencebench_leaderboard-0.0.2/PKG-INFO

@@ -0,0 +1,112 @@
+Metadata-Version: 2.4
+Name: inferencebench-leaderboard
+Version: 0.0.2
+Summary: Static-site leaderboard renderer for signed InferenceBench envelopes
+Project-URL: Homepage, https://github.com/yobitelcomm/bench
+Project-URL: Documentation, https://yobitelcomm.github.io/bench
+Author-email: Yobitel Communications <bench@yobitel.com>
+License: Apache-2.0
+Keywords: ai,benchmark,inference,leaderboard,llm,ml,static-site
+Classifier: Development Status :: 2 - Pre-Alpha
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.12
+Requires-Dist: inferencebench-envelope
+Requires-Dist: jinja2~=3.1
+Requires-Dist: pydantic~=2.9
+Requires-Dist: typer~=0.12
+Description-Content-Type: text/markdown
+
+# inferencebench-leaderboard
+
+Static-site renderer that turns a directory of signed InferenceBench envelope
+JSONs into a plain HTML+CSS+JSON leaderboard suitable for GitHub Pages
+(`https://yobitelcomm.github.io/bench`). Vendor-neutral, no JavaScript
+frameworks; the only client-side code is a ~40-line vanilla sorter for the
+tables.
+
+## Install (workspace)
+
+This package is a `uv` workspace member of the root `bench/` monorepo:
+
+```toml
+# bench/pyproject.toml
+[tool.uv.workspace]
+members = [..., "tools/leaderboard"]
+```
+
+Then `uv sync` from the repo root.
+
+## Build a site
+
+```
+python -m inferencebench_leaderboard build envelopes/ site/
+# or
+inferencebench-leaderboard build envelopes/ site/ --base-url /bench/
+```
+
+`envelopes/` must contain `*.json` files that parse against the canonical
+`inferencebench.envelope.Envelope` Pydantic model. Files that don't parse
+are logged and skipped; the rest of the site still renders.
+
+## Output layout
+
+```
+site/
+  index.html                       — category index
+  static/site.css                  — Hacker News-style table CSS
+  static/sort.js                   — vanilla sort for tables
+  envelopes/<file>.json            — verbatim copies for `bench verify`
+  <suite_id>/index.html            — per-category table
+  <suite_id>/<run_id>.html         — per-entry detail (verify snippet)
+  data/leaderboard.json            — machine-readable index
+```
+
+## Public API
+
+```python
+from inferencebench_leaderboard import (
+    render_site,           # main entry point
+    SiteRenderResult,      # return type
+    load_envelopes,        # directory -> [LoadedEnvelope]
+    compute_pareto,        # Pareto-frontier classifier
+    LoadedEnvelope,
+    PARETO_DIRECTIONS,
+)
+```
+
+`compute_pareto` accepts per-axis `direction={"min","max"}` so the same
+function works for throughput-vs-latency, latency-vs-cost, etc.
+
+## Tests
+
+```
+pytest tools/leaderboard/tests/
+```
+
+Covers: smoke render, Pareto math on synthetic data, schema-validation
+skipping.
+
+## Hosted at GitHub Pages
+
+The marathon corpus committed at
+`validation-runs/2026-05-18-multi-vendor-marathon/marathon/all/` is rendered
+and deployed to **<https://yobitelcomm.github.io/bench/>** by
+`.github/workflows/deploy-pages.yml`. The workflow re-runs on push to main
+(when leaderboard code or corpus envelopes change), weekly on schedule, and on
+manual dispatch from the Actions tab.
+
+To preview locally before pushing:
+
+```
+uv run bench leaderboard --build \
+  --envelopes validation-runs/2026-05-18-multi-vendor-marathon/marathon/all \
+  --out _site --base-url /bench/
+python3 -m http.server --directory _site 8080
+# open http://localhost:8080/bench/  (note the /bench/ prefix matches --base-url)
+```

inferencebench_leaderboard-0.0.2/README.md

@@ -0,0 +1,88 @@
+# inferencebench-leaderboard
+
+Static-site renderer that turns a directory of signed InferenceBench envelope
+JSONs into a plain HTML+CSS+JSON leaderboard suitable for GitHub Pages
+(`https://yobitelcomm.github.io/bench`). Vendor-neutral, no JavaScript
+frameworks; the only client-side code is a ~40-line vanilla sorter for the
+tables.
+
+## Install (workspace)
+
+This package is a `uv` workspace member of the root `bench/` monorepo:
+
+```toml
+# bench/pyproject.toml
+[tool.uv.workspace]
+members = [..., "tools/leaderboard"]
+```
+
+Then `uv sync` from the repo root.
+
+## Build a site
+
+```
+python -m inferencebench_leaderboard build envelopes/ site/
+# or
+inferencebench-leaderboard build envelopes/ site/ --base-url /bench/
+```
+
+`envelopes/` must contain `*.json` files that parse against the canonical
+`inferencebench.envelope.Envelope` Pydantic model. Files that don't parse
+are logged and skipped; the rest of the site still renders.
+
+## Output layout
+
+```
+site/
+  index.html                       — category index
+  static/site.css                  — Hacker News-style table CSS
+  static/sort.js                   — vanilla sort for tables
+  envelopes/<file>.json            — verbatim copies for `bench verify`
+  <suite_id>/index.html            — per-category table
+  <suite_id>/<run_id>.html         — per-entry detail (verify snippet)
+  data/leaderboard.json            — machine-readable index
+```
+
+## Public API
+
+```python
+from inferencebench_leaderboard import (
+    render_site,           # main entry point
+    SiteRenderResult,      # return type
+    load_envelopes,        # directory -> [LoadedEnvelope]
+    compute_pareto,        # Pareto-frontier classifier
+    LoadedEnvelope,
+    PARETO_DIRECTIONS,
+)
+```
+
+`compute_pareto` accepts per-axis `direction={"min","max"}` so the same
+function works for throughput-vs-latency, latency-vs-cost, etc.
+
+## Tests
+
+```
+pytest tools/leaderboard/tests/
+```
+
+Covers: smoke render, Pareto math on synthetic data, schema-validation
+skipping.
+
+## Hosted at GitHub Pages
+
+The marathon corpus committed at
+`validation-runs/2026-05-18-multi-vendor-marathon/marathon/all/` is rendered
+and deployed to **<https://yobitelcomm.github.io/bench/>** by
+`.github/workflows/deploy-pages.yml`. The workflow re-runs on push to main
+(when leaderboard code or corpus envelopes change), weekly on schedule, and on
+manual dispatch from the Actions tab.
+
+To preview locally before pushing:
+
+```
+uv run bench leaderboard --build \
+  --envelopes validation-runs/2026-05-18-multi-vendor-marathon/marathon/all \
+  --out _site --base-url /bench/
+python3 -m http.server --directory _site 8080
+# open http://localhost:8080/bench/  (note the /bench/ prefix matches --base-url)
+```

inferencebench_leaderboard-0.0.2/pyproject.toml

@@ -0,0 +1,47 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "inferencebench-leaderboard"
+version = "0.0.2"
+description = "Static-site leaderboard renderer for signed InferenceBench envelopes"
+readme = "README.md"
+requires-python = ">=3.12"
+license = { text = "Apache-2.0" }
+authors = [
+    { name = "Yobitel Communications", email = "bench@yobitel.com" },
+]
+keywords = ["benchmark", "ai", "ml", "llm", "inference", "leaderboard", "static-site"]
+classifiers = [
+    "Development Status :: 2 - Pre-Alpha",
+    "Environment :: Console",
+    "Intended Audience :: Developers",
+    "Intended Audience :: Science/Research",
+    "License :: OSI Approved :: Apache Software License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+]
+dependencies = [
+    "inferencebench-envelope",
+    "jinja2~=3.1",
+    "typer~=0.12",
+    "pydantic~=2.9",
+]
+
+[project.scripts]
+inferencebench-leaderboard = "inferencebench_leaderboard.cli:app"
+
+[project.urls]
+Homepage = "https://github.com/yobitelcomm/bench"
+Documentation = "https://yobitelcomm.github.io/bench"
+
+[tool.uv.sources]
+inferencebench-envelope = { workspace = true }
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/inferencebench_leaderboard"]
+# templates/ and static/ live inside src/inferencebench_leaderboard so they are
+# already picked up by the package include. A separate force-include would
+# duplicate every entry in the wheel zip — PyPI rejects with 400.

inferencebench_leaderboard-0.0.2/src/inferencebench_leaderboard/__init__.py

@@ -0,0 +1,39 @@
+"""Static-site leaderboard renderer for InferenceBench signed envelopes.
+
+Reads a directory of canonical envelope JSONs, groups them by ``suite_id``,
+and emits a static HTML + CSS + JSON site suitable for hosting on GitHub Pages
+(target: https://yobitelcomm.github.io/bench).
+
+Public surface:
+
+    render_site(envelopes_dir, out_dir, *, base_url="/") -> SiteRenderResult
+    SiteRenderResult       — summary of a render pass (counts, skipped, paths)
+    LoadedEnvelope         — parsed envelope plus its source filename
+    load_envelopes(dir)    — collect parseable envelopes from a directory
+    compute_pareto(...)    — Pareto-frontier classifier for arbitrary axes
+
+The renderer is deliberately framework-free: plain HTML + a small static CSS
+file, no client-side bundler, no JS frameworks. A tiny vanilla sort script
+ships with the site to make the tables sortable.
+"""
+
+from __future__ import annotations
+
+from inferencebench_leaderboard.data import (
+    PARETO_DIRECTIONS,
+    LoadedEnvelope,
+    compute_pareto,
+    load_envelopes,
+)
+from inferencebench_leaderboard.render import SiteRenderResult, render_site
+
+__all__ = [
+    "PARETO_DIRECTIONS",
+    "LoadedEnvelope",
+    "SiteRenderResult",
+    "compute_pareto",
+    "load_envelopes",
+    "render_site",
+]
+
+__version__ = "0.0.0"

inferencebench_leaderboard-0.0.2/src/inferencebench_leaderboard/__main__.py

@@ -0,0 +1,8 @@
+"""Allow ``python -m inferencebench_leaderboard`` to work."""
+
+from __future__ import annotations
+
+from inferencebench_leaderboard.cli import app
+
+if __name__ == "__main__":
+    app()

inferencebench_leaderboard-0.0.2/src/inferencebench_leaderboard/cli.py

@@ -0,0 +1,62 @@
+"""Typer CLI entry point for the static leaderboard renderer.
+
+Invoke with::
+
+    python -m inferencebench_leaderboard build envelopes/ site/
+    inferencebench-leaderboard build envelopes/ site/
+
+The ``build`` subcommand is the only operation today; future subcommands
+(e.g. ``validate``, ``diff``) plug in here.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import typer
+
+from inferencebench_leaderboard.render import render_site
+
+app = typer.Typer(
+    add_completion=False,
+    no_args_is_help=True,
+    help="Render the static InferenceBench leaderboard site.",
+)
+
+
+@app.command("build")
+def build(
+    envelopes_dir: Path = typer.Argument(
+        ...,
+        exists=True,
+        file_okay=False,
+        dir_okay=True,
+        readable=True,
+        help="Directory containing signed envelope JSON files.",
+    ),
+    out_dir: Path = typer.Argument(
+        ...,
+        file_okay=False,
+        dir_okay=True,
+        help="Destination directory for the generated static site.",
+    ),
+    base_url: str = typer.Option(
+        "/",
+        "--base-url",
+        help="URL prefix the site will be served from (e.g. '/bench/').",
+    ),
+) -> None:
+    """Render the static leaderboard site to ``out_dir``."""
+    result = render_site(envelopes_dir, out_dir, base_url=base_url)
+    typer.echo(
+        f"Rendered {result.envelopes_loaded} envelope(s) "
+        f"across {len(result.categories)} category(ies) "
+        f"to {result.out_dir} "
+        f"({result.envelopes_skipped} skipped, {result.pages_written} files written)."
+    )
+    for suite_id, count in sorted(result.categories.items()):
+        typer.echo(f"  - {suite_id}: {count}")
+
+
+if __name__ == "__main__":  # pragma: no cover
+    app()

inferencebench_leaderboard-0.0.2/src/inferencebench_leaderboard/data.py

@@ -0,0 +1,147 @@
+"""Envelope loading and Pareto-frontier math for the static leaderboard.
+
+Both helpers are pure: they do not touch disk except via the directory passed
+in to :func:`load_envelopes`. Malformed JSON files and JSON blobs that fail
+Pydantic validation are *skipped*, never crashed, because the leaderboard is
+populated from community contributions and one bad envelope must not break
+the whole site.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Literal
+
+from inferencebench.envelope import Envelope
+
+logger = logging.getLogger(__name__)
+
+Direction = Literal["min", "max"]
+
+
+# Convention for the leaderboard's headline metrics.  Lower is better for
+# latency and price, higher is better for throughput.  The renderer uses this
+# map to decide which axis direction to feed :func:`compute_pareto`.
+PARETO_DIRECTIONS: dict[str, Direction] = {
+    "ttft_p50_ms": "min",
+    "ttft_p99_ms": "min",
+    "itl_p50_ms": "min",
+    "itl_p99_ms": "min",
+    "throughput_tok_per_s": "max",
+    "goodput_tok_per_s": "max",
+    "cost_per_m_tokens_usd": "min",
+    "joules_per_token": "min",
+    "energy_per_token_j": "min",
+}
+
+
+@dataclass(frozen=True, slots=True)
+class LoadedEnvelope:
+    """A parsed envelope and the source filename it came from.
+
+    The filename is kept so the rendered site can link to a stable JSON path
+    (``/envelopes/<filename>``) for ``bench verify`` to re-download and check.
+    """
+
+    source_filename: str
+    envelope: Envelope
+
+
+def load_envelopes(envelopes_dir: Path) -> list[LoadedEnvelope]:
+    """Load every ``*.json`` under ``envelopes_dir`` that parses as an Envelope.
+
+    Files that are not valid JSON, or that parse as JSON but fail Pydantic
+    validation against :class:`inferencebench.envelope.Envelope`, are logged
+    at WARNING level and skipped.  This is intentional: the leaderboard
+    aggregates community-submitted envelopes and one malformed file must not
+    take down the rest of the site.
+
+    Args:
+        envelopes_dir: Directory containing envelope JSON files.
+
+    Returns:
+        Sorted list of successfully parsed envelopes (by source filename).
+    """
+    if not envelopes_dir.exists() or not envelopes_dir.is_dir():
+        logger.warning("envelopes_dir does not exist or is not a directory: %s", envelopes_dir)
+        return []
+
+    loaded: list[LoadedEnvelope] = []
+    for path in sorted(envelopes_dir.glob("*.json")):
+        try:
+            raw = path.read_text(encoding="utf-8")
+        except OSError as exc:
+            logger.warning("could not read envelope file %s: %s", path, exc)
+            continue
+        try:
+            data = json.loads(raw)
+        except json.JSONDecodeError as exc:
+            logger.warning("skipping non-JSON file %s: %s", path, exc)
+            continue
+        try:
+            envelope = Envelope.model_validate(data)
+        except (ValueError, TypeError) as exc:
+            logger.warning("skipping invalid envelope %s: %s", path, exc)
+            continue
+        loaded.append(LoadedEnvelope(source_filename=path.name, envelope=envelope))
+    return loaded
+
+
+def compute_pareto(
+    entries: list[tuple[float | None, float | None]],
+    *,
+    x_direction: Direction = "max",
+    y_direction: Direction = "min",
+) -> list[bool]:
+    """Classify each ``(x, y)`` pair as on the Pareto frontier or dominated.
+
+    A point ``p`` is *dominated* by another point ``q`` when ``q`` is at least
+    as good as ``p`` on both axes and strictly better on at least one.  The
+    direction parameters say whether higher or lower is "better" per axis:
+
+    - ``x_direction="max", y_direction="min"`` (default): higher x and lower y
+      are both improvements.  Matches the canonical
+      throughput-vs-latency plot.
+    - ``x_direction="min", y_direction="min"``: both axes minimize (e.g.
+      latency vs. cost).
+    - Any other combination is supported analogously.
+
+    Entries containing ``None`` on either axis are never on the frontier
+    (treated as missing data, marked ``False``).
+
+    Args:
+        entries: List of ``(x, y)`` coordinate tuples; ``None`` denotes missing.
+        x_direction: ``"max"`` if higher x is better, ``"min"`` if lower.
+        y_direction: ``"max"`` if higher y is better, ``"min"`` if lower.
+
+    Returns:
+        A list of booleans, one per input, ``True`` iff the entry is on the
+        Pareto frontier.
+    """
+
+    def better_or_equal(a: float, b: float, direction: Direction) -> bool:
+        return a >= b if direction == "max" else a <= b
+
+    def strictly_better(a: float, b: float, direction: Direction) -> bool:
+        return a > b if direction == "max" else a < b
+
+    result = [False] * len(entries)
+    for i, (xi, yi) in enumerate(entries):
+        if xi is None or yi is None:
+            continue
+        dominated = False
+        for j, (xj, yj) in enumerate(entries):
+            if i == j or xj is None or yj is None:
+                continue
+            if (
+                better_or_equal(xj, xi, x_direction)
+                and better_or_equal(yj, yi, y_direction)
+                and (strictly_better(xj, xi, x_direction) or strictly_better(yj, yi, y_direction))
+            ):
+                dominated = True
+                break
+        result[i] = not dominated
+    return result