inferencebench-mt 0.0.2__tar.gz

inferencebench_mt-0.0.2/.gitignore

@@ -0,0 +1,137 @@
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# uv / virtualenv
+.venv/
+venv/
+env/
+ENV/
+uv.lock.tmp
+.python-version
+
+# Testing / coverage
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+htmlcov/
+
+# Type checking
+.mypy_cache/
+.dmypy.json
+dmypy.json
+.pyre/
+.pytype/
+
+# Ruff
+.ruff_cache/
+
+# IDE / editor
+.idea/
+.vscode/
+*.swp
+*.swo
+*~
+.DS_Store
+
+# OS
+Thumbs.db
+desktop.ini
+
+# Secrets / env
+.env
+.env.*
+!.env.example
+.envrc
+
+# Bench-specific local caches
+~/.cache/inferencebench/
+.cache/inferencebench/
+.inferencebench/
+
+# Sigstore dev keys (never commit private keys)
+cosign.key
+cosign-*.key
+cosign-*.pub
+.bench/*.key
+# Local benchmark working dirs (kept local; published outputs land under validation-runs/)
+envelopes-voice/
+envelopes-*/
+*.pem
+!tests/fixtures/**/*.pem
+
+# Real-GPU validation artifacts (kept locally, never pushed)
+# Use slash-star (not trailing slash) so individual subpaths can be re-included below.
+validation-runs/*
+# ...except the canonical published marathon corpus — small, public, used by docs + CI
+!validation-runs/2026-05-18-multi-vendor-marathon
+validation-runs/2026-05-18-multi-vendor-marathon/*
+!validation-runs/2026-05-18-multi-vendor-marathon/marathon
+validation-runs/2026-05-18-multi-vendor-marathon/marathon/*
+!validation-runs/2026-05-18-multi-vendor-marathon/marathon/all
+!validation-runs/2026-05-18-multi-vendor-marathon/marathon/all/*.json
+# Voice ASR validation envelopes (small, public, used by leaderboard build)
+!validation-runs/2026-05-25-voice-rtx4000ada
+!validation-runs/2026-05-25-voice-rtx4000ada/*.json
+!validation-runs/2026-05-29-voice-testbm-h100
+!validation-runs/2026-05-29-voice-testbm-h100/*.json
+
+# Model weights / datasets (use Git LFS or S3)
+*.bin
+*.safetensors
+*.pt
+*.pth
+*.gguf
+*.onnx
+*.parquet
+!tests/fixtures/**/*.parquet
+
+# Logs
+*.log
+logs/
+
+# Documentation build
+docs/_build/
+site/
+
+# Internal-only files (Claude Code context + planning) — kept locally, not pushed
+/CLAUDE.md
+/INDEX.md
+/PROJECT_PLAN.md
+/CONVENTIONS.md
+/HUMAN_REVIEW_GATES.md
+**/CLAUDE.md
+memory/
+skills/
+agents/
+.claude/
+TICKETS/

inferencebench_mt-0.0.2/PKG-INFO

@@ -0,0 +1,39 @@
+Metadata-Version: 2.4
+Name: inferencebench-mt
+Version: 0.0.2
+Summary: Machine-translation plugin for InferenceBench Suite (chrF/BLEU/exact-match on bundled fixtures)
+Project-URL: Homepage, https://github.com/yobitelcomm/bench
+Author-email: Yobitel Communications <bench@yobitel.com>
+License: Apache-2.0
+Keywords: ai,benchmark,bleu,chrf,llm,ml,translation
+Classifier: Development Status :: 2 - Pre-Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.12
+Requires-Dist: inferencebench-envelope
+Requires-Dist: inferencebench-harness
+Requires-Dist: pydantic~=2.9
+Requires-Dist: pyyaml~=6.0
+Description-Content-Type: text/markdown
+
+# inferencebench-mt
+
+Machine-translation plugin for the InferenceBench Suite.
+
+Scores model translations against bundled reference fixtures using chrF (character
+n-gram F-score), token-level BLEU, or exact match. Mirrors the contract of the
+other plugins (`list_benchmarks` / `get_benchmark` / `validate` / `run`) and
+emits the canonical signed envelope.
+
+Two bundled benchmarks ship out of the box:
+
+- `llm.mt.flores-200-mini-en-fr` — FLORES-200-style English to French, chrF.
+- `llm.mt.flores-200-mini-en-de` — FLORES-200-style English to German, chrF.
+
+The fixtures are tiny (eight rows each, mixed across greeting / news / technical
+/ conversational domains) — intended for skeleton verification, not headline
+numbers.

inferencebench_mt-0.0.2/README.md

@@ -0,0 +1,17 @@
+# inferencebench-mt
+
+Machine-translation plugin for the InferenceBench Suite.
+
+Scores model translations against bundled reference fixtures using chrF (character
+n-gram F-score), token-level BLEU, or exact match. Mirrors the contract of the
+other plugins (`list_benchmarks` / `get_benchmark` / `validate` / `run`) and
+emits the canonical signed envelope.
+
+Two bundled benchmarks ship out of the box:
+
+- `llm.mt.flores-200-mini-en-fr` — FLORES-200-style English to French, chrF.
+- `llm.mt.flores-200-mini-en-de` — FLORES-200-style English to German, chrF.
+
+The fixtures are tiny (eight rows each, mixed across greeting / news / technical
+/ conversational domains) — intended for skeleton verification, not headline
+numbers.

inferencebench_mt-0.0.2/pyproject.toml

@@ -0,0 +1,43 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "inferencebench-mt"
+version = "0.0.2"
+description = "Machine-translation plugin for InferenceBench Suite (chrF/BLEU/exact-match on bundled fixtures)"
+readme = "README.md"
+requires-python = ">=3.12"
+license = { text = "Apache-2.0" }
+authors = [
+    { name = "Yobitel Communications", email = "bench@yobitel.com" },
+]
+keywords = ["benchmark", "llm", "translation", "chrf", "bleu", "ai", "ml"]
+classifiers = [
+    "Development Status :: 2 - Pre-Alpha",
+    "Intended Audience :: Developers",
+    "Intended Audience :: Science/Research",
+    "License :: OSI Approved :: Apache Software License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+]
+dependencies = [
+    "inferencebench-envelope",
+    "inferencebench-harness",
+    "pydantic~=2.9",
+    "pyyaml~=6.0",
+]
+
+[project.entry-points."inferencebench.plugins"]
+"llm.mt" = "inferencebench_mt.plugin:LLMMTPlugin"
+
+[project.urls]
+Homepage = "https://github.com/yobitelcomm/bench"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/inferencebench_mt"]
+
+[tool.uv.sources]
+inferencebench-envelope = { workspace = true }
+inferencebench-harness = { workspace = true }

inferencebench_mt-0.0.2/src/inferencebench_mt/__init__.py

@@ -0,0 +1,12 @@
+"""InferenceBench LLM machine-translation plugin."""
+
+from inferencebench_mt.plugin import EXPECTED_METRICS, LLMMTPlugin
+from inferencebench_mt.schemas import BenchmarkSpec, EngineKind, RunContext
+
+__all__ = [
+    "EXPECTED_METRICS",
+    "BenchmarkSpec",
+    "EngineKind",
+    "LLMMTPlugin",
+    "RunContext",
+]

inferencebench_mt-0.0.2/src/inferencebench_mt/benchmarks/flores-200-mini-en-de.yaml

@@ -0,0 +1,14 @@
+benchmark_id: llm.mt.flores-200-mini-en-de
+suite_version: 1.0.0
+description: FLORES-200-style English to German, character F-score.
+modality: llm
+kind: translation
+dataset:
+  id: builtin-flores-mini-en-de
+  path: flores-mini-en-de.jsonl
+slo_template: llm.mt.standard
+warmup:
+  discard_runs: 0
+scoring: chrf
+source_lang: en
+target_lang: de

inferencebench_mt-0.0.2/src/inferencebench_mt/benchmarks/flores-200-mini-en-es.yaml

@@ -0,0 +1,14 @@
+benchmark_id: llm.mt.flores-200-mini-en-es
+suite_version: 1.0.0
+description: FLORES-200-style English to Spanish, character F-score.
+modality: llm
+kind: translation
+dataset:
+  id: builtin-flores-mini-en-es
+  path: flores-mini-en-es.jsonl
+slo_template: llm.mt.standard
+warmup:
+  discard_runs: 0
+scoring: chrf
+source_lang: en
+target_lang: es

inferencebench_mt-0.0.2/src/inferencebench_mt/benchmarks/flores-200-mini-en-fr.yaml

@@ -0,0 +1,14 @@
+benchmark_id: llm.mt.flores-200-mini-en-fr
+suite_version: 1.0.0
+description: FLORES-200-style English to French, character F-score.
+modality: llm
+kind: translation
+dataset:
+  id: builtin-flores-mini-en-fr
+  path: flores-mini-en-fr.jsonl
+slo_template: llm.mt.standard
+warmup:
+  discard_runs: 0
+scoring: chrf
+source_lang: en
+target_lang: fr

inferencebench_mt-0.0.2/src/inferencebench_mt/benchmarks/flores-200-mini-en-ja.yaml

@@ -0,0 +1,14 @@
+benchmark_id: llm.mt.flores-200-mini-en-ja
+suite_version: 1.0.0
+description: FLORES-200-style English to Japanese, character F-score.
+modality: llm
+kind: translation
+dataset:
+  id: builtin-flores-mini-en-ja
+  path: flores-mini-en-ja.jsonl
+slo_template: llm.mt.standard
+warmup:
+  discard_runs: 0
+scoring: chrf
+source_lang: en
+target_lang: ja

inferencebench_mt-0.0.2/src/inferencebench_mt/datasets/flores-mini-en-de.jsonl

@@ -0,0 +1,8 @@
+{"source": "Hello, how are you?", "reference": "Hallo, wie geht es dir?", "domain": "greeting"}
+{"source": "Good morning, my friend.", "reference": "Guten Morgen, mein Freund.", "domain": "greeting"}
+{"source": "The president signed the new climate agreement yesterday.", "reference": "Der Präsident unterzeichnete gestern das neue Klimaabkommen.", "domain": "news"}
+{"source": "Stock markets fell sharply after the central bank announcement.", "reference": "Die Aktienmärkte fielen nach der Ankündigung der Zentralbank stark.", "domain": "news"}
+{"source": "The transformer architecture uses self-attention layers.", "reference": "Die Transformer-Architektur verwendet Self-Attention-Schichten.", "domain": "technical"}
+{"source": "Please restart the server after the update completes.", "reference": "Bitte starten Sie den Server neu, nachdem das Update abgeschlossen ist.", "domain": "technical"}
+{"source": "I would like a coffee with milk and sugar, please.", "reference": "Ich hätte gerne einen Kaffee mit Milch und Zucker, bitte.", "domain": "conversational"}
+{"source": "Where is the nearest train station?", "reference": "Wo ist der nächste Bahnhof?", "domain": "conversational"}

inferencebench_mt-0.0.2/src/inferencebench_mt/datasets/flores-mini-en-es.jsonl

@@ -0,0 +1,8 @@
+{"source": "Hello, how are you?", "reference": "Hola, ¿cómo estás?", "domain": "greeting"}
+{"source": "Good morning, my friend.", "reference": "Buenos días, mi amigo.", "domain": "greeting"}
+{"source": "The president signed the new climate agreement yesterday.", "reference": "El presidente firmó ayer el nuevo acuerdo climático.", "domain": "news"}
+{"source": "Stock markets fell sharply after the central bank announcement.", "reference": "Los mercados bursátiles cayeron bruscamente tras el anuncio del banco central.", "domain": "news"}
+{"source": "The transformer architecture uses self-attention layers.", "reference": "La arquitectura transformer utiliza capas de autoatención.", "domain": "technical"}
+{"source": "Please restart the server after the update completes.", "reference": "Por favor, reinicia el servidor cuando termine la actualización.", "domain": "technical"}
+{"source": "I would like a coffee with milk and sugar, please.", "reference": "Quisiera un café con leche y azúcar, por favor.", "domain": "conversational"}
+{"source": "Where is the nearest train station?", "reference": "¿Dónde está la estación de tren más cercana?", "domain": "conversational"}

inferencebench_mt-0.0.2/src/inferencebench_mt/datasets/flores-mini-en-fr.jsonl

@@ -0,0 +1,8 @@
+{"source": "Hello, how are you?", "reference": "Bonjour, comment allez-vous ?", "domain": "greeting"}
+{"source": "Good morning, my friend.", "reference": "Bonjour, mon ami.", "domain": "greeting"}
+{"source": "The president signed the new climate agreement yesterday.", "reference": "Le président a signé le nouvel accord sur le climat hier.", "domain": "news"}
+{"source": "Stock markets fell sharply after the central bank announcement.", "reference": "Les marchés boursiers ont fortement chuté après l'annonce de la banque centrale.", "domain": "news"}
+{"source": "The transformer architecture uses self-attention layers.", "reference": "L'architecture transformeur utilise des couches d'auto-attention.", "domain": "technical"}
+{"source": "Please restart the server after the update completes.", "reference": "Veuillez redémarrer le serveur une fois la mise à jour terminée.", "domain": "technical"}
+{"source": "I would like a coffee with milk and sugar, please.", "reference": "Je voudrais un café avec du lait et du sucre, s'il vous plaît.", "domain": "conversational"}
+{"source": "Where is the nearest train station?", "reference": "Où se trouve la gare la plus proche ?", "domain": "conversational"}

inferencebench_mt-0.0.2/src/inferencebench_mt/datasets/flores-mini-en-ja.jsonl

@@ -0,0 +1,8 @@
+{"source": "Hello, how are you?", "reference": "こんにちは、お元気ですか？", "domain": "greeting"}
+{"source": "Good morning, my friend.", "reference": "おはよう、友よ。", "domain": "greeting"}
+{"source": "The president signed the new climate agreement yesterday.", "reference": "大統領は昨日、新しい気候協定に署名しました。", "domain": "news"}
+{"source": "Stock markets fell sharply after the central bank announcement.", "reference": "中央銀行の発表を受けて、株式市場は急落しました。", "domain": "news"}
+{"source": "The transformer architecture uses self-attention layers.", "reference": "トランスフォーマーアーキテクチャは自己注意層を使用します。", "domain": "technical"}
+{"source": "Please restart the server after the update completes.", "reference": "アップデートが完了したらサーバーを再起動してください。", "domain": "technical"}
+{"source": "I would like a coffee with milk and sugar, please.", "reference": "ミルクと砂糖入りのコーヒーをお願いします。", "domain": "conversational"}
+{"source": "Where is the nearest train station?", "reference": "最寄りの駅はどこですか？", "domain": "conversational"}