inferencebench-mt 0.0.2__py3-none-any.whl

inferencebench_mt/__init__.py

@@ -0,0 +1,12 @@
+"""InferenceBench LLM machine-translation plugin."""
+
+from inferencebench_mt.plugin import EXPECTED_METRICS, LLMMTPlugin
+from inferencebench_mt.schemas import BenchmarkSpec, EngineKind, RunContext
+
+__all__ = [
+    "EXPECTED_METRICS",
+    "BenchmarkSpec",
+    "EngineKind",
+    "LLMMTPlugin",
+    "RunContext",
+]

inferencebench_mt/benchmarks/flores-200-mini-en-de.yaml

@@ -0,0 +1,14 @@
+benchmark_id: llm.mt.flores-200-mini-en-de
+suite_version: 1.0.0
+description: FLORES-200-style English to German, character F-score.
+modality: llm
+kind: translation
+dataset:
+  id: builtin-flores-mini-en-de
+  path: flores-mini-en-de.jsonl
+slo_template: llm.mt.standard
+warmup:
+  discard_runs: 0
+scoring: chrf
+source_lang: en
+target_lang: de

inferencebench_mt/benchmarks/flores-200-mini-en-es.yaml

@@ -0,0 +1,14 @@
+benchmark_id: llm.mt.flores-200-mini-en-es
+suite_version: 1.0.0
+description: FLORES-200-style English to Spanish, character F-score.
+modality: llm
+kind: translation
+dataset:
+  id: builtin-flores-mini-en-es
+  path: flores-mini-en-es.jsonl
+slo_template: llm.mt.standard
+warmup:
+  discard_runs: 0
+scoring: chrf
+source_lang: en
+target_lang: es

inferencebench_mt/benchmarks/flores-200-mini-en-fr.yaml

@@ -0,0 +1,14 @@
+benchmark_id: llm.mt.flores-200-mini-en-fr
+suite_version: 1.0.0
+description: FLORES-200-style English to French, character F-score.
+modality: llm
+kind: translation
+dataset:
+  id: builtin-flores-mini-en-fr
+  path: flores-mini-en-fr.jsonl
+slo_template: llm.mt.standard
+warmup:
+  discard_runs: 0
+scoring: chrf
+source_lang: en
+target_lang: fr

inferencebench_mt/benchmarks/flores-200-mini-en-ja.yaml

@@ -0,0 +1,14 @@
+benchmark_id: llm.mt.flores-200-mini-en-ja
+suite_version: 1.0.0
+description: FLORES-200-style English to Japanese, character F-score.
+modality: llm
+kind: translation
+dataset:
+  id: builtin-flores-mini-en-ja
+  path: flores-mini-en-ja.jsonl
+slo_template: llm.mt.standard
+warmup:
+  discard_runs: 0
+scoring: chrf
+source_lang: en
+target_lang: ja

inferencebench_mt/datasets/flores-mini-en-de.jsonl

@@ -0,0 +1,8 @@
+{"source": "Hello, how are you?", "reference": "Hallo, wie geht es dir?", "domain": "greeting"}
+{"source": "Good morning, my friend.", "reference": "Guten Morgen, mein Freund.", "domain": "greeting"}
+{"source": "The president signed the new climate agreement yesterday.", "reference": "Der Präsident unterzeichnete gestern das neue Klimaabkommen.", "domain": "news"}
+{"source": "Stock markets fell sharply after the central bank announcement.", "reference": "Die Aktienmärkte fielen nach der Ankündigung der Zentralbank stark.", "domain": "news"}
+{"source": "The transformer architecture uses self-attention layers.", "reference": "Die Transformer-Architektur verwendet Self-Attention-Schichten.", "domain": "technical"}
+{"source": "Please restart the server after the update completes.", "reference": "Bitte starten Sie den Server neu, nachdem das Update abgeschlossen ist.", "domain": "technical"}
+{"source": "I would like a coffee with milk and sugar, please.", "reference": "Ich hätte gerne einen Kaffee mit Milch und Zucker, bitte.", "domain": "conversational"}
+{"source": "Where is the nearest train station?", "reference": "Wo ist der nächste Bahnhof?", "domain": "conversational"}

inferencebench_mt/datasets/flores-mini-en-es.jsonl

@@ -0,0 +1,8 @@
+{"source": "Hello, how are you?", "reference": "Hola, ¿cómo estás?", "domain": "greeting"}
+{"source": "Good morning, my friend.", "reference": "Buenos días, mi amigo.", "domain": "greeting"}
+{"source": "The president signed the new climate agreement yesterday.", "reference": "El presidente firmó ayer el nuevo acuerdo climático.", "domain": "news"}
+{"source": "Stock markets fell sharply after the central bank announcement.", "reference": "Los mercados bursátiles cayeron bruscamente tras el anuncio del banco central.", "domain": "news"}
+{"source": "The transformer architecture uses self-attention layers.", "reference": "La arquitectura transformer utiliza capas de autoatención.", "domain": "technical"}
+{"source": "Please restart the server after the update completes.", "reference": "Por favor, reinicia el servidor cuando termine la actualización.", "domain": "technical"}
+{"source": "I would like a coffee with milk and sugar, please.", "reference": "Quisiera un café con leche y azúcar, por favor.", "domain": "conversational"}
+{"source": "Where is the nearest train station?", "reference": "¿Dónde está la estación de tren más cercana?", "domain": "conversational"}

inferencebench_mt/datasets/flores-mini-en-fr.jsonl

@@ -0,0 +1,8 @@
+{"source": "Hello, how are you?", "reference": "Bonjour, comment allez-vous ?", "domain": "greeting"}
+{"source": "Good morning, my friend.", "reference": "Bonjour, mon ami.", "domain": "greeting"}
+{"source": "The president signed the new climate agreement yesterday.", "reference": "Le président a signé le nouvel accord sur le climat hier.", "domain": "news"}
+{"source": "Stock markets fell sharply after the central bank announcement.", "reference": "Les marchés boursiers ont fortement chuté après l'annonce de la banque centrale.", "domain": "news"}
+{"source": "The transformer architecture uses self-attention layers.", "reference": "L'architecture transformeur utilise des couches d'auto-attention.", "domain": "technical"}
+{"source": "Please restart the server after the update completes.", "reference": "Veuillez redémarrer le serveur une fois la mise à jour terminée.", "domain": "technical"}
+{"source": "I would like a coffee with milk and sugar, please.", "reference": "Je voudrais un café avec du lait et du sucre, s'il vous plaît.", "domain": "conversational"}
+{"source": "Where is the nearest train station?", "reference": "Où se trouve la gare la plus proche ?", "domain": "conversational"}

inferencebench_mt/datasets/flores-mini-en-ja.jsonl

@@ -0,0 +1,8 @@
+{"source": "Hello, how are you?", "reference": "こんにちは、お元気ですか？", "domain": "greeting"}
+{"source": "Good morning, my friend.", "reference": "おはよう、友よ。", "domain": "greeting"}
+{"source": "The president signed the new climate agreement yesterday.", "reference": "大統領は昨日、新しい気候協定に署名しました。", "domain": "news"}
+{"source": "Stock markets fell sharply after the central bank announcement.", "reference": "中央銀行の発表を受けて、株式市場は急落しました。", "domain": "news"}
+{"source": "The transformer architecture uses self-attention layers.", "reference": "トランスフォーマーアーキテクチャは自己注意層を使用します。", "domain": "technical"}
+{"source": "Please restart the server after the update completes.", "reference": "アップデートが完了したらサーバーを再起動してください。", "domain": "technical"}
+{"source": "I would like a coffee with milk and sugar, please.", "reference": "ミルクと砂糖入りのコーヒーをお願いします。", "domain": "conversational"}
+{"source": "Where is the nearest train station?", "reference": "最寄りの駅はどこですか？", "domain": "conversational"}

inferencebench_mt/plugin.py

@@ -0,0 +1,468 @@
+"""LLMMTPlugin — entry point for ``llm.mt`` benchmarks.
+
+The fifth-modality sibling to the perf / quality / voice / embeddings plugins.
+Implements the same contract (``list_benchmarks`` / ``get_benchmark`` /
+``validate`` / ``run``) but its headline metric is translation accuracy —
+chrF (character n-gram F-score) by default, with token-BLEU and exact-match
+as alternates. Scoring is deterministic and dependency-free; see
+:mod:`inferencebench_mt.scoring`.
+
+The plugin drives a real :class:`ModelClient` per fixture row, constructs an
+MT prompt (``"Translate from {src} to {tgt}: ..."``), and scores the model's
+hypothesis against the bundled reference. Self-hosted OpenAI-compatible
+endpoints (vLLM, SGLang) get the LiteLLM ``openai/`` routing prefix added
+exactly once — same convention as the llm-quality plugin.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import math
+import os
+import time
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+import yaml
+
+from inferencebench.envelope import (
+    DatasetSpec as EnvDatasetSpec,
+)
+from inferencebench.envelope import (
+    EngineConfig,
+    Envelope,
+    EnvelopeBuilder,
+    ModelConfig,
+    Quantization,
+    SigningMode,
+    sign_envelope,
+)
+from inferencebench.harness import (
+    CompletionResult,
+    ModelClient,
+    Sample,
+    collect_hardware_fingerprint,
+    collect_software_provenance,
+)
+from inferencebench.harness.metrics import EnergyReport, Percentiles, TelemetryWindow
+from inferencebench_mt.schemas import BenchmarkSpec, EngineKind, RunContext
+from inferencebench_mt.scoring import SCORERS
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
+
+def _json_num(v: float) -> str:
+    """JSON-safe numeric encoder: NaN/inf become null."""
+    if isinstance(v, float) and (math.isnan(v) or math.isinf(v)):
+        return "null"
+    return repr(v)
+
+
+def _json_str(v: str | None) -> str:
+    """JSON-safe string encoder."""
+    return json.dumps(v if v is not None else "")
+
+
+# Engines that require ``base_url`` (self-hosted OpenAI-compatible servers).
+# OPENAI / COHERE here mean "provider-hosted endpoint" — base_url is optional.
+_SELF_HOSTED_ENGINES = frozenset({EngineKind.VLLM, EngineKind.SGLANG})
+
+
+def _fixtures_cache_root() -> Path:
+    """Resolve the bench-fixtures cache root for ``fixtures://`` dataset URIs."""
+    override = os.environ.get("BENCH_FIXTURES_ROOT")
+    if override:
+        return Path(override)
+    return Path.home() / ".cache" / "inferencebench" / "fixtures"
+
+
+def _compute_fixture_hash(items: list[dict[str, str]]) -> str:
+    """SHA-256 over the canonical-JSON-encoded fixture rows."""
+    canonical = json.dumps(items, sort_keys=True, separators=(",", ":"))
+    return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
+
+
+def _build_client(context: RunContext, *, timeout_s: float = 60.0) -> ModelClient:
+    """Build a :class:`ModelClient` from the run context.
+
+    OpenAI-compatible self-hosted servers (vLLM, SGLang) require the LiteLLM
+    ``openai/<model>`` routing prefix; we add it here exactly once, stripping
+    any user-supplied prefix first so a double ``openai/openai/...`` never
+    reaches LiteLLM. Provider-hosted engines (``OPENAI``, ``COHERE``) leave
+    the model id untouched.
+    """
+    model_id = context.model_id
+    api_key: str | None
+    if context.engine_kind in _SELF_HOSTED_ENGINES:
+        if model_id.startswith("openai/"):
+            model_id = model_id[len("openai/") :]
+        model_id = f"openai/{model_id}"
+        api_key = context.api_key or "EMPTY"
+    else:
+        api_key = context.api_key or None
+    return ModelClient(
+        model=model_id,
+        api_key=api_key,
+        base_url=context.base_url or None,
+        timeout_s=timeout_s,
+    )
+
+
+def _build_prompt(source: str, source_lang: str, target_lang: str) -> str:
+    """Construct the translation prompt for one fixture row."""
+    return f"Translate from {source_lang} to {target_lang}:\n\n{source}\n\nTranslation:"
+
+
+# Metrics this plugin is expected to emit. Consumed by ``bench coverage``.
+EXPECTED_METRICS: tuple[str, ...] = (
+    "chrf_mean",
+    "chrf_p50",
+    "chrf_p95",
+    "ok_rate",
+    "n_samples",
+    "ttft_p50_ms",
+    "total_p50_ms",
+)
+
+
+class LLMMTPlugin:
+    """Plugin entry point. Registered via ``inferencebench.plugins`` entrypoint group."""
+
+    suite_id = "llm.mt"
+    version = "0.0.2"
+    description = (
+        "LLM machine-translation benchmarks (chrF / token-BLEU / exact-match on bundled fixtures)."
+    )
+
+    # ----------------------------------------------------------- benchmarks #
+    def list_benchmarks(self) -> list[BenchmarkSpec]:
+        bench_dir = self._benchmarks_dir()
+        specs: list[BenchmarkSpec] = []
+        if not bench_dir.exists():
+            return specs
+        for yml in sorted(bench_dir.glob("*.yaml")):
+            specs.append(self._load_yaml(yml))
+        return specs
+
+    def get_benchmark(self, benchmark_id: str) -> BenchmarkSpec:
+        for spec in self.list_benchmarks():
+            if spec.benchmark_id == benchmark_id:
+                return spec
+        msg = f"benchmark_id not found: {benchmark_id}"
+        raise KeyError(msg)
+
+    # ------------------------------------------------------------- validate #
+    def validate(self, spec: BenchmarkSpec, context: RunContext) -> list[str]:
+        warnings: list[str] = []
+        if not context.model_id:
+            warnings.append("model_id is empty")
+        if context.engine_kind in _SELF_HOSTED_ENGINES and not context.base_url:
+            warnings.append(
+                f"{context.engine_kind.value} needs base_url (e.g. http://localhost:8000/v1)"
+            )
+        if not self._dataset_path(spec).exists():
+            warnings.append(f"fixture not found: {spec.dataset.path}")
+        return warnings
+
+    # ------------------------------------------------------------------ run #
+    def run(self, spec: BenchmarkSpec, context: RunContext) -> Envelope:
+        """Execute the benchmark and return a SIGNED envelope."""
+        client = _build_client(context)
+        items = self._load_fixture(spec)
+        fixture_hash = _compute_fixture_hash(items)
+        scorer = SCORERS[spec.scoring]
+
+        samples, scores, telemetry = self._score_items(client, items, spec, scorer)
+
+        # Best-effort diagnostic dump — never blocks the run on I/O errors.
+        self._dump_samples(context, samples)
+
+        envelope = self._build_envelope(
+            spec,
+            context,
+            samples=samples,
+            scores=scores,
+            dataset_hash=fixture_hash,
+            energy=telemetry.summarise(samples),
+        )
+        signing_mode = context.extra.get("signing_mode", "dev")
+        dev_key_path = context.extra.get("dev_key_path")
+        if signing_mode == "dev":
+            if not dev_key_path:
+                msg = "dev signing requires context.extra['dev_key_path']"
+                raise ValueError(msg)
+            return sign_envelope(
+                envelope,
+                mode=SigningMode.DEV,
+                dev_key_path=Path(str(dev_key_path)),
+            )
+        return sign_envelope(envelope, mode=SigningMode.KEYLESS)
+
+    # -------------------------------------------------------- core scoring #
+    def _score_items(
+        self,
+        client: ModelClient,
+        items: list[dict[str, str]],
+        spec: BenchmarkSpec,
+        scorer: Callable[[str, str], float],
+    ) -> tuple[list[Sample], list[float], TelemetryWindow]:
+        """Iterate fixture items, call the model, score each hypothesis.
+
+        MT runs are per-sentence and order-independent — no driver machinery
+        is required. We still emit ``Sample`` objects so the envelope-building
+        path stays uniform with the perf plugin.
+        """
+        samples: list[Sample] = []
+        scores: list[float] = []
+        telemetry = TelemetryWindow()
+        with telemetry:
+            for idx, item in enumerate(items):
+                source = item["source"]
+                reference = item["reference"]
+                prompt = _build_prompt(source, spec.source_lang, spec.target_lang)
+                t_arrival = time.perf_counter() * 1000.0
+                try:
+                    result: CompletionResult = client.complete(
+                        prompt, stream=True, max_tokens=256
+                    )
+                except Exception as exc:
+                    samples.append(
+                        Sample(
+                            request_idx=idx,
+                            arrival_ms=t_arrival,
+                            start_ms=t_arrival,
+                            ttft_ms=float("nan"),
+                            total_ms=float("nan"),
+                            tpot_ms=float("nan"),
+                            tokens_in=0,
+                            tokens_out=0,
+                            cost_usd=0.0,
+                            finish_reason="error",
+                            ok=False,
+                            error=str(exc),
+                        )
+                    )
+                    continue
+
+                score = float(scorer(reference, result.text))
+                scores.append(score)
+
+                sample_extra: dict[str, str | int | float | bool] = {
+                    "domain": item.get("domain", ""),
+                    "score": score,
+                }
+                samples.append(
+                    Sample(
+                        request_idx=idx,
+                        arrival_ms=t_arrival,
+                        start_ms=t_arrival,
+                        ttft_ms=result.ttft_ms,
+                        total_ms=result.total_ms,
+                        tpot_ms=result.tpot_ms,
+                        tokens_in=result.tokens_in,
+                        tokens_out=result.tokens_out,
+                        cost_usd=result.cost_usd,
+                        finish_reason=result.finish_reason,
+                        ok=True,
+                        extra=sample_extra,
+                    )
+                )
+        return samples, scores, telemetry
+
+    # ------------------------------------------------------------ samples #
+    def _dump_samples(self, context: RunContext, samples: list[Sample]) -> None:
+        """Write per-request samples (incl. score) to ``<output_dir>/samples-<ts>.jsonl``.
+
+        Mirrors the llm-quality plugin's diagnostic dump — failures here
+        never block the run.
+        """
+        try:
+            out_dir = Path(context.output_dir)
+            out_dir.mkdir(parents=True, exist_ok=True)
+            ts = int(time.time())
+            path = out_dir / f"samples-{ts}.jsonl"
+            with path.open("w", encoding="utf-8") as fp:
+                for s in samples:
+                    score = s.extra.get("score") if s.extra else None
+                    score_part = (
+                        ',"score":' + _json_num(float(score))
+                        if isinstance(score, (int, float))
+                        else ""
+                    )
+                    fp.write(
+                        '{"request_idx":'
+                        + str(s.request_idx)
+                        + ',"ok":'
+                        + ("true" if s.ok else "false")
+                        + ',"ttft_ms":'
+                        + _json_num(s.ttft_ms)
+                        + ',"total_ms":'
+                        + _json_num(s.total_ms)
+                        + ',"tokens_in":'
+                        + str(s.tokens_in)
+                        + ',"tokens_out":'
+                        + str(s.tokens_out)
+                        + score_part
+                        + ',"finish_reason":"'
+                        + (s.finish_reason or "")
+                        + '"'
+                        + (',"error":' + _json_str(s.error) if s.error else "")
+                        + "}\n"
+                    )
+        except OSError:
+            pass  # diagnostics-only — never block the run
+
+    # ---------------------------------------------------------- file paths #
+    def _benchmarks_dir(self) -> Path:
+        return Path(__file__).parent / "benchmarks"
+
+    def _datasets_dir(self) -> Path:
+        return Path(__file__).parent / "datasets"
+
+    def _dataset_path(self, spec: BenchmarkSpec) -> Path:
+        raw = spec.dataset.path
+        if raw.startswith("fixtures://"):
+            return _fixtures_cache_root() / f"{raw[len('fixtures://') :]}.jsonl"
+        return self._datasets_dir() / raw
+
+    def _load_yaml(self, path: Path) -> BenchmarkSpec:
+        raw = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
+        return BenchmarkSpec.model_validate(raw)
+
+    def _load_fixture(self, spec: BenchmarkSpec) -> list[dict[str, str]]:
+        path = self._dataset_path(spec)
+        if not path.exists():
+            if spec.dataset.path.startswith("fixtures://"):
+                key = spec.dataset.path[len("fixtures://") :]
+                msg = f"fixture not cached: {path}. Run `bench fixtures fetch {key}` first."
+                raise FileNotFoundError(msg)
+            msg = f"fixture not found: {path}"
+            raise FileNotFoundError(msg)
+        items: list[dict[str, str]] = []
+        with path.open("r", encoding="utf-8") as fp:
+            for line in fp:
+                line = line.strip()
+                if not line:
+                    continue
+                obj = json.loads(line)
+                if not isinstance(obj, dict):
+                    continue
+                if "source" not in obj or "reference" not in obj:
+                    continue
+                items.append(
+                    {
+                        "source": str(obj["source"]),
+                        "reference": str(obj["reference"]),
+                        "domain": str(obj.get("domain", "")),
+                    }
+                )
+        if not items:
+            msg = f"fixture is empty: {path}"
+            raise ValueError(msg)
+        return items
+
+    # ---------------------------------------------------------- envelope #
+    def _build_envelope(
+        self,
+        spec: BenchmarkSpec,
+        context: RunContext,
+        *,
+        samples: list[Sample],
+        scores: list[float],
+        dataset_hash: str,
+        energy: EnergyReport | None = None,
+    ) -> Envelope:
+        hw = collect_hardware_fingerprint()
+        sw = collect_software_provenance()
+
+        metrics: dict[str, float | int | str | None] = {}
+
+        ok_samples = [s for s in samples if s.ok]
+        n_ok = len(ok_samples)
+        metrics["n_samples"] = float(len(samples))
+        metrics["n_ok"] = float(n_ok)
+        metrics["ok_rate"] = float(n_ok) / float(len(samples)) if samples else 0.0
+
+        # Headline scoring metric — keyed by the spec's scoring strategy so
+        # downstream ``bench diff`` knows the direction (all three MT scorers
+        # are higher-is-better).
+        if scores:
+            mean_score = sum(scores) / len(scores)
+            if spec.scoring == "exact_match":
+                metrics["exact_match_rate"] = mean_score
+            else:
+                prefix = spec.scoring  # "chrf" | "bleu_token"
+                # Keep the metric key short for BLEU (``bleu_mean``, not
+                # ``bleu_token_mean``) so diff's _HIGHER_IS_BETTER policy and
+                # the leaderboard column headers stay readable.
+                key_prefix = "bleu" if prefix == "bleu_token" else prefix
+                metrics[f"{key_prefix}_mean"] = mean_score
+                if len(scores) >= 2:
+                    pcts = Percentiles(scores, percentiles=(50.0, 95.0))
+                    metrics[f"{key_prefix}_p50"] = pcts.p50
+                    metrics[f"{key_prefix}_p95"] = pcts.p95
+                else:
+                    metrics[f"{key_prefix}_p50"] = mean_score
+                    metrics[f"{key_prefix}_p95"] = mean_score
+
+        # Latency aggregates — "quality at what cost" comparisons.
+        ttft_vals = [s.ttft_ms for s in ok_samples if math.isfinite(s.ttft_ms)]
+        total_vals = [s.total_ms for s in ok_samples if math.isfinite(s.total_ms)]
+        if ttft_vals:
+            metrics["ttft_p50_ms"] = Percentiles(ttft_vals).p50
+        if total_vals:
+            metrics["total_p50_ms"] = Percentiles(total_vals).p50
+
+        tokens_out_total = sum(s.tokens_out for s in ok_samples)
+        if tokens_out_total:
+            metrics["tokens_out_total"] = float(tokens_out_total)
+
+        # Energy / power summary from telemetry (None on plugins that haven't
+        # threaded a TelemetryWindow through yet). Mirrors llm-inference.
+        if energy is not None:
+            if energy.gpu_power_avg_w > 0:
+                metrics["power_avg_w"] = energy.gpu_power_avg_w
+                metrics["power_peak_w"] = energy.gpu_power_peak_w
+            if energy.total_energy_joules > 0:
+                metrics["energy_joules_total"] = energy.total_energy_joules
+                if energy.joules_per_token == energy.joules_per_token:  # not NaN
+                    metrics["joules_per_token"] = energy.joules_per_token
+
+        # Cost: only emit when the provider actually reported it. Self-hosted
+        # vLLM / SGLang never do; the perf plugin's pricing-registry fallback
+        # is intentionally NOT mirrored here — MT runs are cheap enough
+        # that a missing-cost row is more honest than an estimated one.
+        cost_total = sum(s.cost_usd for s in ok_samples)
+        if tokens_out_total and cost_total > 0:
+            metrics["cost_usd_per_million_tokens"] = (cost_total / tokens_out_total) * 1e6
+            metrics["cost_source"] = "provider"
+
+        builder = EnvelopeBuilder(
+            suite_id=spec.benchmark_id,
+            suite_version=spec.suite_version,
+            model=ModelConfig(
+                id=context.model_id,
+                revision=context.model_revision,
+                provider=context.engine_kind.value,
+                endpoint_hash="0" * 64,
+            ),
+            engine=EngineConfig(
+                name=context.engine_kind.value,
+                version=context.engine_version or "unknown",
+                config_hash="0" * 64,
+            ),
+            hardware_fingerprint=hw,
+            software_provenance=sw,
+            dataset=EnvDatasetSpec(id=spec.dataset.id, hash=dataset_hash),
+            seed=0,
+            quantization=(
+                Quantization(format=context.quantization_format)
+                if context.quantization_format
+                else None
+            ),
+            metrics=metrics,
+            slo_template=spec.slo_template,
+        )
+        return builder.build()

inferencebench_mt/schemas.py

@@ -0,0 +1,91 @@
+"""Pydantic schemas for llm-mt benchmark specs + run context."""
+
+from __future__ import annotations
+
+from enum import StrEnum
+from pathlib import Path
+from typing import Annotated, Literal
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class EngineKind(StrEnum):
+    """Engines this plugin can drive.
+
+    Machine translation is dominated by per-prompt API calls, so the four
+    most useful endpoints are self-hosted OpenAI-compatible servers (vLLM,
+    SGLang), provider-hosted OpenAI, and Cohere — whose Aya / Command
+    models are popular MT picks.
+    """
+
+    VLLM = "vllm"
+    SGLANG = "sglang"
+    OPENAI = "openai"
+    COHERE = "cohere"
+
+
+class DatasetConfig(BaseModel):
+    """Dataset under evaluation.
+
+    For the MT plugin the dataset is a small bundled JSONL fixture with one
+    ``{"source", "reference", "domain"}`` object per line.
+    """
+
+    model_config = ConfigDict(extra="forbid")
+    id: Annotated[str, Field(min_length=1)]
+    path: Annotated[
+        str,
+        Field(
+            min_length=1,
+            description=("Path to the fixture JSONL relative to the plugin's datasets/ directory."),
+        ),
+    ]
+
+
+class WarmupConfig(BaseModel):
+    """Warmup parameters.
+
+    MT scoring is per-sentence and order-independent, so the default is
+    zero discarded runs. Surfaced as a knob so future revisions can warm
+    up server-side weights if needed.
+    """
+
+    model_config = ConfigDict(extra="forbid")
+    discard_runs: Annotated[int, Field(ge=0)] = 0
+
+
+class BenchmarkSpec(BaseModel):
+    """One MT benchmark — fixture + scoring strategy + language pair + metadata."""
+
+    model_config = ConfigDict(extra="forbid")
+    benchmark_id: Annotated[str, Field(min_length=1)]
+    suite_version: Annotated[str, Field(pattern=r"^\d+\.\d+\.\d+(-[\w.]+)?$")]
+    description: str = ""
+    modality: Literal["llm"] = "llm"
+    kind: Literal["translation"] = "translation"
+    dataset: DatasetConfig
+    slo_template: str = "llm.mt.standard"
+    warmup: WarmupConfig = Field(default_factory=WarmupConfig)
+    scoring: Literal["chrf", "bleu_token", "exact_match"] = "chrf"
+    source_lang: Annotated[str, Field(min_length=2, max_length=8)]
+    target_lang: Annotated[str, Field(min_length=2, max_length=8)]
+
+
+class RunContext(BaseModel):
+    """Per-invocation context (where to send requests, where to write results).
+
+    Mirrors the llm-quality plugin shape so cross-plugin tooling can reuse
+    the same context object.
+    """
+
+    model_config = ConfigDict(extra="forbid", arbitrary_types_allowed=True)
+    model_id: Annotated[str, Field(min_length=1)]
+    model_revision: Annotated[str, Field(min_length=7, max_length=40)] = "unknown00"
+    engine_kind: EngineKind
+    engine_version: str = ""
+    base_url: str = ""
+    api_key: str = ""
+    quantization_format: str = ""
+    hardware_class: str = ""
+    output_dir: Path
+    extra: dict[str, str | int | float | bool] = Field(default_factory=dict)

inferencebench_mt/scoring.py

@@ -0,0 +1,131 @@
+"""Deterministic scoring strategies for the llm-mt plugin.
+
+Three pure functions, each ``(reference, hypothesis) -> float`` in ``[0.0, 1.0]``:
+
+- :func:`chrf` — character n-gram F-score (the standard chrF metric).
+- :func:`bleu_token` — corpus-free token BLEU with brevity penalty.
+- :func:`exact_match` — strict strip + lowercase equality (mirrors llm-quality).
+
+All three are higher-is-better — translation accuracy in :math:`[0, 1]`. No
+external dependencies. Whitespace is normalised by collapsing runs before
+character-n-gram extraction so chrF behaves the same on ``"hello  world"``
+and ``"hello world"``.
+"""
+
+from __future__ import annotations
+
+import math
+from collections import Counter
+from collections.abc import Callable
+
+
+def _char_ngrams(text: str, n: int) -> Counter[str]:
+    """Return the multiset of character n-grams of length ``n`` in ``text``.
+
+    Whitespace is collapsed (any run of whitespace becomes a single space) so
+    formatting noise does not perturb the score. ``n`` must be at least 1.
+    """
+    normalised = " ".join(text.split())
+    if len(normalised) < n:
+        return Counter()
+    return Counter(normalised[i : i + n] for i in range(len(normalised) - n + 1))
+
+
+def chrf(reference: str, hypothesis: str, n: int = 6, beta: float = 2.0) -> float:
+    """Character n-gram F-score (chrF).
+
+    Collects character n-grams up to length ``n`` in both ``reference`` and
+    ``hypothesis``, computes a single precision / recall / F_beta over the
+    union of all n-gram orders, and returns the result in ``[0, 1]``. This
+    is the simplest, hand-computable variant of the metric: order-uniform
+    weighting, no separate word-vs-char split.
+
+    Returns 1.0 when reference and hypothesis are identical (after
+    whitespace normalisation) and 0.0 when no n-gram of any order overlaps.
+    An empty reference and hypothesis match exactly (returns 1.0); only one
+    side empty returns 0.0.
+    """
+    if n < 1:
+        msg = "chrf requires n >= 1"
+        raise ValueError(msg)
+    ref_norm = " ".join(reference.split())
+    hyp_norm = " ".join(hypothesis.split())
+    if not ref_norm and not hyp_norm:
+        return 1.0
+    if not ref_norm or not hyp_norm:
+        return 0.0
+
+    total_match = 0
+    total_hyp = 0
+    total_ref = 0
+    for order in range(1, n + 1):
+        ref_ngrams = _char_ngrams(reference, order)
+        hyp_ngrams = _char_ngrams(hypothesis, order)
+        if not ref_ngrams or not hyp_ngrams:
+            continue
+        overlap = sum((ref_ngrams & hyp_ngrams).values())
+        total_match += overlap
+        total_hyp += sum(hyp_ngrams.values())
+        total_ref += sum(ref_ngrams.values())
+
+    if total_hyp == 0 or total_ref == 0 or total_match == 0:
+        return 0.0
+    precision = total_match / total_hyp
+    recall = total_match / total_ref
+    beta_sq = beta * beta
+    denom = beta_sq * precision + recall
+    if denom == 0:
+        return 0.0
+    return (1.0 + beta_sq) * precision * recall / denom
+
+
+def bleu_token(reference: str, hypothesis: str, max_n: int = 4) -> float:
+    """Simple corpus-free BLEU over whitespace-split tokens.
+
+    Computes modified n-gram precisions for ``n`` in ``1..max_n``, takes
+    their geometric mean, and multiplies by the standard brevity penalty
+    ``exp(min(0, 1 - r/c))``. Returns 0.0 when any n-gram order has zero
+    precision (the canonical BLEU behaviour — no smoothing here, since the
+    metric is informational only for the skeleton). Result is in ``[0, 1]``.
+    """
+    if max_n < 1:
+        msg = "bleu_token requires max_n >= 1"
+        raise ValueError(msg)
+    hyp_tokens = hypothesis.split()
+    ref_tokens = reference.split()
+    if not hyp_tokens or not ref_tokens:
+        return 0.0
+
+    precisions: list[float] = []
+    for n in range(1, max_n + 1):
+        if len(hyp_tokens) < n or len(ref_tokens) < n:
+            return 0.0
+        hyp_ngrams = Counter(tuple(hyp_tokens[i : i + n]) for i in range(len(hyp_tokens) - n + 1))
+        ref_ngrams = Counter(tuple(ref_tokens[i : i + n]) for i in range(len(ref_tokens) - n + 1))
+        # Modified count: cap each n-gram by its reference count.
+        clipped = sum(min(c, ref_ngrams[g]) for g, c in hyp_ngrams.items())
+        total = sum(hyp_ngrams.values())
+        if total == 0 or clipped == 0:
+            return 0.0
+        precisions.append(clipped / total)
+
+    # Geometric mean of precisions.
+    log_sum = sum(math.log(p) for p in precisions)
+    geo_mean = math.exp(log_sum / len(precisions))
+
+    c = len(hyp_tokens)
+    r = len(ref_tokens)
+    brevity = 1.0 if c > r else math.exp(1.0 - r / c)
+    return brevity * geo_mean
+
+
+def exact_match(reference: str, hypothesis: str) -> float:
+    """Return 1.0 iff ``hypothesis`` equals ``reference`` after strip + lowercase."""
+    return 1.0 if hypothesis.strip().lower() == reference.strip().lower() else 0.0
+
+
+SCORERS: dict[str, Callable[[str, str], float]] = {
+    "chrf": chrf,
+    "bleu_token": bleu_token,
+    "exact_match": exact_match,
+}

inferencebench_mt-0.0.2.dist-info/METADATA

@@ -0,0 +1,39 @@
+Metadata-Version: 2.4
+Name: inferencebench-mt
+Version: 0.0.2
+Summary: Machine-translation plugin for InferenceBench Suite (chrF/BLEU/exact-match on bundled fixtures)
+Project-URL: Homepage, https://github.com/yobitelcomm/bench
+Author-email: Yobitel Communications <bench@yobitel.com>
+License: Apache-2.0
+Keywords: ai,benchmark,bleu,chrf,llm,ml,translation
+Classifier: Development Status :: 2 - Pre-Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.12
+Requires-Dist: inferencebench-envelope
+Requires-Dist: inferencebench-harness
+Requires-Dist: pydantic~=2.9
+Requires-Dist: pyyaml~=6.0
+Description-Content-Type: text/markdown
+
+# inferencebench-mt
+
+Machine-translation plugin for the InferenceBench Suite.
+
+Scores model translations against bundled reference fixtures using chrF (character
+n-gram F-score), token-level BLEU, or exact match. Mirrors the contract of the
+other plugins (`list_benchmarks` / `get_benchmark` / `validate` / `run`) and
+emits the canonical signed envelope.
+
+Two bundled benchmarks ship out of the box:
+
+- `llm.mt.flores-200-mini-en-fr` — FLORES-200-style English to French, chrF.
+- `llm.mt.flores-200-mini-en-de` — FLORES-200-style English to German, chrF.
+
+The fixtures are tiny (eight rows each, mixed across greeting / news / technical
+/ conversational domains) — intended for skeleton verification, not headline
+numbers.

inferencebench_mt-0.0.2.dist-info/RECORD

@@ -0,0 +1,17 @@
+inferencebench_mt/__init__.py,sha256=bAb5E7Q5wiomAtN_QWcYEQxj7bL_DY4HVidDUHr4WVY,312
+inferencebench_mt/plugin.py,sha256=5S-Hdzymsiivc6YDR5s-ogaYj0esvtKU9LJ15TkIZ08,18607
+inferencebench_mt/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+inferencebench_mt/schemas.py,sha256=tpJ206CggNjdX2VEJZyfOEwNEhoPpFdBlDeNsU6M8I0,2943
+inferencebench_mt/scoring.py,sha256=_z80lCIUrJ3Uy___xG9wiFzP_dW3ejEVFylKrm573bA,4998
+inferencebench_mt/benchmarks/flores-200-mini-en-de.yaml,sha256=kUxm5fJOJ_a3QUUAiUmNGxZuSXW7-Sa0jir7a53vdEA,339
+inferencebench_mt/benchmarks/flores-200-mini-en-es.yaml,sha256=lkCSbufCcn9lvPoKxQ1HJrRrOLHdWWiu4pScxyrLOts,340
+inferencebench_mt/benchmarks/flores-200-mini-en-fr.yaml,sha256=BEnt14wkrwfBV1-ZrD9vli39x4zV6E4CQf_eOebl7jg,339
+inferencebench_mt/benchmarks/flores-200-mini-en-ja.yaml,sha256=oCGXI2NuZI658Z83Azo4CqCHVjguXo5pl9p2zWCRkp8,341
+inferencebench_mt/datasets/flores-mini-en-de.jsonl,sha256=ZweW2R0Ku98t8NBrFJP-eiVIgORVZ6TaJMLjtNpIjN8,1194
+inferencebench_mt/datasets/flores-mini-en-es.jsonl,sha256=z9i5MYZGNeZSnSEeqjHPvT45kKZ5yTJ8QBto3EzdqwY,1194
+inferencebench_mt/datasets/flores-mini-en-fr.jsonl,sha256=tmp6wfNyvAGgMJJsec7U4pP5TXhKp0j8bBFLn3mq8LY,1217
+inferencebench_mt/datasets/flores-mini-en-ja.jsonl,sha256=s9mOfQN2WIwf-o-bbRaLl2jM9-QtwNwynmIc0xy3Cl4,1263
+inferencebench_mt-0.0.2.dist-info/METADATA,sha256=jRL7nbrmomSuHuuE5xMWelnJ1lGbFK_VSb5nkU3_6B0,1657
+inferencebench_mt-0.0.2.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
+inferencebench_mt-0.0.2.dist-info/entry_points.txt,sha256=vsbyHlBHcuBrS1IEEg2YJRNR5cMOQs2IX8PoRO3pALw,71
+inferencebench_mt-0.0.2.dist-info/RECORD,,

inferencebench_mt-0.0.2.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.29.0
+Root-Is-Purelib: true
+Tag: py3-none-any

inferencebench_mt-0.0.2.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[inferencebench.plugins]
+llm.mt = inferencebench_mt.plugin:LLMMTPlugin