mt-eval-harness 0.1.0__py3-none-any.whl

mt_eval_harness/__init__.py

@@ -0,0 +1,99 @@
+"""
+mt-eval — A configurable, language-agnostic evaluation
+harness for machine translation pipelines.
+
+Two-component architecture:
+    Run Harness:  Executes translation experiments with configurable
+                  model, prompt, tools, batching, and caching.
+    Test Harness: Analyzes run results offline with deterministic
+                  linguistic metrics (chrF++, BLEU, COMET, exact match),
+                  bootstrap confidence intervals, significance testing,
+                  and extensible plugin metrics.
+
+Any translation method can plug into the harness for regularized,
+apple-to-apple comparison. See GUIDE.md for full documentation.
+"""
+
+__version__ = "0.1.0"
+
+# Public API — importable from the top-level package
+from mt_eval_harness.config import (
+    RunConfig,
+    load_registry,
+    resolve_dataset,
+    load_method_card,
+    validate_method_card,
+)
+from mt_eval_harness.exporter import ExportConfig, export_plugin
+from mt_eval_harness.corpus_loader import load_corpus
+from mt_eval_harness.champollion_config import (
+    ChampollionRunConfig,
+    ChampollionPromptConfig,  # Back-compat alias for ChampollionRunConfig
+    load_champollion_config,
+    build_champollion_system_prompt,
+)
+from mt_eval_harness.plugins.metrics import MetricPlugin
+from mt_eval_harness.plugins.prompts import PromptProvider
+from mt_eval_harness.plugins.champollion_prompts import ChampollionPromptProvider
+from mt_eval_harness.plugins.hooks import PostTranslationHook
+from mt_eval_harness.plugins.tools import ToolProvider
+from mt_eval_harness.significance import (
+    SignificanceResult,
+    paired_bootstrap,
+    paired_approximate_randomization,
+)
+from mt_eval_harness.confidence import ConfidenceInterval, bootstrap_ci, compute_all_cis
+from mt_eval_harness.metrics_comet import (
+    HAS_COMET,
+    COMETResult,
+    compute_comet,
+    corpus_comet,
+    DEFAULT_COMET_MODEL,
+    resolve_comet_model,
+)
+from mt_eval_harness.metrics_metricx import (
+    HAS_METRICX,
+    MetricXResult,
+    compute_metricx,
+    corpus_metricx,
+    DEFAULT_METRICX_MODEL,
+    METRICX_LOWER_IS_BETTER,
+)
+
+__all__ = [
+    "RunConfig",
+    "ExportConfig",
+    "export_plugin",
+    "load_corpus",
+    "ChampollionRunConfig",
+    "ChampollionPromptConfig",  # Back-compat alias
+    "load_champollion_config",
+    "build_champollion_system_prompt",
+    "ChampollionPromptProvider",
+    "MetricPlugin",
+    "PromptProvider",
+    "PostTranslationHook",
+    "ToolProvider",
+    "SignificanceResult",
+    "paired_bootstrap",
+    "paired_approximate_randomization",
+    "ConfidenceInterval",
+    "bootstrap_ci",
+    "compute_all_cis",
+    "HAS_COMET",
+    "COMETResult",
+    "compute_comet",
+    "corpus_comet",
+    "DEFAULT_COMET_MODEL",
+    "resolve_comet_model",
+    "HAS_METRICX",
+    "MetricXResult",
+    "compute_metricx",
+    "corpus_metricx",
+    "DEFAULT_METRICX_MODEL",
+    "METRICX_LOWER_IS_BETTER",
+    "load_registry",
+    "resolve_dataset",
+    "load_method_card",
+    "validate_method_card",
+]

mt_eval_harness/__main__.py

@@ -0,0 +1,6 @@
+"""Allow running the harness as a module: python -m mt_eval_harness"""
+
+from mt_eval_harness.cli import main
+
+if __name__ == "__main__":
+    main()

mt_eval_harness/_dashboard_charts.py

@@ -0,0 +1,148 @@
+"""
+Chart.js chart-builder functions, injected into the dashboard JS.
+Separated from _dashboard_js.py to keep files manageable.
+"""
+
+CHART_JS = r"""
+// ── Shared Global State ─────────────────────────────────
+// These are hoisted to global scope so both chart builders and the
+// core IIFE can share R (reports), activeIdx, and helpers.
+const R = window.__REPORTS || [];
+let activeIdx = 0;
+const $ = id => document.getElementById(id);
+function runLabel(r, i) {
+    const c = r.config || {};
+    return (c.run_name || c.model || r.run_id || 'Run '+(i+1)).substring(0, 35);
+}
+
+// ── Chart.js Builders ───────────────────────────────────
+const COLORS = ['#6366f1','#34d399','#f59e0b','#f87171','#a78bfa','#22d3ee','#fb923c','#e879f9'];
+let barChart = null, histChart = null, radarChart = null;
+
+function destroyCharts() {
+    [barChart, histChart, radarChart].forEach(c => { if (c) c.destroy(); });
+    barChart = histChart = radarChart = null;
+}
+
+// Grouped bar chart: Exact Match % and chrF++ per run
+function buildBarChart() {
+    const sec = $('barchart-section');
+    if (R.length < 2) { sec.style.display = 'none'; return; }
+    sec.style.display = '';
+    const canvas = $('bar-canvas');
+    if (barChart) barChart.destroy();
+    const labels = R.map((r, i) => runLabel(r, i).substring(0, 22));
+    const isDark = document.documentElement.getAttribute('data-theme') !== 'light';
+    const gridColor = isDark ? 'rgba(255,255,255,0.06)' : 'rgba(0,0,0,0.06)';
+    const textColor = isDark ? 'rgba(255,255,255,0.5)' : 'rgba(0,0,0,0.5)';
+    barChart = new Chart(canvas, {
+        type: 'bar',
+        data: {
+            labels,
+            datasets: [
+                {
+                    label: 'Exact Match %',
+                    data: R.map(r => ((r.overall || {}).exact_match_rate || 0) * 100),
+                    backgroundColor: COLORS[0] + 'cc',
+                    borderRadius: 4,
+                    barPercentage: 0.7,
+                },
+                {
+                    label: 'chrF++',
+                    data: R.map(r => (r.overall || {}).corpus_chrf || 0),
+                    backgroundColor: COLORS[1] + 'cc',
+                    borderRadius: 4,
+                    barPercentage: 0.7,
+                }
+            ]
+        },
+        options: {
+            responsive: true,
+            maintainAspectRatio: true,
+            plugins: {
+                legend: { labels: { color: textColor, font: { size: 11 } } },
+                tooltip: { mode: 'index', intersect: false }
+            },
+            scales: {
+                x: { ticks: { color: textColor, font: { size: 10 } }, grid: { color: gridColor } },
+                y: { beginAtZero: true, max: 100, ticks: { color: textColor }, grid: { color: gridColor } }
+            }
+        }
+    });
+}
+
+// chrF++ distribution histogram for active run
+function buildHistogram() {
+    const entries = R[activeIdx].entries || [];
+    const sec = $('histogram-section');
+    if (!entries.length) { sec.style.display = 'none'; return; }
+    sec.style.display = '';
+    const canvas = $('hist-canvas');
+    if (histChart) histChart.destroy();
+    const bins = new Array(10).fill(0);
+    entries.forEach(e => { const b = Math.min(9, Math.floor((e.chrf_score || 0) / 10)); bins[b]++; });
+    const isDark = document.documentElement.getAttribute('data-theme') !== 'light';
+    const gridColor = isDark ? 'rgba(255,255,255,0.06)' : 'rgba(0,0,0,0.06)';
+    const textColor = isDark ? 'rgba(255,255,255,0.5)' : 'rgba(0,0,0,0.5)';
+    histChart = new Chart(canvas, {
+        type: 'bar',
+        data: {
+            labels: ['0-10','10-20','20-30','30-40','40-50','50-60','60-70','70-80','80-90','90-100'],
+            datasets: [{
+                label: 'Entries',
+                data: bins,
+                backgroundColor: COLORS[activeIdx % COLORS.length] + 'aa',
+                borderRadius: 3,
+            }]
+        },
+        options: {
+            responsive: true,
+            maintainAspectRatio: true,
+            plugins: {
+                legend: { display: false },
+                tooltip: { callbacks: { title: ctx => 'chrF++ ' + ctx[0].label, label: ctx => ctx.raw + ' entries' } }
+            },
+            scales: {
+                x: { title: { display: true, text: 'chrF++ Score Range', color: textColor }, ticks: { color: textColor }, grid: { display: false } },
+                y: { beginAtZero: true, ticks: { color: textColor, stepSize: 1 }, grid: { color: gridColor } }
+            }
+        }
+    });
+}
+
+// Radar chart: multi-metric comparison across runs
+function buildRadar() {
+    const sec = $('radar-section');
+    if (R.length < 2) { sec.style.display = 'none'; return; }
+    sec.style.display = '';
+    const canvas = $('radar-canvas');
+    if (radarChart) radarChart.destroy();
+    const isDark = document.documentElement.getAttribute('data-theme') !== 'light';
+    const gridColor = isDark ? 'rgba(255,255,255,0.08)' : 'rgba(0,0,0,0.08)';
+    const textColor = isDark ? 'rgba(255,255,255,0.5)' : 'rgba(0,0,0,0.5)';
+    const metrics = ['exact_match_rate', 'corpus_chrf', 'corpus_bleu'];
+    const labels = ['Exact Match %', 'chrF++', 'BLEU'];
+    const datasets = R.map((r, i) => ({
+        label: runLabel(r, i).substring(0, 20),
+        data: metrics.map(k => {
+            let v = (r.overall || {})[k] || 0;
+            if (k === 'exact_match_rate') v *= 100;
+            return v;
+        }),
+        borderColor: COLORS[i % COLORS.length],
+        backgroundColor: COLORS[i % COLORS.length] + '22',
+        pointBackgroundColor: COLORS[i % COLORS.length],
+        pointRadius: 3,
+    }));
+    radarChart = new Chart(canvas, {
+        type: 'radar',
+        data: { labels, datasets },
+        options: {
+            responsive: true,
+            maintainAspectRatio: true,
+            plugins: { legend: { labels: { color: textColor, font: { size: 10 } } } },
+            scales: { r: { beginAtZero: true, max: 100, ticks: { color: textColor, backdropColor: 'transparent' }, grid: { color: gridColor }, pointLabels: { color: textColor } } }
+        }
+    });
+}
+"""