synthbench-eval 0.4.0__py3-none-any.whl

synthbench/cli.py

@@ -0,0 +1,2929 @@
+"""SynthBench CLI — synthbench run, synthbench report."""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import sys
+from pathlib import Path
+
+import click
+
+from synthbench import __version__
+
+# sb-ymux: re-exported so tests that monkeypatch SYNTHBENCH_API_URL + read the
+# fallback constant don't need a second import path.
+from synthbench.submission import DEFAULT_API_URL
+
+# Model aliases for convenience.
+#
+# These must be REAL, currently-served model IDs — a chimeric or retired ID
+# either hard-errors at request time or silently benchmarks a different
+# model than the leaderboard row claims.
+MODEL_ALIASES = {
+    "haiku": "claude-haiku-4-5-20251001",
+    # Real Sonnet 4.5 snapshot (the previous "-20241022" suffix belonged to
+    # a different, retired model — the combination never existed).
+    "sonnet": "claude-sonnet-4-5-20250929",
+    # Real Opus 4 ID has no "-0-" infix.
+    "opus": "claude-opus-4-20250514",
+    "gpt-4o": "gpt-4o",
+    "gpt-4o-mini": "gpt-4o-mini",
+    # GA Gemini 2.5 IDs (the "-preview-*" snapshots are retired).
+    "gemini-flash": "gemini-2.5-flash",
+    "gemini-flash-lite": "gemini-2.5-flash-lite",
+    "gemini-pro": "gemini-2.5-pro",
+}
+
+# Providers whose constructor accepts a model identifier / temperature.
+_MODEL_PROVIDERS = frozenset(
+    {"raw-anthropic", "raw-openai", "raw-gemini", "openrouter", "ollama", "synthpanel"}
+)
+
+# Providers that can actually honour a reasoning-effort level. Everything
+# else must hard-error on --effort (mirroring the --temperature strictness
+# below): a leaderboard row tagged "high effort" for a provider that never
+# threaded the knob would be untrue metadata.
+_EFFORT_PROVIDERS = frozenset(
+    {"raw-anthropic", "raw-openai", "raw-gemini", "openrouter"}
+)
+
+EFFORT_CHOICES = ("low", "medium", "high")
+
+
+def _provider_kwargs(
+    provider_name,
+    *,
+    model=None,
+    url=None,
+    temperature=None,
+    prompt_template=None,
+    effort=None,
+    elicitation=None,
+):
+    """Build constructor kwargs appropriate for *provider_name*.
+
+    Provider constructors no longer swallow unknown kwargs, so the CLI must
+    only pass options each provider actually supports — and must hard-error
+    (instead of silently ignoring) when the user passes an option the
+    selected provider cannot honour. A silently dropped --temperature is
+    exactly how leaderboard rows ended up labeled with temperatures that
+    never applied.
+    """
+    kwargs = {}
+    if provider_name in _MODEL_PROVIDERS and model is not None:
+        kwargs["model"] = model
+    if url is not None:
+        if provider_name == "http":
+            kwargs["url"] = url
+        elif provider_name == "ollama":
+            kwargs["base_url"] = url
+        else:
+            raise click.UsageError(
+                f"--url is not supported by provider '{provider_name}' "
+                "(only http and ollama)."
+            )
+    if temperature is not None:
+        if provider_name not in _MODEL_PROVIDERS:
+            raise click.UsageError(
+                f"--temperature is not supported by provider '{provider_name}'."
+            )
+        kwargs["temperature"] = temperature
+    if prompt_template is not None:
+        if provider_name != "synthpanel":
+            raise click.UsageError(
+                "--prompt-template is only supported by the synthpanel provider."
+            )
+        kwargs["prompt_template"] = prompt_template
+    if elicitation is not None:
+        if provider_name != "synthpanel":
+            raise click.UsageError(
+                "--elicitation is only supported by the synthpanel provider."
+            )
+        kwargs["elicitation"] = elicitation
+    if effort is not None:
+        if provider_name == "synthpanel":
+            raise click.UsageError(
+                "--effort is not supported by provider 'synthpanel': the "
+                "synthpanel CLI does not yet expose a reasoning-effort knob, "
+                "so the run could not actually honour it. Once synthpanel "
+                "grows one, wire it through here rather than tagging runs "
+                "with metadata that never applied."
+            )
+        if provider_name not in _EFFORT_PROVIDERS:
+            raise click.UsageError(
+                f"--effort is not supported by provider '{provider_name}' "
+                "(no reasoning control to thread it to; supported: "
+                f"{', '.join(sorted(_EFFORT_PROVIDERS))})."
+            )
+        kwargs["effort"] = effort
+    return kwargs
+
+
+@click.group()
+@click.version_option(__version__)
+def main():
+    """SynthBench — benchmark harness for synthetic survey respondents."""
+    pass
+
+
+# Vendor adapter-driven submission flow (refs #256). Lives in its own module
+# because it pulls in adapter discovery + artifact rendering that doesn't
+# belong inline with the run/report commands above.
+from synthbench.submit_adapter import submit_adapter as _submit_adapter_cmd  # noqa: E402
+
+main.add_command(_submit_adapter_cmd)
+
+
+@main.command()
+@click.option(
+    "--provider",
+    "-p",
+    required=True,
+    help="Provider name (raw-anthropic, raw-openai, raw-gemini, openrouter, ollama, synthpanel, http).",
+)
+@click.option(
+    "--model",
+    "-m",
+    default="haiku",
+    help="Model name or alias (haiku, sonnet, gpt-4o-mini, etc.).",
+)
+@click.option(
+    "--dataset",
+    "-d",
+    default="opinionsqa",
+    help="Dataset to benchmark against.",
+)
+@click.option(
+    "--n",
+    "-n",
+    type=int,
+    default=None,
+    help="Number of questions to evaluate (default: all).",
+)
+@click.option(
+    "--samples",
+    "-s",
+    type=int,
+    default=30,
+    help="Samples per question for distribution estimation.",
+)
+@click.option(
+    "--concurrency",
+    "-c",
+    type=int,
+    default=10,
+    help="Max concurrent API requests.",
+)
+@click.option(
+    "--output",
+    "-o",
+    type=click.Path(),
+    default="results",
+    help="Output directory for score cards.",
+)
+@click.option(
+    "--data-dir",
+    type=click.Path(),
+    default=None,
+    help="Custom data directory for datasets.",
+)
+@click.option(
+    "--url",
+    default=None,
+    help="Endpoint URL (required for --provider http).",
+)
+@click.option(
+    "--json-only", is_flag=True, help="Output JSON to stdout instead of files."
+)
+@click.option(
+    "--suite",
+    type=click.Choice(["smoke", "core", "full"]),
+    default=None,
+    help="Use a pinned question set (smoke=28, core=200, full=all).",
+)
+@click.option(
+    "--topic",
+    type=click.Choice(["political", "consumer", "neutral"]),
+    default=None,
+    help="Filter to a topic subset (political, consumer, neutral).",
+)
+@click.option(
+    "--baselines-dir",
+    type=click.Path(exists=True),
+    default=None,
+    help="Directory with baseline results for 'vs Baselines' section.",
+)
+@click.option(
+    "--demographics",
+    default=None,
+    help="Comma-separated demographic attributes to evaluate (e.g., AGE,POLIDEOLOGY).",
+)
+@click.option(
+    "--full-evaluation",
+    is_flag=True,
+    help="Run all demographic attributes supported by the dataset.",
+)
+@click.option(
+    "--country",
+    default=None,
+    help="Country for GlobalOpinionQA ground truth (e.g., France, Japan).",
+)
+@click.option(
+    "--temperature",
+    type=float,
+    default=None,
+    help="Sampling temperature to pass to the provider (e.g., 0.7, 1.0).",
+)
+@click.option(
+    "--effort",
+    type=click.Choice(list(EFFORT_CHOICES)),
+    default=None,
+    help="Reasoning effort level for reasoning-capable models (low, medium, "
+    "high). Absent = provider default, and the run is stamped as "
+    "effort-absent. Errors on providers with no reasoning control.",
+)
+@click.option(
+    "--prompt-template",
+    type=click.Path(exists=True),
+    default=None,
+    help="Path to a custom persona prompt template file (synthpanel only).",
+)
+@click.option(
+    "--elicitation",
+    type=click.Choice(["natural", "structured"]),
+    default=None,
+    help=(
+        "Elicitation mode (synthpanel only). 'natural' (default) parses "
+        "prose responses; 'structured' forces a schema-constrained tool "
+        "call (enum of the option strings) so no prose parsing occurs. "
+        "Structured runs are stamped as a template variant "
+        "(tpl=structured) — a distinct leaderboard identity."
+    ),
+)
+# sb-ymux: `synthbench run --submit` wiring. Keeping all submission flags on
+# the `run` subcommand (not a new subcommand) means a user's first try at
+# "benchmark → leaderboard" is literally one invocation. --wait folds the
+# current "run, copy path, synthbench submit" two-step into a single process
+# whose exit code reflects the validation outcome — suitable for CI gates.
+@click.option(
+    "--submit",
+    "submit_after",
+    is_flag=True,
+    help="After the benchmark saves, upload the JSON to the SynthBench leaderboard.",
+)
+@click.option(
+    "--wait",
+    is_flag=True,
+    help="With --submit: poll submission status until published/rejected. Exit "
+    "code 0 on published, 1 on rejected/error, 2 on timeout.",
+)
+@click.option(
+    "--api-key",
+    "submit_api_key",
+    default=None,
+    help="API key for --submit (defaults to SYNTHBENCH_API_KEY env var).",
+)
+@click.option(
+    "--api-url",
+    "submit_api_url",
+    default=None,
+    help="Submission endpoint base for --submit (defaults to SYNTHBENCH_API_URL env).",
+)
+@click.option(
+    "--submit-message",
+    default=None,
+    help="Optional note recorded on the submission (e.g. 'first run with new prompt template').",
+)
+@click.option(
+    "--submit-timeout",
+    type=int,
+    default=60,
+    show_default=True,
+    help="HTTP timeout (seconds) for the /submit request.",
+)
+@click.option(
+    "--poll-interval",
+    type=float,
+    default=10.0,
+    show_default=True,
+    help="With --wait: seconds between status polls.",
+)
+@click.option(
+    "--poll-timeout",
+    type=float,
+    default=900.0,
+    show_default=True,
+    help="With --wait: give up after this many seconds and exit 2.",
+)
+def run(
+    provider,
+    model,
+    dataset,
+    n,
+    samples,
+    concurrency,
+    output,
+    data_dir,
+    url,
+    json_only,
+    suite,
+    topic,
+    baselines_dir,
+    demographics,
+    full_evaluation,
+    country,
+    temperature,
+    effort,
+    prompt_template,
+    elicitation,
+    submit_after,
+    wait,
+    submit_api_key,
+    submit_api_url,
+    submit_message,
+    submit_timeout,
+    poll_interval,
+    poll_timeout,
+):
+    """Run a benchmark evaluation.
+
+    Example:
+        synthbench run --provider raw-anthropic --model haiku --n 100
+        synthbench run --provider raw-anthropic --model haiku --suite core
+        synthbench run --provider raw-anthropic --topic consumer
+        synthbench run --provider raw-anthropic --demographics AGE,POLIDEOLOGY
+        synthbench run --provider raw-anthropic --full-evaluation
+        synthbench run -p raw-anthropic -m haiku -s 30 -n 100 --submit --wait
+    """
+    # Resolve demographics (attribute list resolved in _run_async after dataset loads)
+    demo_list = None
+    if full_evaluation:
+        demo_list = "__all__"  # sentinel: resolved per-dataset in _run_async
+    elif demographics:
+        demo_list = [d.strip().upper() for d in demographics.split(",")]
+
+    # --wait without --submit is a user error: we have nothing to wait on.
+    # Fail fast rather than silently ignore so a misconfigured CI job
+    # doesn't pass when it should have blocked on validation.
+    if wait and not submit_after:
+        click.echo("Error: --wait requires --submit.", err=True)
+        sys.exit(2)
+    if submit_message and not submit_after:
+        click.echo("Error: --submit-message requires --submit.", err=True)
+        sys.exit(2)
+    # --submit + --json-only would write the result JSON to stdout instead of
+    # a file, leaving no artifact on disk for the user to re-submit later if
+    # the upload fails. Refuse rather than quietly drop one of the two.
+    if submit_after and json_only:
+        click.echo(
+            "Error: --submit is incompatible with --json-only (no file to upload).",
+            err=True,
+        )
+        sys.exit(2)
+
+    asyncio.run(
+        _run_async(
+            provider,
+            model,
+            dataset,
+            n,
+            samples,
+            concurrency,
+            output,
+            data_dir,
+            url,
+            json_only,
+            suite,
+            topic,
+            baselines_dir,
+            demo_list,
+            country,
+            temperature,
+            effort,
+            prompt_template,
+            elicitation,
+            submit_after=submit_after,
+            wait=wait,
+            submit_api_key=submit_api_key,
+            submit_api_url=submit_api_url,
+            submit_message=submit_message,
+            submit_timeout=submit_timeout,
+            poll_interval=poll_interval,
+            poll_timeout=poll_timeout,
+        )
+    )
+
+
+async def _run_async(
+    provider_name,
+    model,
+    dataset_name,
+    n,
+    samples,
+    concurrency,
+    output,
+    data_dir,
+    url,
+    json_only,
+    suite,
+    topic,
+    baselines_dir,
+    demographics=None,
+    country=None,
+    temperature=None,
+    effort=None,
+    prompt_template=None,
+    elicitation=None,
+    *,
+    submit_after: bool = False,
+    wait: bool = False,
+    submit_api_key=None,
+    submit_api_url=None,
+    submit_message=None,
+    submit_timeout: int = 60,
+    poll_interval: float = 10.0,
+    poll_timeout: float = 900.0,
+):
+    from synthbench.datasets import DATASETS
+    from synthbench.providers import load_provider
+    from synthbench.runner import BenchmarkRunner
+    from synthbench import report
+
+    # Resolve model alias
+    resolved_model = MODEL_ALIASES.get(model, model)
+
+    # Load dataset
+    if dataset_name not in DATASETS:
+        click.echo(
+            f"Unknown dataset '{dataset_name}'. Available: {list(DATASETS)}", err=True
+        )
+        sys.exit(1)
+
+    ds_kwargs = {}
+    if data_dir:
+        ds_kwargs["data_dir"] = data_dir
+    if country:
+        ds_kwargs["country"] = country
+    ds = DATASETS[dataset_name](**ds_kwargs)
+
+    # Resolve --full-evaluation sentinel to dataset-specific attributes
+    if demographics == "__all__":
+        ds_attrs = getattr(ds, "DEMOGRAPHIC_ATTRIBUTES", None)
+        if ds_attrs:
+            demographics = list(ds_attrs)
+        else:
+            click.echo(
+                f"Dataset '{dataset_name}' does not support demographic evaluation.",
+                err=True,
+            )
+            demographics = None
+
+    # Load provider
+    provider_kwargs = _provider_kwargs(
+        provider_name,
+        model=resolved_model,
+        url=url,
+        temperature=temperature,
+        prompt_template=prompt_template,
+        effort=effort,
+        elicitation=elicitation,
+    )
+    try:
+        prov = load_provider(provider_name, **provider_kwargs)
+    except KeyError as e:
+        click.echo(str(e), err=True)
+        sys.exit(1)
+    except ImportError as e:
+        click.echo(str(e), err=True)
+        sys.exit(1)
+    except ValueError as e:
+        # Provider constructors raise ValueError for knob combinations they
+        # cannot honour (e.g. --effort with a custom --temperature on
+        # reasoning models). Surface cleanly instead of a traceback.
+        click.echo(f"Error: {e}", err=True)
+        sys.exit(2)
+
+    # Load suite if specified
+    question_keys = None
+    if suite:
+        from synthbench.suites import load_suite
+
+        question_keys = load_suite(suite)
+
+    # Load topic suite if specified (intersect with suite if both given)
+    if topic:
+        from synthbench.suites import load_topic_suite
+
+        topic_keys = load_topic_suite(topic)
+        if question_keys is not None:
+            # Intersect: keep only keys in both suite and topic
+            topic_set = set(topic_keys)
+            question_keys = [k for k in question_keys if k in topic_set]
+        else:
+            question_keys = topic_keys
+
+    # Run benchmark
+    runner = BenchmarkRunner(
+        dataset=ds,
+        provider=prov,
+        samples_per_question=samples,
+        concurrency=concurrency,
+    )
+
+    suite_label = f"suite={suite}" if suite else (str(n) if n else "all")
+    if topic:
+        suite_label += f" topic={topic}"
+    click.echo(f"SynthBench v{__version__}")
+    click.echo(f"  Provider: {prov.name}")
+    click.echo(f"  Dataset:  {ds.name}")
+    click.echo(f"  Questions: {suite_label}")
+    click.echo(f"  Samples/q: {samples}")
+    if effort:
+        click.echo(f"  Effort:   {effort}")
+    if demographics:
+        click.echo(f"  Demographics: {', '.join(demographics)}")
+    click.echo()
+
+    def progress(done, total, qr):
+        pct = done / total * 100
+        click.echo(
+            f"\r  [{done}/{total}] {pct:5.1f}% | "
+            f"JSD={qr.jsd:.4f} tau={qr.kendall_tau:.4f}",
+            nl=False,
+        )
+
+    try:
+        if demographics:
+            result = await runner.run_with_demographics(
+                demographics=demographics,
+                n=n,
+                progress_callback=progress,
+                question_keys=question_keys,
+            )
+        else:
+            result = await runner.run(
+                n=n, progress_callback=progress, question_keys=question_keys
+            )
+    finally:
+        await prov.close()
+
+    # Tag result with topic if specified
+    if topic:
+        result.config["topic"] = topic
+    if temperature is not None:
+        result.config["temperature"] = temperature
+    # Stamped only when the knob was actually threaded to the provider —
+    # an absent key means "provider default reasoning behaviour", and
+    # effort-absent runs hash to the same config_id as before the effort
+    # dimension existed.
+    if effort is not None:
+        result.config["effort"] = effort
+    if prompt_template is not None:
+        result.config["prompt_template"] = prompt_template
+    if elicitation is not None and elicitation != "natural":
+        # Structured elicitation is stamped on the template axis so it
+        # groups/dedups/config_ids as a distinct variant (tpl=structured),
+        # plus an explicit elicitation key for human readers.
+        result.config["elicitation"] = elicitation
+        result.config["prompt_template"] = elicitation
+
+    click.echo()  # Newline after progress
+    click.echo()
+
+    # Per-metric SPS summary with CIs
+    components = result.sps_components
+    cis = result.per_metric_ci
+    sps_ci = cis.get("sps")
+    if sps_ci:
+        click.echo(
+            f"  SPS: {result.sps:.4f} [{sps_ci[0]:.4f}, {sps_ci[1]:.4f}]  ({len(components)} metrics)"
+        )
+    else:
+        click.echo(f"  SPS: {result.sps:.4f}  ({len(components)} metrics)")
+    for key, score in components.items():
+        ci = cis.get(key)
+        if ci:
+            click.echo(f"    {key}: {score:.4f} [{ci[0]:.4f}, {ci[1]:.4f}]")
+        else:
+            click.echo(f"    {key}: {score:.4f}")
+    if result.total_parse_failures > 0:
+        click.echo(f"  Parse failures: {result.total_parse_failures}")
+
+    # Demographic breakdown summary
+    if result.demographic_breakdown:
+        click.echo("  Demographic breakdown:")
+        for attr, group_results in result.demographic_breakdown.items():
+            best = max(group_results, key=lambda g: g.p_dist)
+            worst = min(group_results, key=lambda g: g.p_dist)
+            click.echo(
+                f"    {attr}: Best={best.group} (P_dist={best.p_dist:.4f}) "
+                f"/ Worst={worst.group} (P_dist={worst.p_dist:.4f})"
+            )
+    click.echo()
+
+    if json_only:
+        click.echo(json.dumps(report.to_json(result), indent=2))
+    else:
+        # Print markdown summary to terminal
+        bl_dir = Path(baselines_dir) if baselines_dir else None
+        click.echo(report.to_markdown(result, baselines_dir=bl_dir))
+
+        # Save files
+        json_path, md_path = report.save(result, output)
+        click.echo("\nResults saved:")
+        click.echo(f"  JSON: {json_path}")
+        click.echo(f"  Report: {md_path}")
+
+        if submit_after:
+            # sb-ymux: end-to-end "benchmark → leaderboard" path. The JSON was
+            # just written to disk above; we read it back rather than re-
+            # serializing `result` so the bytes on disk and the bytes
+            # uploaded are guaranteed identical (the Worker's Tier-1
+            # validator and the downstream GH Action both run against the
+            # staged R2 copy, so any drift here would be invisible until
+            # validation fails much later).
+            _submit_and_maybe_wait(
+                json_path=Path(json_path),
+                submit_api_key=submit_api_key,
+                submit_api_url=submit_api_url,
+                submit_message=submit_message,
+                submit_timeout=submit_timeout,
+                wait=wait,
+                poll_interval=poll_interval,
+                poll_timeout=poll_timeout,
+            )
+
+
+@main.command()
+@click.argument("json_file", type=click.Path(exists=True))
+def report(json_file):
+    """Regenerate a markdown report from a JSON score card."""
+    from synthbench.runner import BenchmarkResult, QuestionResult
+    from synthbench import report as report_mod
+
+    with open(json_file) as f:
+        data = json.load(f)
+
+    # Reconstruct BenchmarkResult from JSON
+    questions = [
+        QuestionResult(
+            key=q["key"],
+            text=q["text"],
+            options=q["options"],
+            # Committed gated files are stripped of human_distribution
+            # (issue #308); the regenerated markdown degrades gracefully.
+            human_distribution=q.get("human_distribution") or {},
+            model_distribution=q["model_distribution"],
+            jsd=q["jsd"],
+            kendall_tau=q["kendall_tau"],
+            parity=q["parity"],
+            n_samples=q["n_samples"],
+        )
+        for q in data["per_question"]
+    ]
+
+    result = BenchmarkResult(
+        provider_name=data["config"]["provider"],
+        dataset_name=data["config"]["dataset"],
+        questions=questions,
+        config=data["config"],
+        elapsed_seconds=data["aggregate"].get("elapsed_seconds", 0),
+    )
+
+    click.echo(report_mod.to_markdown(result))
+
+
+@main.command()
+@click.option("--dataset", "-d", default="opinionsqa", help="Dataset to download.")
+@click.option("--data-dir", type=click.Path(), default=None, help="Target directory.")
+def download(dataset, data_dir):
+    """Download a benchmark dataset."""
+    from synthbench.datasets import DATASETS
+
+    if dataset not in DATASETS:
+        click.echo(
+            f"Unknown dataset '{dataset}'. Available: {list(DATASETS)}", err=True
+        )
+        sys.exit(1)
+
+    ds_kwargs = {}
+    if data_dir:
+        ds_kwargs["data_dir"] = data_dir
+    ds = DATASETS[dataset](**ds_kwargs)
+
+    click.echo(f"Loading {dataset} dataset...")
+    try:
+        questions = ds.load()
+        click.echo(f"Loaded {len(questions)} questions.")
+    except Exception as e:
+        click.echo(f"Error: {e}", err=True)
+        sys.exit(1)
+
+
+@main.command()
+@click.argument("files", nargs=-1, required=True, type=click.Path(exists=True))
+@click.option(
+    "--output", "-o", type=click.Path(), default=None, help="Save comparison to file."
+)
+def compare(files, output):
+    """Compare 2+ result JSON files side-by-side with significance testing.
+
+    When exactly 2 files are given, also runs a paired bootstrap test on
+    per-question parity scores and reports delta, p-value, and verdict.
+
+    Example:
+        synthbench compare result1.json result2.json
+    """
+    if len(files) < 2:
+        click.echo("Error: need at least 2 result files to compare.", err=True)
+        sys.exit(1)
+
+    from synthbench.leaderboard import load_result, compare_results
+    from synthbench.stats import paired_bootstrap_test
+
+    results = [load_result(Path(f)) for f in files]
+    md = compare_results(results)
+
+    click.echo(md)
+
+    # Pairwise significance test for exactly 2 results
+    if len(files) == 2:
+        r1, r2 = results
+        cfg1 = r1.get("config", {}).get("provider", "A")
+        cfg2 = r2.get("config", {}).get("provider", "B")
+        pq1 = {q["key"]: q["parity"] for q in r1.get("per_question", [])}
+        pq2 = {q["key"]: q["parity"] for q in r2.get("per_question", [])}
+        common_keys = sorted(set(pq1) & set(pq2))
+
+        if common_keys:
+            scores_a = [pq1[k] for k in common_keys]
+            scores_b = [pq2[k] for k in common_keys]
+            delta, p_val, verdict = paired_bootstrap_test(scores_a, scores_b, seed=42)
+
+            alpha = 0.05
+            sig_str = (
+                f"SIGNIFICANT at alpha={alpha}"
+                if verdict == "significant"
+                else f"NOT significant at alpha={alpha}"
+            )
+
+            click.echo()
+            click.echo("## Statistical Significance")
+            click.echo()
+            click.echo(f"  {cfg1} vs {cfg2}")
+            click.echo(f"  Paired questions: {len(common_keys)}")
+            click.echo(f"  Delta SPS = {delta:+.4f}, p = {p_val:.4f}, {sig_str}")
+            click.echo()
+
+            md += "\n## Statistical Significance\n\n"
+            md += f"- {cfg1} vs {cfg2}\n"
+            md += f"- Paired questions: {len(common_keys)}\n"
+            md += f"- Delta SPS = {delta:+.4f}, p = {p_val:.4f}, **{sig_str}**\n"
+        else:
+            click.echo("\nNo common questions for significance test.", err=True)
+
+    if output:
+        Path(output).write_text(md)
+        click.echo(f"\nSaved to {output}", err=True)
+
+
+# sb-t61h: Tier-2 CLI submission. Wraps the same Worker /submit endpoint the
+# web upload uses, but auths with a personal API key minted on /account so the
+# user never has to leave the terminal. Designed to be the last step of a
+# benchmark pipeline:
+#
+#   synthbench run -p openrouter -m gpt-4o-mini -d opinionsqa -o results/
+#   synthbench submit results/openrouter_gpt-4o-mini_opinionsqa.json
+#
+# sb-ymux: `synthbench run --submit` collapses the two steps into one — the
+# CLI wiring lives here, but the HTTP round-trip is in synthbench.submission
+# so `run --submit` and `submit` share exactly one implementation.
+#
+# Configuration via env vars is the supported path for shell-script automation:
+#   export SYNTHBENCH_API_KEY=sb_xxxx...
+#   export SYNTHBENCH_API_URL=https://api.synthbench.org   # default
+#
+# Flags (--api-key, --api-url) override env so ad-hoc invocations don't have
+# to mutate the shell.
+
+
+def _echo_submit_error(err, *, prefix: str = "Error") -> None:
+    """Render a SubmitError/PollError to stderr in the `synthbench submit`
+    format (HTTP code + server message + hint line), keeping output identical
+    across `submit` and `run --submit`."""
+    if err.status_code:
+        click.echo(f"{prefix} (HTTP {err.status_code}): {err.message}", err=True)
+    else:
+        click.echo(f"{prefix}: {err.message}", err=True)
+    hint = getattr(err, "hint", "")
+    if hint:
+        click.echo(f"  Hint: {hint}", err=True)
+
+
+def _submit_body(
+    *,
+    body: str,
+    source_label: str,
+    api_key: str | None,
+    api_url: str | None,
+    timeout: int,
+    json_out: bool,
+):
+    """Shared POST + stderr/stdout emitter used by both `submit` and
+    `run --submit`. Returns the resolved api_base plus the submission outcome
+    so callers that poll (e.g. --wait) don't re-derive them.
+    """
+    from synthbench import submission as sub
+
+    key = sub.resolve_api_key(api_key)
+    if isinstance(key, sub.SubmitError):
+        _echo_submit_error(key)
+        sys.exit(2)
+
+    base = sub.resolve_api_base(api_url)
+    click.echo(f"Uploading {source_label} → {base}/submit", err=True)
+
+    outcome = sub.post_submission(
+        body=body, api_key=key, api_base=base, timeout=timeout
+    )
+
+    if json_out and isinstance(outcome, sub.SubmitSuccess):
+        click.echo(json.dumps(outcome.payload))
+    elif (
+        json_out
+        and isinstance(outcome, sub.SubmitError)
+        and outcome.payload is not None
+    ):
+        click.echo(json.dumps(outcome.payload))
+
+    if isinstance(outcome, sub.SubmitError):
+        _echo_submit_error(outcome)
+        sys.exit(1)
+
+    click.echo(
+        f"  ✓ submitted (id={outcome.submission_id}, status={outcome.status})",
+        err=True,
+    )
+    warning = outcome.payload.get("warning")
+    if warning:
+        click.echo(f"  ! {warning}", err=True)
+    # Deep-link the user to the in-flight submission so they can poll
+    # status without leaving the terminal-browser loop.
+    site = base.replace("api.", "", 1) if base.startswith("https://api.") else base
+    click.echo(f"  Status URL: {site.rstrip('/')}/account/submissions/", err=True)
+    return key, base, outcome
+
+
+def _submit_and_maybe_wait(
+    *,
+    json_path: Path,
+    submit_api_key,
+    submit_api_url,
+    submit_message,
+    submit_timeout: int,
+    wait: bool,
+    poll_interval: float,
+    poll_timeout: float,
+) -> None:
+    """Upload the saved run file, optionally poll status, and sys.exit() with
+    a code the CI job can branch on.
+
+    Exit code contract (documented in the `synthbench run --submit` help):
+        0  — submitted, and if --wait, status=published (leaderboard updated)
+        1  — submission rejected by the Worker, OR rejected by the validator,
+              OR a hard error (bad key, HTTP 5xx)
+        2  — --wait timed out before the validator reached a terminal state
+    """
+    from synthbench import submission as sub
+
+    body_or_err = sub.read_run_body(str(json_path))
+    if isinstance(body_or_err, sub.SubmitError):
+        _echo_submit_error(body_or_err)
+        sys.exit(1)
+    body = sub.inject_submit_message(body_or_err, submit_message)
+
+    click.echo()
+    click.echo("✓ Benchmark complete. Submitting to leaderboard...", err=True)
+
+    key, base, outcome = _submit_body(
+        body=body,
+        source_label=json_path.name,
+        api_key=submit_api_key,
+        api_url=submit_api_url,
+        timeout=submit_timeout,
+        json_out=False,
+    )
+    if not wait:
+        return
+
+    click.echo(
+        f"  Waiting for validation (poll={poll_interval:g}s, timeout={poll_timeout:g}s)...",
+        err=True,
+    )
+    result = sub.poll_submission_status(
+        submission_id=outcome.submission_id,
+        api_key=key,
+        api_base=base,
+        interval=poll_interval,
+        timeout=poll_timeout,
+        on_poll=lambda s: click.echo(f"  status={s}", err=True),
+    )
+    if isinstance(result, sub.StatusResult):
+        if result.status == "published":
+            entry = result.leaderboard_entry_id or "(no entry id)"
+            click.echo(
+                f"  ✓ published (leaderboard_entry_id={entry})",
+                err=True,
+            )
+            return
+        # Only "rejected" remains for StatusResult since TERMINAL_STATUSES is
+        # {"published", "rejected"}. Surface the reason so CI logs tell the
+        # user what to fix without them having to click through /account.
+        reason = result.rejection_reason or "(no reason given)"
+        click.echo(f"  ✗ rejected: {reason}", err=True)
+        sys.exit(1)
+    if isinstance(result, sub.PollTimeout):
+        click.echo(
+            f"  ! polling timed out after {poll_timeout:g}s "
+            f"(last status={result.last_status}).",
+            err=True,
+        )
+        click.echo(
+            "  Submission continues server-side; check /account/submissions/ or "
+            "re-poll with synthbench submit.",
+            err=True,
+        )
+        sys.exit(2)
+    # PollError
+    _echo_submit_error(result, prefix="Poll error")
+    sys.exit(1)
+
+
+@main.command()
+@click.argument("run_file", type=click.Path(exists=True, dir_okay=False))
+@click.option(
+    "--api-key",
+    default=None,
+    help="API key (defaults to SYNTHBENCH_API_KEY env var). Generate at /account.",
+)
+@click.option(
+    "--api-url",
+    default=None,
+    help=f"Submission endpoint base (defaults to SYNTHBENCH_API_URL or {DEFAULT_API_URL}).",
+)
+@click.option(
+    "--timeout",
+    type=int,
+    default=60,
+    show_default=True,
+    help="HTTP timeout in seconds.",
+)
+@click.option(
+    "--json-out",
+    is_flag=True,
+    help="Print the raw response JSON to stdout (useful for piping).",
+)
+@click.option(
+    "--config",
+    "config_file",
+    type=click.Path(exists=True, dir_okay=False),
+    default=None,
+    help=(
+        "Path to a user-config YAML (sb-vgb / GH#257). Validates the artifact "
+        "and stamps it onto the submission so the leaderboard renders a "
+        "{vendor}/{config_name} row instead of bare {vendor}."
+    ),
+)
+@click.option(
+    "--allow-harness-mismatch",
+    is_flag=True,
+    help=(
+        "Submit even if --config pins a different synthbench version. Off by "
+        "default because the leaderboard treats the harness version as part "
+        "of the reproducible-artifact contract."
+    ),
+)
+def submit(
+    run_file, api_key, api_url, timeout, json_out, config_file, allow_harness_mismatch
+):
+    """Submit a benchmark result JSON to the SynthBench leaderboard.
+
+    Authenticates with a personal API key (sb_<...>) generated at
+    https://synthbench.org/account. The same Tier-1 schema validation that
+    runs in the browser flow runs server-side; on success the submission
+    enters the validation queue and a moderator-free GitHub Actions run
+    publishes it to the leaderboard if it passes the full pipeline.
+
+    Example:
+
+        export SYNTHBENCH_API_KEY=sb_xxxx
+        synthbench submit results/openrouter_gpt-4o-mini_opinionsqa.json
+
+    Submit with a user-configuration artifact (sb-vgb / GH#257):
+
+        synthbench submit --config configs/healthcare-nc.yaml results/run.json
+    """
+    from synthbench import submission as sub
+
+    path = Path(run_file)
+    body = sub.read_run_body(str(path))
+    if isinstance(body, sub.SubmitError):
+        _echo_submit_error(body)
+        sys.exit(1)
+
+    if config_file:
+        from synthbench import user_config as uc
+
+        result = uc.load_and_validate(config_file)
+        if isinstance(result, list):
+            click.echo(f"Config validation failed ({config_file}):", err=True)
+            for err in result:
+                click.echo(f"  - {err}", err=True)
+            sys.exit(2)
+        mismatch = uc.check_harness_version(result)
+        if mismatch is not None:
+            if not allow_harness_mismatch:
+                click.echo(f"Error: {mismatch}", err=True)
+                click.echo(
+                    "  Hint: pass --allow-harness-mismatch to override.", err=True
+                )
+                sys.exit(2)
+            click.echo(f"  ! {mismatch}", err=True)
+        body = uc.attach_to_submission_body(body, result)
+        click.echo(
+            f"  + attaching user_config {result.leaderboard_row_key} "
+            f"(contributor={result.contributor})",
+            err=True,
+        )
+
+    _submit_body(
+        body=body,
+        source_label=path.name,
+        api_key=api_key,
+        api_url=api_url,
+        timeout=timeout,
+        json_out=json_out,
+    )
+
+
+@main.command("validate-config")
+@click.argument("config_file", type=click.Path(exists=True, dir_okay=False))
+def validate_config_cmd(config_file):
+    """Validate a user-configuration YAML against the schema (sb-vgb / GH#257).
+
+    Run this before `synthbench submit --config` to surface schema errors
+    locally. Exit code 0 on success, 2 on validation failure.
+
+    Example:
+
+        synthbench validate-config configs/healthcare-northcentral.yaml
+    """
+    from synthbench import user_config as uc
+
+    result = uc.load_and_validate(config_file)
+    if isinstance(result, list):
+        click.echo(f"✗ {config_file}: invalid", err=True)
+        for err in result:
+            click.echo(f"  - {err}", err=True)
+        sys.exit(2)
+    click.echo(f"✓ {config_file}: valid")
+    click.echo(f"  row: {result.leaderboard_row_key}")
+    click.echo(f"  contributor: {result.contributor}")
+    click.echo(f"  harness: {result.harness_version}")
+    mismatch = uc.check_harness_version(result)
+    if mismatch is not None:
+        click.echo(f"  ! {mismatch}", err=True)
+
+
+@main.command()
+@click.option("--provider", "-p", required=True, help="Provider name.")
+@click.option("--model", "-m", default="haiku", help="Model name or alias.")
+@click.option("--dataset", "-d", default="opinionsqa", help="Dataset.")
+@click.option("--n", "-n", type=int, default=None, help="Questions per run.")
+@click.option("--samples", "-s", type=int, default=30, help="Samples per question.")
+@click.option("--n-runs", type=int, default=5, help="Number of independent runs.")
+@click.option(
+    "--concurrency", "-c", type=int, default=10, help="Max concurrent requests."
+)
+@click.option(
+    "--data-dir", type=click.Path(), default=None, help="Custom data directory."
+)
+@click.option("--url", default=None, help="Endpoint URL for http provider.")
+@click.option(
+    "--suite",
+    type=click.Choice(["smoke", "core", "full"]),
+    default=None,
+    help="Use a pinned question set.",
+)
+@click.option(
+    "--output", "-o", type=click.Path(), default="results", help="Output directory."
+)
+def replicate(
+    provider,
+    model,
+    dataset,
+    n,
+    samples,
+    n_runs,
+    concurrency,
+    data_dir,
+    url,
+    suite,
+    output,
+):
+    """Run a benchmark N times and report stability metrics.
+
+    Answers: "how stable is this provider's score?"
+
+    Example:
+        synthbench replicate --provider raw-anthropic --n-runs 5 --suite core
+    """
+    asyncio.run(
+        _replicate_async(
+            provider,
+            model,
+            dataset,
+            n,
+            samples,
+            n_runs,
+            concurrency,
+            data_dir,
+            url,
+            suite,
+            output,
+        )
+    )
+
+
+async def _replicate_async(
+    provider_name,
+    model,
+    dataset_name,
+    n,
+    samples,
+    n_runs,
+    concurrency,
+    data_dir,
+    url,
+    suite,
+    output,
+):
+    from synthbench.datasets import DATASETS
+    from synthbench.providers import load_provider
+    from synthbench.runner import BenchmarkRunner
+    import statistics
+
+    resolved_model = MODEL_ALIASES.get(model, model)
+
+    if dataset_name not in DATASETS:
+        click.echo(f"Unknown dataset '{dataset_name}'.", err=True)
+        sys.exit(1)
+
+    ds_kwargs = {}
+    if data_dir:
+        ds_kwargs["data_dir"] = data_dir
+    ds = DATASETS[dataset_name](**ds_kwargs)
+
+    provider_kwargs = _provider_kwargs(provider_name, model=resolved_model, url=url)
+    try:
+        prov = load_provider(provider_name, **provider_kwargs)
+    except (KeyError, ImportError) as e:
+        click.echo(str(e), err=True)
+        sys.exit(1)
+
+    # Load suite
+    question_keys = None
+    if suite:
+        from synthbench.suites import load_suite
+
+        question_keys = load_suite(suite)
+
+    runner = BenchmarkRunner(
+        dataset=ds,
+        provider=prov,
+        samples_per_question=samples,
+        concurrency=concurrency,
+    )
+
+    suite_label = f"suite={suite}" if suite else (str(n) if n else "all")
+    click.echo(f"SynthBench Replicate v{__version__}")
+    click.echo(f"  Provider: {prov.name}")
+    click.echo(f"  Dataset:  {ds.name}")
+    click.echo(f"  Questions: {suite_label}")
+    click.echo(f"  Samples/q: {samples}")
+    click.echo(f"  Runs: {n_runs}")
+    click.echo()
+
+    run_metrics: list[dict[str, float]] = []
+
+    try:
+        for run_i in range(n_runs):
+            click.echo(f"  Run {run_i + 1}/{n_runs}...", nl=False)
+            result = await runner.run(n=n, question_keys=question_keys)
+            metrics = {"sps": result.sps, **result.sps_components}
+            run_metrics.append(metrics)
+            click.echo(f" SPS={result.sps:.4f}")
+    finally:
+        await prov.close()
+
+    click.echo()
+
+    # Aggregate
+    all_metric_names = sorted(set().union(*run_metrics))
+    lines = [
+        "# SynthBench Replication Report",
+        "",
+        f"**Provider:** {prov.name}",
+        f"**Runs:** {n_runs}",
+        f"**Questions:** {suite_label}",
+        f"**Samples/q:** {samples}",
+        "",
+        "| Metric | Mean | Std | Min | Max | CV |",
+        "|--------|------|-----|-----|-----|-----|",
+    ]
+
+    summary = {}
+    for metric in all_metric_names:
+        vals = [rm[metric] for rm in run_metrics if metric in rm]
+        if not vals:
+            continue
+        mean_val = statistics.mean(vals)
+        std_val = statistics.stdev(vals) if len(vals) > 1 else 0.0
+        min_val = min(vals)
+        max_val = max(vals)
+        cv = std_val / mean_val if mean_val > 0 else 0.0
+
+        lines.append(
+            f"| {metric} | {mean_val:.4f} | {std_val:.4f} "
+            f"| {min_val:.4f} | {max_val:.4f} | {cv:.4f} |"
+        )
+        summary[metric] = {
+            "mean": round(mean_val, 6),
+            "std": round(std_val, 6),
+            "min": round(min_val, 6),
+            "max": round(max_val, 6),
+            "cv": round(cv, 6),
+            "values": [round(v, 6) for v in vals],
+        }
+
+    lines.append("")
+    md = "\n".join(lines)
+    click.echo(md)
+
+    # Save
+    out_dir = Path(output)
+    out_dir.mkdir(parents=True, exist_ok=True)
+    from datetime import datetime
+
+    ts = datetime.now().strftime("%Y%m%d_%H%M%S")
+    provider_slug = prov.name.replace("/", "_")
+
+    json_data = {
+        "benchmark": "synthbench",
+        "type": "replication",
+        "version": __version__,
+        "config": {
+            "provider": prov.name,
+            "dataset": ds.name,
+            "n_runs": n_runs,
+            "n_questions": suite_label,
+            "samples_per_question": samples,
+        },
+        "per_run": run_metrics,
+        "summary": summary,
+    }
+    json_path = out_dir / f"replicate_{dataset_name}_{provider_slug}_{ts}.json"
+    json_path.write_text(json.dumps(json_data, indent=2))
+    click.echo(f"\nSaved: {json_path}")
+
+
+@main.command()
+@click.option(
+    "--results-dir",
+    "-d",
+    type=click.Path(exists=True),
+    default="results",
+    help="Directory containing result JSON files.",
+)
+@click.option(
+    "--output",
+    "-o",
+    type=click.Path(),
+    default=None,
+    help="Save markdown + JSON output.",
+)
+@click.option("--json", "json_only", is_flag=True, help="Output JSON only.")
+@click.option(
+    "--topic",
+    type=click.Choice(["political", "consumer", "neutral"]),
+    default=None,
+    help="Filter to results for a specific topic.",
+)
+@click.option("--all", "show_all", is_flag=True, help="Show all runs (no dedup).")
+@click.option(
+    "--model", "model_filter", default=None, help="Filter to a specific model."
+)
+def leaderboard(results_dir, output, json_only, topic, show_all, model_filter):
+    """Build a ranked leaderboard from all result files in a directory.
+
+    Example:
+        synthbench leaderboard --results-dir ./results
+        synthbench leaderboard --results-dir ./results --all
+        synthbench leaderboard --results-dir ./results --model haiku
+        synthbench leaderboard --results-dir ./results --topic consumer
+    """
+    from synthbench.leaderboard import load_result, build_leaderboard
+
+    results_path = Path(results_dir)
+    json_files = sorted(results_path.glob("*.json"))
+
+    if not json_files:
+        click.echo(f"No JSON files found in {results_dir}", err=True)
+        sys.exit(1)
+
+    # Load and filter to valid synthbench results
+    results = []
+    for jf in json_files:
+        try:
+            data = load_result(jf)
+            if data.get("benchmark") == "synthbench":
+                results.append(data)
+        except (json.JSONDecodeError, KeyError):
+            continue
+
+    if not results:
+        click.echo("No valid SynthBench result files found.", err=True)
+        sys.exit(1)
+
+    # Filter by topic if specified
+    if topic:
+        results = [r for r in results if r.get("config", {}).get("topic") == topic]
+        if not results:
+            click.echo(f"No results found for topic '{topic}'.", err=True)
+            sys.exit(1)
+
+    md, lb_json = build_leaderboard(
+        results, show_all=show_all, model_filter=model_filter
+    )
+
+    if json_only:
+        click.echo(json.dumps(lb_json, indent=2))
+    else:
+        click.echo(md)
+
+    if output:
+        out = Path(output)
+        if json_only:
+            out.write_text(json.dumps(lb_json, indent=2))
+        else:
+            out.write_text(md)
+            # Also write JSON sidecar
+            json_out = out.with_suffix(".json")
+            json_out.write_text(json.dumps(lb_json, indent=2))
+            click.echo(f"\nSaved: {out} + {json_out}", err=True)
+
+
+@main.command()
+@click.option(
+    "--provider",
+    "-p",
+    required=True,
+    help="Provider name (raw-anthropic, openrouter, etc.).",
+)
+@click.option(
+    "--model",
+    "-m",
+    default="haiku",
+    help="Model name or alias.",
+)
+@click.option(
+    "--output",
+    "-o",
+    type=click.Path(),
+    default="results",
+    help="Output directory for results.",
+)
+@click.option(
+    "--concurrency",
+    "-c",
+    type=int,
+    default=10,
+    help="Max concurrent API requests.",
+)
+@click.option(
+    "--check",
+    is_flag=True,
+    help="Dry-run: show [DONE]/[MISSING]/[PARTIAL] for each suite item.",
+)
+@click.option(
+    "--force", is_flag=True, help="Re-run all benchmarks ignoring existing results."
+)
+@click.option(
+    "--repeats",
+    type=int,
+    default=None,
+    help="Override per-run repeat count (default: from suite YAML, typically 3).",
+)
+@click.option(
+    "--suite-name",
+    default="standard",
+    help="Suite config name (default: standard).",
+)
+@click.option("--url", default=None, help="Endpoint URL for http provider.")
+@click.option(
+    "--data-dir", type=click.Path(), default=None, help="Custom data directory."
+)
+@click.option(
+    "--temperature",
+    type=float,
+    default=None,
+    help="Sampling temperature to pass to the provider (e.g., 0.7, 1.0).",
+)
+def suite(
+    provider,
+    model,
+    output,
+    concurrency,
+    check,
+    force,
+    repeats,
+    suite_name,
+    url,
+    data_dir,
+    temperature,
+):
+    """Run a full structured benchmark matrix for a provider.
+
+    Executes all runs defined in the suite config (default: 7 runs x 3 repeats
+    covering OpinionsQA, GlobalOpinionQA, SubPOP, and a replicate).
+
+    Gap-fill mode (default): only runs missing or partially-completed repeats.
+    New tests added to the suite YAML are automatically picked up.
+    Use --force to re-run everything.
+
+    Variance reporting: after repeats, shows mean, std, CV for each run.
+    Flags runs where CV > 2% as high variance.
+
+    Examples:
+        synthbench suite --provider openrouter --model anthropic/claude-haiku-4-5
+        synthbench suite --check --provider openrouter --model anthropic/claude-haiku-4-5
+        synthbench suite --provider openrouter --model anthropic/claude-haiku-4-5 --force
+        synthbench suite --provider openrouter --model anthropic/claude-haiku-4-5 --repeats 5
+    """
+    from synthbench.suite import (
+        check_suite,
+        format_check,
+        format_summary,
+        run_suite as _run_suite,
+    )
+    from synthbench.providers import load_provider
+
+    output_dir = Path(output)
+    resolved_model = MODEL_ALIASES.get(model, model)
+
+    # Resolve the provider name for matching
+    provider_kwargs = _provider_kwargs(
+        provider, model=resolved_model, url=url, temperature=temperature
+    )
+    try:
+        prov = load_provider(provider, **provider_kwargs)
+        resolved_provider = prov.name
+    except (KeyError, ImportError) as e:
+        click.echo(str(e), err=True)
+        sys.exit(1)
+
+    if check:
+        annotated = check_suite(
+            resolved_provider, output_dir, suite_name, repeats_override=repeats
+        )
+        click.echo(format_check(annotated, resolved_provider))
+        return
+
+    mode = "force (re-run all)" if force else "gap-fill (skip existing)"
+    click.echo(f"SynthBench Suite v{__version__}")
+    click.echo(f"  Provider: {resolved_provider}")
+    click.echo(f"  Suite:    {suite_name}")
+    click.echo(f"  Output:   {output_dir}")
+    click.echo(f"  Mode:     {mode}")
+    if repeats is not None:
+        click.echo(f"  Repeats:  {repeats} (override)")
+    click.echo()
+
+    summaries = asyncio.run(
+        _run_suite(
+            provider_name=provider,
+            model=model,
+            output_dir=output_dir,
+            concurrency=concurrency,
+            suite_name=suite_name,
+            url=url,
+            data_dir=data_dir,
+            force=force,
+            repeats_override=repeats,
+            temperature=temperature,
+        )
+    )
+
+    click.echo()
+    click.echo(format_summary(summaries, resolved_provider))
+
+
+@main.group("convergence")
+def convergence_group():
+    """Convergence analysis: cross-model contamination + bootstrap curves."""
+    pass
+
+
+@convergence_group.command("cross-model")
+@click.option(
+    "--results-dir",
+    "-d",
+    type=click.Path(exists=True),
+    default="leaderboard-results",
+    help="Directory containing result JSON files.",
+)
+@click.option(
+    "--output",
+    "-o",
+    type=click.Path(),
+    default=None,
+    help="Save report to file (markdown + JSON sidecar).",
+)
+@click.option("--json", "json_only", is_flag=True, help="Output JSON only.")
+@click.option(
+    "--min-models",
+    type=int,
+    default=2,
+    help="Minimum models a question must appear in (default: 2).",
+)
+@click.option(
+    "--model",
+    "model_filter",
+    default=None,
+    help="Filter to results matching a specific model/provider substring.",
+)
+@click.option(
+    "--dataset",
+    "dataset_filter",
+    default=None,
+    help="Filter to results for a specific dataset.",
+)
+def contamination_convergence(
+    results_dir, output, json_only, min_models, model_filter, dataset_filter
+):
+    """Detect per-question contamination via cross-model convergence.
+
+    For each question across all result files, computes the standard deviation
+    of model response distributions. Low variance = likely contaminated (all
+    models recall the same memorized data). High variance = genuine reasoning.
+
+    Example:
+        synthbench convergence cross-model
+        synthbench convergence cross-model --results-dir ./results
+        synthbench convergence cross-model --dataset opinionsqa --min-models 3
+        synthbench convergence cross-model --json
+    """
+    from synthbench.contamination import (
+        convergence_analysis,
+        convergence_to_json,
+        format_convergence_report,
+    )
+
+    results_path = Path(results_dir)
+    json_files = sorted(results_path.glob("*.json"))
+
+    if not json_files:
+        click.echo(f"No JSON files found in {results_dir}", err=True)
+        sys.exit(1)
+
+    # Filter files
+    if model_filter or dataset_filter:
+        filtered = []
+        for jf in json_files:
+            try:
+                with open(jf) as f:
+                    data = json.load(f)
+                if data.get("benchmark") != "synthbench":
+                    continue
+                cfg = data.get("config", {})
+                if model_filter and model_filter not in cfg.get("provider", ""):
+                    continue
+                if dataset_filter and cfg.get("dataset") != dataset_filter:
+                    continue
+                filtered.append(jf)
+            except (json.JSONDecodeError, KeyError):
+                continue
+        json_files = filtered
+
+    if len(json_files) < 2:
+        click.echo(
+            f"Need at least 2 result files for convergence analysis, found {len(json_files)}",
+            err=True,
+        )
+        sys.exit(1)
+
+    click.echo(f"Analyzing {len(json_files)} result files...", err=True)
+
+    try:
+        analysis = convergence_analysis(json_files, min_models=min_models)
+    except ValueError as e:
+        click.echo(f"Error: {e}", err=True)
+        sys.exit(1)
+
+    if json_only:
+        click.echo(json.dumps(convergence_to_json(analysis), indent=2))
+    else:
+        click.echo(format_convergence_report(analysis))
+
+    if output:
+        out = Path(output)
+        if json_only:
+            out.write_text(json.dumps(convergence_to_json(analysis), indent=2))
+        else:
+            out.write_text(format_convergence_report(analysis))
+            json_out = out.with_suffix(".json")
+            json_out.write_text(json.dumps(convergence_to_json(analysis), indent=2))
+            click.echo(f"\nSaved: {out} + {json_out}", err=True)
+
+
+@convergence_group.command("bootstrap")
+@click.option(
+    "--dataset",
+    "-d",
+    required=True,
+    help="Dataset name (gss, wvs, eurobarometer, opinionsqa, pewtech, "
+    "michigan, ntia, globalopinionqa, subpop).",
+)
+@click.option(
+    "--question",
+    "-q",
+    default=None,
+    help="Restrict to a single question key (default: all questions).",
+)
+@click.option(
+    "--output",
+    "-o",
+    type=click.Path(),
+    default=None,
+    help="Write JSON to this path (default: stdout).",
+)
+@click.option(
+    "--plot",
+    is_flag=True,
+    help="Also emit a matplotlib PDF of per-question curves "
+    "(requires matplotlib; written alongside --output).",
+)
+@click.option(
+    "--bootstraps",
+    "-b",
+    type=int,
+    default=None,
+    help="Bootstrap replicates per sample size (default: 500).",
+)
+@click.option(
+    "--sample-sizes",
+    default=None,
+    help="Comma-separated sample sizes (default: 20,50,100,200,500,1000,2000,5000,10000).",
+)
+@click.option(
+    "--epsilon",
+    type=float,
+    default=None,
+    help="JSD ceiling for 'convergence n' threshold (default: 0.02).",
+)
+@click.option(
+    "--delta",
+    type=float,
+    default=None,
+    help="Max allowed drop in jsd_mean across next 2 points (default: 0.005).",
+)
+@click.option(
+    "--seed",
+    type=int,
+    default=None,
+    help="Seed for reproducibility (default: non-deterministic).",
+)
+@click.option(
+    "--n",
+    type=int,
+    default=None,
+    help="Limit to first N questions of the dataset (debugging).",
+)
+def convergence_bootstrap(
+    dataset,
+    question,
+    output,
+    plot,
+    bootstraps,
+    sample_sizes,
+    epsilon,
+    delta,
+    seed,
+    n,
+):
+    """Bootstrap convergence curves from aggregate human distributions.
+
+    For each question in DATASET, multinomially sub-samples responses at
+    increasing sample sizes and measures JSD(sample, full_distribution). The
+    resulting curve is the theoretical ~1/sqrt(n) floor an idealized i.i.d.
+    sampler would achieve. Use this as the baseline that real human and
+    synthetic sampling gets compared against.
+
+    Redistribution policy is honored: 'full' and 'gated' datasets emit the
+    full per-question artifact (gated routes to R2 at publish time);
+    'aggregates_only' emits dataset-wide summary stats only; 'citation_only'
+    suppresses all artifacts.
+
+    Example:
+        synthbench convergence bootstrap --dataset opinionsqa --output curves.json
+        synthbench convergence bootstrap --dataset gss --plot --output gss.json
+    """
+    from synthbench.convergence.cli_report import run_bootstrap
+
+    try:
+        payload, pdf_path = run_bootstrap(
+            dataset_name=dataset,
+            question_key=question,
+            output=Path(output) if output else None,
+            plot=plot,
+            bootstraps=bootstraps,
+            sample_sizes=sample_sizes,
+            epsilon=epsilon,
+            delta=delta,
+            seed=seed,
+            n_limit=n,
+        )
+    except ValueError as e:
+        click.echo(f"Error: {e}", err=True)
+        sys.exit(1)
+
+    if output:
+        click.echo(f"Wrote {output}", err=True)
+        if pdf_path:
+            click.echo(f"Wrote {pdf_path}", err=True)
+    else:
+        click.echo(json.dumps(payload, indent=2))
+
+
+@convergence_group.command("real")
+@click.option("--dataset", "-d", required=True, help="Dataset name (e.g. gss).")
+@click.option(
+    "--question",
+    "-q",
+    required=True,
+    help="Question key (bare upstream id like 'SPKATH' or prefixed 'GSS_SPKATH').",
+)
+@click.option(
+    "--output",
+    "-o",
+    type=click.Path(),
+    default=None,
+    help="Write JSON to this path (default: stdout).",
+)
+@click.option(
+    "--bootstraps",
+    "-b",
+    type=int,
+    default=None,
+    help="Sub-sample replicates per sample size (default: 500).",
+)
+@click.option(
+    "--sample-sizes",
+    default=None,
+    help="Comma-separated sample sizes (default: 20,50,100,200,500,1000,2000,5000,10000).",
+)
+@click.option("--epsilon", type=float, default=None)
+@click.option("--delta", type=float, default=None)
+@click.option("--seed", type=int, default=None)
+def convergence_real(
+    dataset, question, output, bootstraps, sample_sizes, epsilon, delta, seed
+):
+    """Real-sampling convergence curve from individual-level microdata.
+
+    Sub-samples actual respondents (without replacement) at each sample size
+    and measures JSD vs. the full-population distribution. Errors cleanly
+    when the dataset's adapter does not provide microdata.
+
+    Example:
+        synthbench convergence real --dataset gss --question SPKATH \\
+            --output /tmp/gss-real.json
+    """
+    from synthbench.convergence.cli_report import run_real
+
+    try:
+        payload = run_real(
+            dataset_name=dataset,
+            question_key=question,
+            output=Path(output) if output else None,
+            bootstraps=bootstraps,
+            sample_sizes=sample_sizes,
+            epsilon=epsilon,
+            delta=delta,
+            seed=seed,
+        )
+    except ValueError as e:
+        click.echo(f"Error: {e}", err=True)
+        sys.exit(1)
+
+    if output:
+        click.echo(f"Wrote {output}", err=True)
+    else:
+        click.echo(json.dumps(payload, indent=2))
+
+
+@convergence_group.command("compare")
+@click.option("--dataset", "-d", required=True)
+@click.option(
+    "--question",
+    "-q",
+    required=True,
+    help="Question key (bare upstream id like 'SPKATH' or prefixed 'GSS_SPKATH').",
+)
+@click.option("--output", "-o", type=click.Path(), default=None)
+@click.option("--bootstraps", "-b", type=int, default=None)
+@click.option("--sample-sizes", default=None)
+@click.option("--epsilon", type=float, default=None)
+@click.option("--delta", type=float, default=None)
+@click.option("--seed", type=int, default=None)
+def convergence_compare(
+    dataset, question, output, bootstraps, sample_sizes, epsilon, delta, seed
+):
+    """Side-by-side bootstrap (idealized) and real-sampling curves.
+
+    Emits both curves under the same parameters with a precomputed
+    ``delta_jsd_mean`` per sample size -- the headline gap between the
+    1/sqrt(n) i.i.d. floor and real sampling under population heterogeneity.
+
+    Example:
+        synthbench convergence compare --dataset gss --question SPKATH \\
+            --output /tmp/gss-compare.json
+    """
+    from synthbench.convergence.cli_report import run_compare
+
+    try:
+        payload = run_compare(
+            dataset_name=dataset,
+            question_key=question,
+            output=Path(output) if output else None,
+            bootstraps=bootstraps,
+            sample_sizes=sample_sizes,
+            epsilon=epsilon,
+            delta=delta,
+            seed=seed,
+        )
+    except ValueError as e:
+        click.echo(f"Error: {e}", err=True)
+        sys.exit(1)
+
+    if output:
+        click.echo(f"Wrote {output}", err=True)
+    else:
+        click.echo(json.dumps(payload, indent=2))
+
+
+@main.command()
+@click.option(
+    "--provider",
+    "-p",
+    required=True,
+    help="Provider name (raw-anthropic, openrouter, etc.).",
+)
+@click.option(
+    "--model",
+    "-m",
+    default="haiku",
+    help="Model name or alias.",
+)
+@click.option(
+    "--output",
+    "-o",
+    type=click.Path(),
+    default="results",
+    help="Output directory for results.",
+)
+@click.option(
+    "--samples",
+    "-s",
+    type=int,
+    default=30,
+    help="Samples per question for distribution estimation.",
+)
+@click.option(
+    "--concurrency",
+    "-c",
+    type=int,
+    default=10,
+    help="Max concurrent API requests.",
+)
+@click.option("--url", default=None, help="Endpoint URL for http provider.")
+@click.option(
+    "--json-only", is_flag=True, help="Output JSON to stdout instead of files."
+)
+def contamination(provider, model, output, samples, concurrency, url, json_only):
+    """Run paraphrase sensitivity test to detect training corpus contamination.
+
+    Evaluates 50 original questions + 150 paraphrased variants and reports
+    the sensitivity delta. High sensitivity suggests memorization.
+
+    Example:
+        synthbench contamination --provider openrouter --model anthropic/claude-haiku-4-5
+    """
+    asyncio.run(
+        _contamination_async(
+            provider, model, output, samples, concurrency, url, json_only
+        )
+    )
+
+
+async def _contamination_async(
+    provider_name, model, output, samples, concurrency, url, json_only
+):
+    from synthbench.contamination import (
+        run_contamination_test,
+        result_to_json,
+    )
+    from synthbench.providers import load_provider
+
+    resolved_model = MODEL_ALIASES.get(model, model)
+
+    provider_kwargs = _provider_kwargs(provider_name, model=resolved_model, url=url)
+    try:
+        prov = load_provider(provider_name, **provider_kwargs)
+    except (KeyError, ImportError) as e:
+        click.echo(str(e), err=True)
+        sys.exit(1)
+
+    # Suite path
+    suite_path = Path(__file__).parent / "suites" / "paraphrase_test.json"
+
+    click.echo(f"SynthBench Contamination Test v{__version__}")
+    click.echo(f"  Provider: {prov.name}")
+    click.echo(f"  Samples/q: {samples}")
+    click.echo("  Questions: 50 original + 150 paraphrased")
+    click.echo()
+
+    try:
+        result = await run_contamination_test(
+            provider=prov,
+            samples_per_question=samples,
+            concurrency=concurrency,
+            suite_path=suite_path,
+        )
+    finally:
+        await prov.close()
+
+    # Display results
+    click.echo(
+        f"  Parity {result.original_sps:.3f} "
+        f"(adjusted: {result.adjusted_sps:.3f}, "
+        f"sensitivity: {result.sensitivity_pct:.1f}%)"
+    )
+    click.echo(f"  Elapsed: {result.elapsed_seconds:.1f}s")
+    click.echo()
+
+    result_data = result_to_json(result)
+
+    if json_only:
+        click.echo(json.dumps(result_data, indent=2))
+    else:
+        # Per-question breakdown (top 10 most sensitive)
+        sorted_qs = sorted(
+            result.per_question, key=lambda q: q.sensitivity_pct, reverse=True
+        )
+        click.echo("## Most Sensitive Questions (top 10)")
+        click.echo()
+        click.echo("| Question | Original | Adjusted | Delta | Sensitivity |")
+        click.echo("|----------|----------|----------|-------|-------------|")
+        for q in sorted_qs[:10]:
+            click.echo(
+                f"| {q.original_text[:50]}... "
+                f"| {q.original_parity:.4f} "
+                f"| {q.mean_paraphrase_parity:.4f} "
+                f"| {q.delta:+.4f} "
+                f"| {q.sensitivity_pct:.1f}% |"
+            )
+        click.echo()
+
+        # Save
+        out_dir = Path(output)
+        out_dir.mkdir(parents=True, exist_ok=True)
+        from datetime import datetime
+
+        ts = datetime.now().strftime("%Y%m%d_%H%M%S")
+        provider_slug = prov.name.replace("/", "_")
+        json_path = out_dir / f"contamination_{provider_slug}_{ts}.json"
+        json_path.write_text(json.dumps(result_data, indent=2))
+        click.echo(f"Results saved: {json_path}")
+
+
+@main.command("contamination-deident")
+@click.option(
+    "--provider",
+    "-p",
+    required=True,
+    help="Provider name (raw-anthropic, openrouter, etc.).",
+)
+@click.option(
+    "--model",
+    "-m",
+    default="haiku",
+    help="Model name or alias.",
+)
+@click.option(
+    "--output",
+    "-o",
+    type=click.Path(),
+    default="results",
+    help="Output directory for results.",
+)
+@click.option(
+    "--samples",
+    "-s",
+    type=int,
+    default=30,
+    help="Samples per question for distribution estimation.",
+)
+@click.option(
+    "--concurrency",
+    "-c",
+    type=int,
+    default=10,
+    help="Max concurrent API requests.",
+)
+@click.option("--url", default=None, help="Endpoint URL for http provider.")
+@click.option(
+    "--json-only", is_flag=True, help="Output JSON to stdout instead of files."
+)
+def contamination_deident(
+    provider, model, output, samples, concurrency, url, json_only
+):
+    """Run de-identification sensitivity test to detect brand recognition.
+
+    Evaluates 20 well-known topics at 5 progressively abstracted levels
+    (full brand -> abstract feature description). Providers that reason
+    from features produce stable distributions; providers that recognize
+    the brand drift as identifying information is stripped.
+
+    Example:
+        synthbench contamination-deident --provider openrouter --model anthropic/claude-haiku-4-5
+    """
+    asyncio.run(
+        _contamination_deident_async(
+            provider, model, output, samples, concurrency, url, json_only
+        )
+    )
+
+
+async def _contamination_deident_async(
+    provider_name, model, output, samples, concurrency, url, json_only
+):
+    from synthbench.contamination import (
+        deident_result_to_json,
+        run_deident_test,
+    )
+    from synthbench.providers import load_provider
+
+    resolved_model = MODEL_ALIASES.get(model, model)
+
+    provider_kwargs = _provider_kwargs(provider_name, model=resolved_model, url=url)
+    try:
+        prov = load_provider(provider_name, **provider_kwargs)
+    except (KeyError, ImportError) as e:
+        click.echo(str(e), err=True)
+        sys.exit(1)
+
+    suite_path = Path(__file__).parent / "suites" / "deident_test.json"
+
+    click.echo(f"SynthBench De-identification Test v{__version__}")
+    click.echo(f"  Provider: {prov.name}")
+    click.echo(f"  Samples/q: {samples}")
+    click.echo("  Topics: 20 × 5 levels = 100 distributions")
+    click.echo()
+
+    try:
+        result = await run_deident_test(
+            provider=prov,
+            samples_per_question=samples,
+            concurrency=concurrency,
+            suite_path=suite_path,
+        )
+    finally:
+        await prov.close()
+
+    click.echo(
+        f"  Mean pairwise JSD: {result.mean_pairwise_jsd:.4f} "
+        f"(option std {result.mean_option_std:.4f}, "
+        f"L1->L5 drift {result.mean_drift_l1_to_l5:.4f})"
+    )
+    click.echo(f"  Elapsed: {result.elapsed_seconds:.1f}s")
+    click.echo()
+
+    result_data = deident_result_to_json(result)
+
+    if json_only:
+        click.echo(json.dumps(result_data, indent=2))
+    else:
+        sorted_topics = sorted(
+            result.per_topic, key=lambda t: t.mean_pairwise_jsd, reverse=True
+        )
+        click.echo("## Most Recognition-Sensitive Topics (top 10)")
+        click.echo()
+        click.echo("| Topic | Mean Pairwise JSD | Option Std | L1->L5 Drift |")
+        click.echo("|-------|-------------------|------------|--------------|")
+        for t in sorted_topics[:10]:
+            click.echo(
+                f"| {t.topic[:40]} "
+                f"| {t.mean_pairwise_jsd:.4f} "
+                f"| {t.mean_option_std:.4f} "
+                f"| {t.drift_l1_to_l5:.4f} |"
+            )
+        click.echo()
+
+        out_dir = Path(output)
+        out_dir.mkdir(parents=True, exist_ok=True)
+        from datetime import datetime
+
+        ts = datetime.now().strftime("%Y%m%d_%H%M%S")
+        provider_slug = prov.name.replace("/", "_")
+        json_path = out_dir / f"contamination_deident_{provider_slug}_{ts}.json"
+        json_path.write_text(json.dumps(result_data, indent=2))
+        click.echo(f"Results saved: {json_path}")
+
+
+@main.command("publish-data")
+@click.option(
+    "--results-dir",
+    "-d",
+    type=click.Path(exists=True),
+    default="leaderboard-results",
+    help="Directory containing result JSON files.",
+)
+@click.option(
+    "--output",
+    "-o",
+    type=click.Path(),
+    default="site/src/data/leaderboard.json",
+    help="Output path for the leaderboard JSON file.",
+)
+def publish_data(results_dir, output):
+    """Export leaderboard data as JSON for the Astro frontend.
+
+    Example:
+        synthbench publish-data --results-dir ./leaderboard-results --output site/src/data/leaderboard.json
+    """
+    from synthbench.publish import publish_leaderboard_data
+
+    try:
+        out_path = publish_leaderboard_data(
+            results_dir=Path(results_dir),
+            output_path=Path(output),
+            version=__version__,
+        )
+        click.echo(f"Leaderboard data exported: {out_path}")
+    except ValueError as e:
+        click.echo(str(e), err=True)
+        sys.exit(1)
+
+
+def _resolve_r2_uploader(no_r2: bool):
+    """Return an R2Uploader (or None when R2 is unavailable/disabled).
+
+    Auto-detects R2 credentials in the environment. ``--no-r2`` disables
+    uploads regardless of env vars. Without an uploader, gated-tier
+    artifacts are SKIPPED at publish time — they are never written to the
+    local static output (fail closed; the local output ships to a public
+    origin with no auth in front of it).
+    """
+    from synthbench.r2_upload import R2Uploader, env_has_r2_config
+
+    if no_r2 or not env_has_r2_config():
+        return None
+    return R2Uploader.from_env()
+
+
+def _warn_gated_skips(n_skipped: int) -> None:
+    """Print the prominent fail-closed warning for skipped gated artifacts."""
+    if not n_skipped:
+        return
+    click.echo(
+        f"WARNING: {n_skipped} gated artifact(s) skipped — R2 uploader not "
+        "configured; gated data will be unavailable until credentials are "
+        "provided (R2_ACCOUNT_ID/R2_ACCESS_KEY_ID/R2_SECRET_ACCESS_KEY/"
+        "R2_BUCKET) and the publish is re-run. Gated artifacts are never "
+        "written to the local static output. Pass --strict-gating (or set "
+        "SYNTHBENCH_PUBLISH_STRICT=1) to fail instead of skipping.",
+        err=True,
+    )
+
+
+_STRICT_GATING_OPTION = click.option(
+    "--strict-gating",
+    is_flag=True,
+    default=False,
+    envvar="SYNTHBENCH_PUBLISH_STRICT",
+    show_envvar=True,
+    help=(
+        "Fail (exit 1) instead of skipping when gated-tier artifacts have no "
+        "R2 uploader configured. CI deploys set SYNTHBENCH_PUBLISH_STRICT=1 "
+        "so a missing R2 configuration fails the build loudly instead of "
+        "deploying without gated data."
+    ),
+)
+
+
+@main.command("publish-runs")
+@click.option(
+    "--results-dir",
+    "-d",
+    type=click.Path(exists=True),
+    default="leaderboard-results",
+    help="Directory containing result JSON files.",
+)
+@click.option(
+    "--output-dir",
+    "-o",
+    type=click.Path(),
+    default="site/public/data",
+    help=(
+        "Output directory for run-explorer artifacts. Emits "
+        "runs-index.json + config/ + run/ subtrees."
+    ),
+)
+@click.option(
+    "--no-r2",
+    is_flag=True,
+    default=False,
+    help=(
+        "Disable R2 uploads even when R2 env vars are set. Gated-tier "
+        "per-run/config/question JSONs are then SKIPPED — they are never "
+        "written to the local static output (fail closed). Default behavior "
+        "uploads gated artifacts to Cloudflare R2 when R2_ACCOUNT_ID/"
+        "R2_ACCESS_KEY_ID/R2_SECRET_ACCESS_KEY/R2_BUCKET are all present "
+        "(sb-sjs)."
+    ),
+)
+@_STRICT_GATING_OPTION
+def publish_runs_cmd(results_dir, output_dir, no_r2, strict_gating):
+    """Emit run-explorer artifacts (runs-index, per-config, per-run JSON).
+
+    Also emits per-question rollups under ``<output-dir>/question/`` for the
+    question-explorer view (sb-eiv) — the two pivots share the same source
+    data, so regenerating runs without questions would leave /question
+    pages stale.
+
+    Example:
+        synthbench publish-runs --results-dir ./leaderboard-results --output-dir site/public/data
+    """
+    from synthbench.publish import publish_questions, publish_runs
+
+    try:
+        r2_uploader = _resolve_r2_uploader(no_r2)
+        counts = publish_runs(
+            results_dir=Path(results_dir),
+            output_dir=Path(output_dir),
+            version=__version__,
+            r2_uploader=r2_uploader,
+            strict_gating=strict_gating,
+        )
+        sink = (
+            f"R2 bucket {r2_uploader.bucket} (gated) + {output_dir} (public)"
+            if r2_uploader is not None
+            else str(output_dir)
+        )
+        click.echo(
+            f"Run explorer data exported: {counts['runs']} runs, "
+            f"{counts['configs']} configs → {sink}"
+        )
+        q_counts = publish_questions(
+            results_dir=Path(results_dir),
+            output_dir=Path(output_dir),
+            version=__version__,
+            r2_uploader=r2_uploader,
+            strict_gating=strict_gating,
+        )
+        click.echo(
+            f"Question explorer data exported: {q_counts['questions']} questions "
+            f"across {q_counts['datasets']} datasets → {sink}"
+        )
+        if r2_uploader is not None:
+            click.echo(
+                f"R2 upload summary: {r2_uploader.object_count} objects to "
+                f"{r2_uploader.bucket}"
+            )
+        _warn_gated_skips(
+            counts.get("gated_skipped", 0) + q_counts.get("gated_skipped", 0)
+        )
+    except ValueError as e:
+        click.echo(str(e), err=True)
+        sys.exit(1)
+
+
+@main.command("scan-invalid")
+@click.option(
+    "--results-dir",
+    "-d",
+    type=click.Path(exists=True),
+    default="leaderboard-results",
+    help="Directory containing result JSON files to scan.",
+)
+@click.option(
+    "--json",
+    "json_output",
+    is_flag=True,
+    default=False,
+    help="Emit machine-readable JSON instead of a human-readable table.",
+)
+@click.option(
+    "--quarantine",
+    type=click.Path(),
+    default=None,
+    help=(
+        "Move detected invalid run files into this directory (created if "
+        "needed). Operator action; non-default. Default behaviour is to "
+        "report only."
+    ),
+)
+def scan_invalid(results_dir, json_output, quarantine):
+    """Scan result JSON files for invalid runs (uniform-distribution garbage).
+
+    Reports runs whose per-question ``model_distribution`` fields are
+    overwhelmingly uniform — the signature of a silent API failure or
+    provider fallback that parsed cleanly but carries no real signal.
+
+    Examples:
+        synthbench scan-invalid --results-dir leaderboard-results
+        synthbench scan-invalid --json | jq
+        synthbench scan-invalid --quarantine leaderboard-results/_quarantine
+    """
+    from synthbench.run_validity import is_invalid_run, run_identity
+
+    import shutil
+
+    results_path = Path(results_dir)
+    json_files = sorted(results_path.glob("*.json"))
+    invalid_records: list[dict] = []
+    scanned = 0
+
+    for jf in json_files:
+        try:
+            with open(jf) as f:
+                data = json.load(f)
+        except (OSError, json.JSONDecodeError):
+            continue
+        if data.get("benchmark") != "synthbench":
+            continue
+        scanned += 1
+        invalid, reason, metrics = is_invalid_run(data)
+        if not invalid:
+            continue
+        invalid_records.append(
+            {
+                "run_id": jf.stem,
+                "path": str(jf),
+                "reason": reason,
+                **run_identity(data),
+                "metrics": metrics,
+            }
+        )
+
+    quarantine_dir = Path(quarantine) if quarantine else None
+    moved: list[str] = []
+    if quarantine_dir and invalid_records:
+        quarantine_dir.mkdir(parents=True, exist_ok=True)
+        for rec in invalid_records:
+            src = Path(rec["path"])
+            dst = quarantine_dir / src.name
+            shutil.move(str(src), str(dst))
+            rec["quarantined_to"] = str(dst)
+            moved.append(str(dst))
+
+    if json_output:
+        click.echo(
+            json.dumps(
+                {
+                    "results_dir": str(results_path),
+                    "n_scanned": scanned,
+                    "n_invalid": len(invalid_records),
+                    "quarantine_dir": str(quarantine_dir) if quarantine_dir else None,
+                    "invalid_runs": invalid_records,
+                },
+                indent=2,
+            )
+        )
+        return
+
+    if not invalid_records:
+        click.echo(
+            f"Scanned {scanned} runs in {results_path} — no invalid runs detected."
+        )
+        return
+
+    click.echo(
+        f"Scanned {scanned} runs in {results_path}. Detected "
+        f"{len(invalid_records)} invalid run(s):"
+    )
+    click.echo("")
+    for rec in invalid_records:
+        m = rec["metrics"]
+        click.echo(f"  ✗ {rec['run_id']}")
+        click.echo(f"      provider : {rec.get('provider')}")
+        click.echo(f"      dataset  : {rec.get('dataset')}")
+        click.echo(
+            f"      uniform  : {m['n_uniform_questions']}/{m['n_questions']} "
+            f"({m['uniform_fraction']:.1%})"
+        )
+        click.echo(f"      refusal  : {m['refusal_rate']:.4f}")
+        click.echo(f"      reason   : {rec['reason']}")
+        if "quarantined_to" in rec:
+            click.echo(f"      moved to : {rec['quarantined_to']}")
+        click.echo("")
+
+    if quarantine_dir:
+        click.echo(f"Moved {len(moved)} file(s) to {quarantine_dir}")
+
+
+@main.command("publish-questions")
+@click.option(
+    "--results-dir",
+    "-d",
+    type=click.Path(exists=True),
+    default="leaderboard-results",
+    help="Directory containing result JSON files.",
+)
+@click.option(
+    "--output-dir",
+    "-o",
+    type=click.Path(),
+    default="site/public/data",
+    help=(
+        "Output directory root. Emits question/<dataset>/<key>.json + "
+        "question/<dataset>/index.json beneath this path."
+    ),
+)
+@click.option(
+    "--no-r2",
+    is_flag=True,
+    default=False,
+    help=(
+        "Disable R2 uploads even when R2 env vars are set. Gated-tier "
+        "artifacts are then SKIPPED, never written locally (fail closed; "
+        "sb-sjs). See `synthbench publish-runs --help` for details."
+    ),
+)
+@_STRICT_GATING_OPTION
+def publish_questions_cmd(results_dir, output_dir, no_r2, strict_gating):
+    """Emit per-question rollups for the /question explorer view (sb-eiv).
+
+    Example:
+        synthbench publish-questions --results-dir ./leaderboard-results --output-dir site/public/data
+    """
+    from synthbench.publish import publish_questions
+
+    try:
+        r2_uploader = _resolve_r2_uploader(no_r2)
+        counts = publish_questions(
+            results_dir=Path(results_dir),
+            output_dir=Path(output_dir),
+            version=__version__,
+            r2_uploader=r2_uploader,
+            strict_gating=strict_gating,
+        )
+        sink = (
+            f"R2 bucket {r2_uploader.bucket} (gated) + {output_dir} (public)"
+            if r2_uploader is not None
+            else f"{output_dir}/question"
+        )
+        click.echo(
+            f"Question explorer data exported: {counts['questions']} questions "
+            f"across {counts['datasets']} datasets → {sink}"
+        )
+        if r2_uploader is not None:
+            click.echo(
+                f"R2 upload summary: {r2_uploader.object_count} objects to "
+                f"{r2_uploader.bucket}"
+            )
+        _warn_gated_skips(counts.get("gated_skipped", 0))
+    except ValueError as e:
+        click.echo(str(e), err=True)
+        sys.exit(1)
+
+
+@main.command()
+@click.argument("files", nargs=-1, required=True, type=click.Path(exists=True))
+@click.option(
+    "--output", "-o", type=click.Path(), default="results", help="Output directory."
+)
+@click.option(
+    "--weights",
+    default=None,
+    help="Comma-separated weights for each input file (default: equal).",
+)
+def ensemble(files, output, weights):
+    """Blend N result files into a single ensemble result.
+
+    Aligns questions by key (intersection), averages model distributions
+    with equal (or custom) weights, and recomputes all metrics.
+
+    Example:
+        synthbench ensemble results/haiku.json results/gemini.json results/gpt4omini.json
+        synthbench ensemble a.json b.json --weights 0.6,0.4
+    """
+    from datetime import datetime, timezone
+    from synthbench.metrics.distributional import jensen_shannon_divergence
+    from synthbench.metrics.ranking import kendall_tau_b
+    from synthbench.metrics.composite import parity_score
+
+    if len(files) < 2:
+        click.echo("Error: need at least 2 result files to ensemble.", err=True)
+        sys.exit(1)
+
+    # Parse weights
+    if weights:
+        w = [float(x) for x in weights.split(",")]
+        if len(w) != len(files):
+            click.echo(f"Error: got {len(w)} weights for {len(files)} files.", err=True)
+            sys.exit(1)
+        w_total = sum(w)
+        w = [x / w_total for x in w]  # normalize
+    else:
+        w = [1.0 / len(files)] * len(files)
+
+    # Load all result files
+    datasets = []
+    source_providers = []
+    for fpath in files:
+        with open(fpath) as f:
+            data = json.load(f)
+        datasets.append(data)
+        source_providers.append(
+            data.get("config", {}).get("provider", Path(fpath).stem)
+        )
+
+    # Index per-question data by key for each file
+    per_q_maps = []
+    for data in datasets:
+        pq = {q["key"]: q for q in data.get("per_question", [])}
+        per_q_maps.append(pq)
+
+    # Intersection of question keys across all files
+    common_keys = set(per_q_maps[0].keys())
+    for pq in per_q_maps[1:]:
+        common_keys &= set(pq.keys())
+    common_keys = sorted(common_keys)
+
+    if not common_keys:
+        click.echo("Error: no common questions across all input files.", err=True)
+        sys.exit(1)
+
+    click.echo(
+        f"Ensembling {len(files)} results over {len(common_keys)} common questions"
+    )
+
+    # Committed gated-dataset result files are stripped of human_distribution
+    # (issue #308); rehydrate from the canonical registry so blends can still
+    # be computed from committed inputs. Fresh (unstripped) run outputs skip
+    # this entirely.
+    ensemble_dataset = datasets[0].get("config", {}).get("dataset", "unknown")
+    canonical_dists: dict = {}
+    if any(
+        not q.get("human_distribution")
+        for pq in per_q_maps
+        for q in (pq[k] for k in common_keys)
+    ):
+        from synthbench.human_distributions import load_canonical_distributions
+
+        canonical_dists = load_canonical_distributions(ensemble_dataset) or {}
+        if not canonical_dists:
+            click.echo(
+                f"Error: input files are missing human_distribution (stripped "
+                f"gated data) and no canonical distribution source is "
+                f"available for dataset {ensemble_dataset!r}. See "
+                f"scripts/generate-canonical-distributions.py.",
+                err=True,
+            )
+            sys.exit(1)
+
+    # Blend distributions and recompute metrics
+    per_question = []
+    for key in common_keys:
+        qs = [pq[key] for pq in per_q_maps]
+        ref_q = qs[0]
+
+        # Human distribution — canonical when the committed input was
+        # stripped; otherwise from the first file (should be identical).
+        human_dist = ref_q.get("human_distribution") or canonical_dists.get(key)
+        if not human_dist:
+            click.echo(
+                f"Error: no human_distribution available for question "
+                f"{key!r} (not in canonical registry).",
+                err=True,
+            )
+            sys.exit(1)
+
+        # Blend model distributions with weights
+        all_option_keys = set()
+        for q in qs:
+            all_option_keys.update(q["model_distribution"].keys())
+
+        blended = {}
+        for opt in all_option_keys:
+            blended[opt] = sum(
+                w[i] * qs[i]["model_distribution"].get(opt, 0.0) for i in range(len(qs))
+            )
+
+        # Round distributions FIRST, then compute metrics from the rounded
+        # values — the file publishes rounded distributions, and the
+        # validator recomputes metrics from what is published. Metrics from
+        # unrounded values can flip Kendall-tau tie handling when two
+        # blended options round to the same 4-dp value (PER_Q_TAU errors on
+        # an otherwise-correct blend).
+        human_rounded = {k: round(v, 4) for k, v in human_dist.items()}
+        blended_rounded = {k: round(v, 4) for k, v in blended.items()}
+
+        # Recompute metrics
+        jsd = jensen_shannon_divergence(human_rounded, blended_rounded)
+        tau = kendall_tau_b(human_rounded, blended_rounded)
+        parity = parity_score(jsd, tau)
+
+        per_question.append(
+            {
+                "key": key,
+                "text": ref_q.get("text", ""),
+                "options": ref_q.get("options", []),
+                "human_distribution": human_rounded,
+                "model_distribution": blended_rounded,
+                "jsd": round(jsd, 6),
+                "kendall_tau": round(tau, 6),
+                "parity": round(parity, 6),
+                "n_samples": sum(q.get("n_samples", 0) for q in qs),
+                "n_parse_failures": sum(q.get("n_parse_failures", 0) for q in qs),
+                "model_refusal_rate": round(
+                    sum(
+                        w[i] * qs[i].get("model_refusal_rate", 0.0)
+                        for i in range(len(qs))
+                    ),
+                    6,
+                ),
+                "human_refusal_rate": ref_q.get("human_refusal_rate", 0.0),
+                "temporal_year": ref_q.get("temporal_year", 0),
+            }
+        )
+
+    # Aggregate metrics
+    n_q = len(per_question)
+    mean_jsd = sum(q["jsd"] for q in per_question) / n_q
+    mean_tau = sum(q["kendall_tau"] for q in per_question) / n_q
+    p_dist = round(1.0 - mean_jsd, 6)
+    p_rank = round((1.0 + mean_tau) / 2.0, 6)
+
+    # P_refuse from blended refusal rates
+    from synthbench.metrics.refusal import refusal_calibration
+
+    p_refuse = round(
+        refusal_calibration(
+            [q["model_refusal_rate"] for q in per_question],
+            [q["human_refusal_rate"] for q in per_question],
+        ),
+        6,
+    )
+
+    # SPS = mean of all available metrics
+    sps = round((p_dist + p_rank + p_refuse) / 3.0, 6)
+
+    # Build result JSON
+    provider_label = f"ensemble/{len(files)}-model-blend"
+    dataset_name = datasets[0].get("config", {}).get("dataset", "unknown")
+
+    result_json = {
+        "benchmark": "synthbench",
+        "version": __version__,
+        "timestamp": datetime.now(timezone.utc).isoformat(),
+        "config": {
+            "dataset": dataset_name,
+            "provider": provider_label,
+            "ensemble_sources": [
+                {"file": str(Path(f).name), "provider": sp, "weight": round(w[i], 6)}
+                for i, (f, sp) in enumerate(zip(files, source_providers))
+            ],
+            "n_common_questions": len(common_keys),
+            # The blend is evaluated on exactly the common-question
+            # intersection. Without this, publish renders ensemble rows as
+            # "scored on zero questions" (n: 0).
+            "n_evaluated": len(common_keys),
+        },
+        "scores": {
+            "sps": sps,
+            "p_dist": p_dist,
+            "p_rank": p_rank,
+            "p_refuse": p_refuse,
+        },
+        "aggregate": {
+            "mean_jsd": round(mean_jsd, 6),
+            "median_jsd": round(sorted(q["jsd"] for q in per_question)[n_q // 2], 6),
+            "mean_kendall_tau": round(mean_tau, 6),
+            "composite_parity": sps,
+            "n_questions": n_q,
+            "elapsed_seconds": 0.0,
+        },
+        "per_question": per_question,
+    }
+
+    # Summary output
+    click.echo(f"  Provider: {provider_label}")
+    click.echo(f"  P_dist:   {p_dist:.4f}")
+    click.echo(f"  P_rank:   {p_rank:.4f}")
+    click.echo(f"  P_refuse: {p_refuse:.4f}")
+    click.echo(f"  SPS:      {sps:.4f}")
+    click.echo()
+
+    # Save
+    out_dir = Path(output)
+    out_dir.mkdir(parents=True, exist_ok=True)
+    ts = datetime.now().strftime("%Y%m%d_%H%M%S")
+    json_path = out_dir / f"{dataset_name}_ensemble_{len(files)}blend_{ts}.json"
+    json_path.write_text(json.dumps(result_json, indent=2))
+    click.echo(f"Saved: {json_path}")
+
+
+@main.command()
+@click.argument("paths", nargs=-1, required=True, type=click.Path())
+@click.option(
+    "--expected-question-hash",
+    default=None,
+    help="Canonical question_set_hash the submission must match.",
+)
+@click.option(
+    "--skip-recompute",
+    is_flag=True,
+    help="Run only tier 1 (schema + plausibility); skip recomputation.",
+)
+@click.option(
+    "--tier3",
+    is_flag=True,
+    help=(
+        "Run Tier-3 checks: statistical anomaly detection, raw-response "
+        "samples, and reproducibility metadata (warnings by default)."
+    ),
+)
+@click.option(
+    "--peers",
+    "peers_dir",
+    type=click.Path(exists=True, file_okay=False),
+    default=None,
+    help=(
+        "Directory of peer submission JSONs used by the Tier-3 peer-"
+        "distribution outlier detector (same-family, same-dataset runs)."
+    ),
+)
+@click.option(
+    "--strict",
+    is_flag=True,
+    help="Treat warnings as errors (exit non-zero).",
+)
+@click.option(
+    "--rehydrate-canonical",
+    is_flag=True,
+    default=False,
+    help=(
+        "Recompute tier-2 metrics against the CANONICAL per-question human "
+        "distributions (local artifact, gated R2 object, or dataset adapter "
+        "cache — see synthbench.human_distributions) instead of any "
+        "submitter-supplied copies. Required to fully validate committed "
+        "gated-dataset files, which are stripped of human_distribution "
+        "(issue #308); without it those files validate with an explicit "
+        "RECOMPUTE_SKIPPED_NO_DISTRIBUTION warning."
+    ),
+)
+@click.option(
+    "--json",
+    "json_output",
+    is_flag=True,
+    help="Emit a JSON summary instead of a human-readable report.",
+)
+def validate(
+    paths,
+    expected_question_hash,
+    skip_recompute,
+    tier3,
+    peers_dir,
+    strict,
+    rehydrate_canonical,
+    json_output,
+):
+    """Validate one or more submission result JSONs against the integrity rules.
+
+    Tier 1 checks: JSON schema shape, SPS/score bounds, distribution sums,
+    question-set hash, record-count cross-checks, parse-failure plausibility.
+
+    Tier 2 checks: recompute per-question JSD + Kendall's tau from the
+    reported distributions, recompute aggregate mean/composite parity,
+    and flag mismatches.
+
+    Exits non-zero if any submission has validation errors (or warnings
+    under --strict).
+    """
+
+    from synthbench.validation import validate_file
+
+    reports = []
+    exit_code = 0
+
+    def glob_paths():
+        for p in paths:
+            pth = Path(p)
+            if pth.is_dir():
+                yield from sorted(pth.glob("*.json"))
+            else:
+                yield pth
+
+    targets = list(glob_paths())
+    if not targets:
+        click.echo("No files to validate.", err=True)
+        sys.exit(2)
+
+    peer_data: list[dict] = []
+    if peers_dir is not None:
+        peers_path = Path(peers_dir)
+        for jf in sorted(peers_path.glob("*.json")):
+            try:
+                peer_data.append(json.loads(jf.read_text()))
+            except (OSError, json.JSONDecodeError):
+                continue
+
+    def _canonical_for(target: Path):
+        """Resolve canonical distributions for a file's dataset (or None)."""
+        if not rehydrate_canonical:
+            return None
+        from synthbench.human_distributions import load_canonical_distributions
+
+        try:
+            dataset = (
+                json.loads(Path(target).read_text()).get("config", {}) or {}
+            ).get("dataset")
+        except (OSError, json.JSONDecodeError):
+            return None
+        if not dataset:
+            return None
+        canonical = load_canonical_distributions(dataset)
+        if canonical is None:
+            click.echo(
+                f"WARNING: --rehydrate-canonical: no canonical distribution "
+                f"source available for dataset {dataset!r} "
+                f"({target}) — falling back to submitted distributions.",
+                err=True,
+            )
+        return canonical
+
+    for target in targets:
+        report = validate_file(
+            target,
+            expected_question_hash=expected_question_hash,
+            tier2=not skip_recompute,
+            tier3=tier3,
+            peers=peer_data,
+            canonical_distributions=_canonical_for(target),
+        )
+        reports.append(report)
+        has_errors = bool(report.errors)
+        has_warnings = bool(report.warnings)
+        if has_errors or (strict and has_warnings):
+            exit_code = 1
+
+    if json_output:
+        payload = [
+            {
+                "source": r.source,
+                "ok": r.ok and (not strict or not r.warnings),
+                "metadata": r.metadata,
+                "issues": [
+                    {
+                        "code": i.code,
+                        "severity": i.severity.value,
+                        "path": i.path,
+                        "message": i.message,
+                    }
+                    for i in r.issues
+                ],
+            }
+            for r in reports
+        ]
+        click.echo(json.dumps(payload, indent=2))
+    else:
+        total_errors = 0
+        total_warnings = 0
+        for r in reports:
+            click.echo(r.format())
+            total_errors += len(r.errors)
+            total_warnings += len(r.warnings)
+        click.echo("")
+        click.echo(
+            f"Summary: {len(reports)} file(s), "
+            f"{total_errors} error(s), {total_warnings} warning(s)."
+        )
+
+    sys.exit(exit_code)
+
+
+if __name__ == "__main__":
+    main()