synthbench-eval 0.6.0__tar.gz → 0.6.2__tar.gz

{synthbench_eval-0.6.0 → synthbench_eval-0.6.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: synthbench-eval
-Version: 0.6.0
+Version: 0.6.2
 Summary: Open benchmark harness for synthetic survey respondent quality
 License-Expression: MIT
 Keywords: benchmark,survey,synthetic-data,llm,evaluation,opinionsqa,globalopinionqa,gss,convergence,althing

{synthbench_eval-0.6.0 → synthbench_eval-0.6.2}/pyproject.toml

@@ -8,7 +8,7 @@ build-backend = "setuptools.build_meta"
 name = "synthbench-eval"
 # Placeholder — the publish job stamps the real version from the release tag
 # at build time (see publish-pypi in .github/workflows/auto-tag.yml).
-version = "0.6.0"
+version = "0.6.2"
 description = "Open benchmark harness for synthetic survey respondent quality"
 readme = "README.md"
 license = "MIT"

{synthbench_eval-0.6.0 → synthbench_eval-0.6.2}/src/synthbench/__init__.py

@@ -6,7 +6,7 @@ from synthbench.convergence.baseline import (
     load_convergence_baseline,
 )
 
-__version__ = "0.6.0"
+__version__ = "0.6.2"
 
 __all__ = [
     "BaselineGatedError",

{synthbench_eval-0.6.0 → synthbench_eval-0.6.2}/src/synthbench/cli.py

@@ -390,7 +390,7 @@ def run(
         )
         sys.exit(2)
 
-    asyncio.run(
+    _run_benchmark_or_exit(
         _run_async(
             provider,
             model,
@@ -423,6 +423,17 @@ def run(
     )
 
 
+def _run_benchmark_or_exit(coro) -> None:
+    """Run a benchmark coroutine, turning an empty question set into a clean CLI error."""
+    from synthbench.runner import EmptyQuestionSetError
+
+    try:
+        asyncio.run(coro)
+    except EmptyQuestionSetError as exc:
+        click.echo(f"Error: {exc}", err=True)
+        sys.exit(2)
+
+
 async def _run_async(
     provider_name,
     model,
@@ -1147,7 +1158,7 @@ def replicate(
     Example:
         synthbench replicate --provider raw-anthropic --n-runs 5 --suite core
     """
-    asyncio.run(
+    _run_benchmark_or_exit(
         _replicate_async(
             provider,
             model,
@@ -1284,8 +1295,10 @@ async def _replicate_async(
     out_dir.mkdir(parents=True, exist_ok=True)
     from datetime import datetime
 
+    from synthbench.report import provider_slug as report_provider_slug
+
     ts = datetime.now().strftime("%Y%m%d_%H%M%S")
-    provider_slug = prov.name.replace("/", "_")
+    provider_slug = report_provider_slug(prov.name)
 
     json_data = {
         "benchmark": "synthbench",
@@ -2018,8 +2031,10 @@ async def _contamination_async(
         out_dir.mkdir(parents=True, exist_ok=True)
         from datetime import datetime
 
+        from synthbench.report import provider_slug as report_provider_slug
+
         ts = datetime.now().strftime("%Y%m%d_%H%M%S")
-        provider_slug = prov.name.replace("/", "_")
+        provider_slug = report_provider_slug(prov.name)
         json_path = out_dir / f"contamination_{provider_slug}_{ts}.json"
         json_path.write_text(json.dumps(result_data, indent=2))
         click.echo(f"Results saved: {json_path}")
@@ -2152,8 +2167,10 @@ async def _contamination_deident_async(
         out_dir.mkdir(parents=True, exist_ok=True)
         from datetime import datetime
 
+        from synthbench.report import provider_slug as report_provider_slug
+
         ts = datetime.now().strftime("%Y%m%d_%H%M%S")
-        provider_slug = prov.name.replace("/", "_")
+        provider_slug = report_provider_slug(prov.name)
         json_path = out_dir / f"contamination_deident_{provider_slug}_{ts}.json"
         json_path.write_text(json.dumps(result_data, indent=2))
         click.echo(f"Results saved: {json_path}")

{synthbench_eval-0.6.0 → synthbench_eval-0.6.2}/src/synthbench/report.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import json
+import re
 from datetime import datetime, timezone
 from pathlib import Path
 
@@ -469,6 +470,18 @@ def to_markdown(
     return "\n".join(lines)
 
 
+# Characters that are path separators or invalid in Windows filenames. A ':'
+# is the dangerous one: NTFS treats ``name:rest`` as an alternate data
+# stream, so a provider like ``althing/claude-code:haiku`` would silently
+# write its score card into a hidden stream of a truncated file.
+_UNSAFE_FILENAME_CHARS = re.compile(r'[\\/:*?"<>|]')
+
+
+def provider_slug(provider_name: str) -> str:
+    """Return *provider_name* made safe for use in a result filename."""
+    return _UNSAFE_FILENAME_CHARS.sub("_", provider_name)
+
+
 def save(result: BenchmarkResult, output_dir: Path | str) -> tuple[Path, Path]:
     """Save JSON and markdown score cards to output_dir.
 
@@ -478,16 +491,17 @@ def save(result: BenchmarkResult, output_dir: Path | str) -> tuple[Path, Path]:
     output_dir.mkdir(parents=True, exist_ok=True)
 
     ts = datetime.now().strftime("%Y%m%d_%H%M%S")
-    provider_slug = result.provider_name.replace("/", "_")
-    base = f"{result.dataset_name}_{provider_slug}_{ts}"
+    base = f"{result.dataset_name}_{provider_slug(result.provider_name)}_{ts}"
 
     json_path = output_dir / f"{base}.json"
     md_path = output_dir / f"{base}.md"
 
-    with open(json_path, "w") as f:
+    # Explicit UTF-8: the markdown card contains box-drawing bars (█░) that
+    # the Windows default codepage (cp1252) cannot encode.
+    with open(json_path, "w", encoding="utf-8") as f:
         json.dump(to_json(result), f, indent=2)
 
-    with open(md_path, "w") as f:
+    with open(md_path, "w", encoding="utf-8") as f:
         f.write(to_markdown(result))
 
     return json_path, md_path

{synthbench_eval-0.6.0 → synthbench_eval-0.6.2}/src/synthbench/runner.py

@@ -26,6 +26,33 @@ from synthbench.providers.base import Distribution, PersonaSpec, Provider, Respo
 from synthbench.stats import bootstrap_ci, question_set_hash
 
 
+class EmptyQuestionSetError(ValueError):
+    """Raised when a run would evaluate zero questions.
+
+    Scoring an empty set yields empty-input defaults that look like a real
+    SPS (synthbench#353), so the runner refuses instead.
+    """
+
+
+def _empty_question_set_message(
+    dataset_name: str, *, loaded: int, filtered: bool
+) -> str:
+    if not filtered:
+        return f"Dataset '{dataset_name}' loaded 0 questions; nothing to evaluate."
+    from synthbench.suites import SUITE_SOURCE_DATASET
+
+    msg = (
+        f"The --suite/--topic filter matched 0 of {loaded} questions in dataset "
+        f"'{dataset_name}'; nothing to evaluate."
+    )
+    if dataset_name != SUITE_SOURCE_DATASET:
+        msg += (
+            f" Pinned suites and topics are built from '{SUITE_SOURCE_DATASET}' "
+            "question keys — use --n to size a run on this dataset."
+        )
+    return msg
+
+
 def _sha256_of(s: str) -> str:
     """Return ``sha256:<hex>`` digest of a UTF-8 string."""
     return "sha256:" + hashlib.sha256(s.encode("utf-8")).hexdigest()
@@ -405,6 +432,7 @@ class BenchmarkRunner:
     ) -> BenchmarkResult:
         t0 = time.monotonic()
         questions = self.dataset.load(n=n)
+        loaded = len(questions)
 
         # Filter by pinned question set if provided
         if question_keys is not None:
@@ -412,6 +440,13 @@ class BenchmarkRunner:
 
             questions = filter_questions_by_suite(questions, question_keys)
 
+        if not questions:
+            raise EmptyQuestionSetError(
+                _empty_question_set_message(
+                    self.dataset.name, loaded=loaded, filtered=question_keys is not None
+                )
+            )
+
         # Use batched evaluation when provider supports it
         use_batch = self.provider.supports_distribution and hasattr(
             self.provider, "batch_get_distribution"

{synthbench_eval-0.6.0 → synthbench_eval-0.6.2}/src/synthbench/suites/__init__.py

@@ -25,6 +25,10 @@ AVAILABLE_SUITES = ("smoke", "core", "full")
 
 AVAILABLE_TOPICS = ("political", "consumer", "neutral")
 
+# Every pinned suite and topic set above is built from OpinionsQA question
+# keys (``<VAR>_W<wave>``), so against any other dataset they match nothing.
+SUITE_SOURCE_DATASET = "opinionsqa"
+
 
 def load_suite(name: str) -> list[str] | None:
     """Load a pinned question set by name.

{synthbench_eval-0.6.0 → synthbench_eval-0.6.2}/src/synthbench/validation.py

@@ -649,6 +649,17 @@ def _validate_counts(data: Mapping[str, Any]) -> list[Issue]:
     per_question = data.get("per_question") or []
     reported = aggregate.get("n_questions")
     actual = len(per_question) if isinstance(per_question, list) else 0
+    if isinstance(per_question, list) and actual == 0:
+        # A zero-question run has no evidence behind it; its scores are
+        # empty-input defaults (synthbench#353), so never accept one.
+        issues.append(
+            Issue(
+                code="EMPTY_RUN",
+                severity=Severity.ERROR,
+                message="per_question is empty — the run evaluated zero questions",
+                path="per_question",
+            )
+        )
     if isinstance(reported, int) and reported != actual:
         issues.append(
             Issue(

{synthbench_eval-0.6.0 → synthbench_eval-0.6.2}/src/synthbench_eval.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: synthbench-eval
-Version: 0.6.0
+Version: 0.6.2
 Summary: Open benchmark harness for synthetic survey respondent quality
 License-Expression: MIT
 Keywords: benchmark,survey,synthetic-data,llm,evaluation,opinionsqa,globalopinionqa,gss,convergence,althing

{synthbench_eval-0.6.0 → synthbench_eval-0.6.2}/tests/test_report.py

@@ -7,7 +7,7 @@ import json
 import pytest
 
 from synthbench.runner import BenchmarkResult, QuestionResult
-from synthbench.report import to_json, to_markdown
+from synthbench.report import provider_slug, save, to_json, to_markdown
 
 
 @pytest.fixture
@@ -160,3 +160,25 @@ def test_latency_appears_in_per_question_records():
     pq = data["per_question"]
     assert pq[0]["latency_seconds"] == pytest.approx(0.42)
     assert "latency_seconds" not in pq[1]
+
+
+@pytest.mark.parametrize(
+    ("name", "expected"),
+    [
+        ("althing/claude-code:haiku", "althing_claude-code_haiku"),
+        ("openrouter/openai/gpt-4o-mini t=0.7", "openrouter_openai_gpt-4o-mini t=0.7"),
+        ('a\\b*c?d"e<f>g|h', "a_b_c_d_e_f_g_h"),
+    ],
+)
+def test_provider_slug_is_filename_safe(name, expected):
+    assert provider_slug(name) == expected
+
+
+def test_save_uses_filename_safe_slug(sample_result, tmp_path):
+    sample_result.provider_name = "althing/claude-code:haiku"
+    json_path, md_path = save(sample_result, tmp_path)
+    assert ":" not in json_path.name
+    assert json_path.name.startswith("test_althing_claude-code_haiku_")
+    card = json.loads(json_path.read_text(encoding="utf-8"))
+    assert card["config"]["provider"] == "althing/claude-code:haiku"
+    assert "█" in md_path.read_text(encoding="utf-8")

{synthbench_eval-0.6.0 → synthbench_eval-0.6.2}/tests/test_runner.py

@@ -14,6 +14,7 @@ from synthbench.providers.base import (
 from synthbench.runner import (
     BenchmarkRunner,
     DemographicGroupResult,
+    EmptyQuestionSetError,
     _aggregate_token_usage,
     _normalize_model_dist,
 )
@@ -660,3 +661,40 @@ async def test_runner_records_per_question_latency(mock_dataset, mock_provider):
         assert qr.latency_seconds >= 0.0
         # Per-question latency cannot exceed the total run elapsed time.
         assert qr.latency_seconds <= result.elapsed_seconds + 1e-3
+
+
+@pytest.mark.asyncio
+async def test_runner_refuses_suite_that_matches_no_questions(
+    mock_dataset, mock_provider
+):
+    """synthbench#353: a filter that matches nothing must not yield a score."""
+    runner = BenchmarkRunner(
+        dataset=mock_dataset, provider=mock_provider, samples_per_question=2
+    )
+    with pytest.raises(EmptyQuestionSetError) as exc:
+        await runner.run(question_keys=["BIOTECHC_W34", "DIFF1B_W29"])
+    msg = str(exc.value)
+    assert "matched 0 of" in msg
+    assert "'mock'" in msg
+    assert "opinionsqa" in msg  # points at the dataset the suites come from
+    assert "--n" in msg
+
+
+@pytest.mark.asyncio
+async def test_runner_refuses_empty_dataset(mock_provider):
+    class EmptyDataset(Dataset):
+        @property
+        def name(self) -> str:
+            return "empty"
+
+        def load(self, n=None):
+            return []
+
+        def info(self) -> dict:
+            return {}
+
+    runner = BenchmarkRunner(
+        dataset=EmptyDataset(), provider=mock_provider, samples_per_question=2
+    )
+    with pytest.raises(EmptyQuestionSetError, match="loaded 0 questions"):
+        await runner.run()

{synthbench_eval-0.6.0 → synthbench_eval-0.6.2}/tests/test_validation.py

@@ -236,6 +236,16 @@ class TestCountMismatch:
         assert any(i.code == "COUNT_MISMATCH" for i in report.errors)
 
 
+class TestEmptyRun:
+    def test_zero_question_run_rejected(self, clean_submission):
+        bad = copy.deepcopy(clean_submission)
+        bad["per_question"] = []
+        bad["aggregate"]["n_questions"] = 0
+        report = validate_submission(bad, tier2=False)
+        assert not report.ok
+        assert any(i.code == "EMPTY_RUN" for i in report.errors)
+
+
 class TestParseFailurePlausibility:
     """PARSE_SUSPICIOUS was retired in sb-a613.