synthbench-eval 0.6.1__tar.gz → 0.6.2__tar.gz

{synthbench_eval-0.6.1 → synthbench_eval-0.6.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: synthbench-eval
-Version: 0.6.1
+Version: 0.6.2
 Summary: Open benchmark harness for synthetic survey respondent quality
 License-Expression: MIT
 Keywords: benchmark,survey,synthetic-data,llm,evaluation,opinionsqa,globalopinionqa,gss,convergence,althing

{synthbench_eval-0.6.1 → synthbench_eval-0.6.2}/pyproject.toml

@@ -8,7 +8,7 @@ build-backend = "setuptools.build_meta"
 name = "synthbench-eval"
 # Placeholder — the publish job stamps the real version from the release tag
 # at build time (see publish-pypi in .github/workflows/auto-tag.yml).
-version = "0.6.1"
+version = "0.6.2"
 description = "Open benchmark harness for synthetic survey respondent quality"
 readme = "README.md"
 license = "MIT"

{synthbench_eval-0.6.1 → synthbench_eval-0.6.2}/src/synthbench/__init__.py

@@ -6,7 +6,7 @@ from synthbench.convergence.baseline import (
     load_convergence_baseline,
 )
 
-__version__ = "0.6.1"
+__version__ = "0.6.2"
 
 __all__ = [
     "BaselineGatedError",

{synthbench_eval-0.6.1 → synthbench_eval-0.6.2}/src/synthbench/cli.py

@@ -390,7 +390,7 @@ def run(
         )
         sys.exit(2)
 
-    asyncio.run(
+    _run_benchmark_or_exit(
         _run_async(
             provider,
             model,
@@ -423,6 +423,17 @@ def run(
     )
 
 
+def _run_benchmark_or_exit(coro) -> None:
+    """Run a benchmark coroutine, turning an empty question set into a clean CLI error."""
+    from synthbench.runner import EmptyQuestionSetError
+
+    try:
+        asyncio.run(coro)
+    except EmptyQuestionSetError as exc:
+        click.echo(f"Error: {exc}", err=True)
+        sys.exit(2)
+
+
 async def _run_async(
     provider_name,
     model,
@@ -1147,7 +1158,7 @@ def replicate(
     Example:
         synthbench replicate --provider raw-anthropic --n-runs 5 --suite core
     """
-    asyncio.run(
+    _run_benchmark_or_exit(
         _replicate_async(
             provider,
             model,

{synthbench_eval-0.6.1 → synthbench_eval-0.6.2}/src/synthbench/runner.py

@@ -26,6 +26,33 @@ from synthbench.providers.base import Distribution, PersonaSpec, Provider, Respo
 from synthbench.stats import bootstrap_ci, question_set_hash
 
 
+class EmptyQuestionSetError(ValueError):
+    """Raised when a run would evaluate zero questions.
+
+    Scoring an empty set yields empty-input defaults that look like a real
+    SPS (synthbench#353), so the runner refuses instead.
+    """
+
+
+def _empty_question_set_message(
+    dataset_name: str, *, loaded: int, filtered: bool
+) -> str:
+    if not filtered:
+        return f"Dataset '{dataset_name}' loaded 0 questions; nothing to evaluate."
+    from synthbench.suites import SUITE_SOURCE_DATASET
+
+    msg = (
+        f"The --suite/--topic filter matched 0 of {loaded} questions in dataset "
+        f"'{dataset_name}'; nothing to evaluate."
+    )
+    if dataset_name != SUITE_SOURCE_DATASET:
+        msg += (
+            f" Pinned suites and topics are built from '{SUITE_SOURCE_DATASET}' "
+            "question keys — use --n to size a run on this dataset."
+        )
+    return msg
+
+
 def _sha256_of(s: str) -> str:
     """Return ``sha256:<hex>`` digest of a UTF-8 string."""
     return "sha256:" + hashlib.sha256(s.encode("utf-8")).hexdigest()
@@ -405,6 +432,7 @@ class BenchmarkRunner:
     ) -> BenchmarkResult:
         t0 = time.monotonic()
         questions = self.dataset.load(n=n)
+        loaded = len(questions)
 
         # Filter by pinned question set if provided
         if question_keys is not None:
@@ -412,6 +440,13 @@ class BenchmarkRunner:
 
             questions = filter_questions_by_suite(questions, question_keys)
 
+        if not questions:
+            raise EmptyQuestionSetError(
+                _empty_question_set_message(
+                    self.dataset.name, loaded=loaded, filtered=question_keys is not None
+                )
+            )
+
         # Use batched evaluation when provider supports it
         use_batch = self.provider.supports_distribution and hasattr(
             self.provider, "batch_get_distribution"

{synthbench_eval-0.6.1 → synthbench_eval-0.6.2}/src/synthbench/suites/__init__.py

@@ -25,6 +25,10 @@ AVAILABLE_SUITES = ("smoke", "core", "full")
 
 AVAILABLE_TOPICS = ("political", "consumer", "neutral")
 
+# Every pinned suite and topic set above is built from OpinionsQA question
+# keys (``<VAR>_W<wave>``), so against any other dataset they match nothing.
+SUITE_SOURCE_DATASET = "opinionsqa"
+
 
 def load_suite(name: str) -> list[str] | None:
     """Load a pinned question set by name.

{synthbench_eval-0.6.1 → synthbench_eval-0.6.2}/src/synthbench/validation.py

@@ -649,6 +649,17 @@ def _validate_counts(data: Mapping[str, Any]) -> list[Issue]:
     per_question = data.get("per_question") or []
     reported = aggregate.get("n_questions")
     actual = len(per_question) if isinstance(per_question, list) else 0
+    if isinstance(per_question, list) and actual == 0:
+        # A zero-question run has no evidence behind it; its scores are
+        # empty-input defaults (synthbench#353), so never accept one.
+        issues.append(
+            Issue(
+                code="EMPTY_RUN",
+                severity=Severity.ERROR,
+                message="per_question is empty — the run evaluated zero questions",
+                path="per_question",
+            )
+        )
     if isinstance(reported, int) and reported != actual:
         issues.append(
             Issue(

{synthbench_eval-0.6.1 → synthbench_eval-0.6.2}/src/synthbench_eval.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: synthbench-eval
-Version: 0.6.1
+Version: 0.6.2
 Summary: Open benchmark harness for synthetic survey respondent quality
 License-Expression: MIT
 Keywords: benchmark,survey,synthetic-data,llm,evaluation,opinionsqa,globalopinionqa,gss,convergence,althing

{synthbench_eval-0.6.1 → synthbench_eval-0.6.2}/tests/test_runner.py

@@ -14,6 +14,7 @@ from synthbench.providers.base import (
 from synthbench.runner import (
     BenchmarkRunner,
     DemographicGroupResult,
+    EmptyQuestionSetError,
     _aggregate_token_usage,
     _normalize_model_dist,
 )
@@ -660,3 +661,40 @@ async def test_runner_records_per_question_latency(mock_dataset, mock_provider):
         assert qr.latency_seconds >= 0.0
         # Per-question latency cannot exceed the total run elapsed time.
         assert qr.latency_seconds <= result.elapsed_seconds + 1e-3
+
+
+@pytest.mark.asyncio
+async def test_runner_refuses_suite_that_matches_no_questions(
+    mock_dataset, mock_provider
+):
+    """synthbench#353: a filter that matches nothing must not yield a score."""
+    runner = BenchmarkRunner(
+        dataset=mock_dataset, provider=mock_provider, samples_per_question=2
+    )
+    with pytest.raises(EmptyQuestionSetError) as exc:
+        await runner.run(question_keys=["BIOTECHC_W34", "DIFF1B_W29"])
+    msg = str(exc.value)
+    assert "matched 0 of" in msg
+    assert "'mock'" in msg
+    assert "opinionsqa" in msg  # points at the dataset the suites come from
+    assert "--n" in msg
+
+
+@pytest.mark.asyncio
+async def test_runner_refuses_empty_dataset(mock_provider):
+    class EmptyDataset(Dataset):
+        @property
+        def name(self) -> str:
+            return "empty"
+
+        def load(self, n=None):
+            return []
+
+        def info(self) -> dict:
+            return {}
+
+    runner = BenchmarkRunner(
+        dataset=EmptyDataset(), provider=mock_provider, samples_per_question=2
+    )
+    with pytest.raises(EmptyQuestionSetError, match="loaded 0 questions"):
+        await runner.run()

{synthbench_eval-0.6.1 → synthbench_eval-0.6.2}/tests/test_validation.py

@@ -236,6 +236,16 @@ class TestCountMismatch:
         assert any(i.code == "COUNT_MISMATCH" for i in report.errors)
 
 
+class TestEmptyRun:
+    def test_zero_question_run_rejected(self, clean_submission):
+        bad = copy.deepcopy(clean_submission)
+        bad["per_question"] = []
+        bad["aggregate"]["n_questions"] = 0
+        report = validate_submission(bad, tier2=False)
+        assert not report.ok
+        assert any(i.code == "EMPTY_RUN" for i in report.errors)
+
+
 class TestParseFailurePlausibility:
     """PARSE_SUSPICIOUS was retired in sb-a613.