synthbench-eval 0.6.2__tar.gz → 0.7.0__tar.gz

{synthbench_eval-0.6.2 → synthbench_eval-0.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: synthbench-eval
-Version: 0.6.2
+Version: 0.7.0
 Summary: Open benchmark harness for synthetic survey respondent quality
 License-Expression: MIT
 Keywords: benchmark,survey,synthetic-data,llm,evaluation,opinionsqa,globalopinionqa,gss,convergence,althing

{synthbench_eval-0.6.2 → synthbench_eval-0.7.0}/pyproject.toml

@@ -8,7 +8,7 @@ build-backend = "setuptools.build_meta"
 name = "synthbench-eval"
 # Placeholder — the publish job stamps the real version from the release tag
 # at build time (see publish-pypi in .github/workflows/auto-tag.yml).
-version = "0.6.2"
+version = "0.7.0"
 description = "Open benchmark harness for synthetic survey respondent quality"
 readme = "README.md"
 license = "MIT"

{synthbench_eval-0.6.2 → synthbench_eval-0.7.0}/src/synthbench/__init__.py

@@ -6,7 +6,7 @@ from synthbench.convergence.baseline import (
     load_convergence_baseline,
 )
 
-__version__ = "0.6.2"
+__version__ = "0.7.0"
 
 __all__ = [
     "BaselineGatedError",

{synthbench_eval-0.6.2 → synthbench_eval-0.7.0}/src/synthbench/providers/_parsing.py

@@ -28,6 +28,24 @@ This module is the single replacement. Matching order:
 
 Anything else is a parse failure: ``option is None`` and ``refusal`` is
 False. Callers must NOT substitute a default option.
+
+Parser versions (``config.option_parser_version``; absent key = v1):
+
+* **v1** — the matching order above, verbatim. Kept callable so historical
+  runs re-parse bit-for-bit.
+* **v2** (current, synthbench#352) — newer models answer in markdown prose
+  (``**(D) Depends on the situation.**`` followed by an explanation). v1
+  counted those as parse failures, and its whole-text containment could
+  mis-score them by matching an option mentioned in the explanation. v2:
+
+  - strips markdown emphasis/heading/quote markup before matching;
+  - accepts a *leading* option label (``(D) ...``, ``D. ...``, ``D) ...``)
+    as the answer, unless the text right after it exactly names a
+    different option (contradictory → parse failure);
+  - restricts containment to the first line, falling back to the whole
+    response only when exactly one option is contained in it.
+
+  Refusal detection is unchanged (it is versioned separately).
 """
 
 from __future__ import annotations
@@ -47,6 +65,28 @@ _BARE_LETTER_RE = re.compile(r"^\s*\(?([A-Za-z])\)?[.):]?\s*$")
 
 _WS_RE = re.compile(r"\s+")
 
+#: Parser version stamped into run metadata (``config.option_parser_version``).
+#: Absent key on a committed run file means v1.
+OPTION_PARSER_VERSION = 2
+
+# v2: markdown markup to drop before matching — emphasis runs (``**``,
+# ``__``, ``*``), and line-leading heading / blockquote / bullet markers.
+_MD_EMPHASIS_RE = re.compile(r"(\*{1,3}|_{2,3})")
+_MD_LINE_PREFIX_RE = re.compile(
+    r"^[ \t]*(?:#{1,6}[ \t]+|>[ \t]*|[-+][ \t]+)", re.MULTILINE
+)
+
+# v2: leading option label, optionally after an "Answer:" style prefix.
+# Parenthesized letters may be either case; bare letters must be uppercase
+# and delimited (checked in _leading_label_match) so "I think..." or
+# "A lot of people..." never parse as a label.
+_LEADING_LABEL_RE = re.compile(
+    r"^\s*(?:(?:answer|my answer|choice)\s*[:\-]\s*)?"
+    r"(?:\(([A-Za-z])\)|([A-Za-z])[.):](?=\s|$))\s*(.*)",
+    re.IGNORECASE | re.DOTALL,
+)
+_PAREN_LABEL_RE = re.compile(r"\(([A-Za-z])\)")
+
 
 @dataclass(frozen=True)
 class ParsedResponse:
@@ -96,11 +136,91 @@ def _containment_match(text: str, options: list[str]) -> str | None:
     return None
 
 
+def _strip_markdown(text: str) -> str:
+    """Drop markdown emphasis and line-leading markup (v2)."""
+    return _MD_EMPHASIS_RE.sub("", _MD_LINE_PREFIX_RE.sub("", text))
+
+
+def _leading_label_match(text: str, options: list[str]) -> ParsedResponse | None:
+    """Resolve a leading ``(D) ...`` / ``D. ...`` label (v2).
+
+    Returns ``None`` when there is no label; a parse failure when the label
+    is out of range or contradicted by an exact option name right after it.
+    The lowercase bare form (``a. ...``) is not a label — only ``(a)``.
+    """
+    match = _LEADING_LABEL_RE.match(text)
+    if not match:
+        return None
+    paren, bare, rest = match.groups()
+    if bare is not None and not bare.isupper():
+        return None
+    letter = (paren or bare).upper()
+    idx = ord(letter) - ord("A")
+    if not 0 <= idx < len(options):
+        return ParsedResponse()
+    # "(B) Favor" where B is "Oppose": the label and the echoed text
+    # disagree, so neither can be trusted.
+    first_line = rest.strip().split("\n", 1)[0]
+    by_norm = {_normalize(opt): opt for opt in options}
+    echoed = by_norm.get(_normalize(first_line))
+    if echoed is not None and echoed != options[idx]:
+        return ParsedResponse()
+    return ParsedResponse(option=options[idx])
+
+
+def _parse_v2_options(stripped: str, options: list[str]) -> ParsedResponse:
+    """v2 option matching on markdown-stripped text (after refusal checks)."""
+    text = _strip_markdown(stripped).strip()
+    if not text:
+        return ParsedResponse()
+
+    by_norm = {_normalize(opt): opt for opt in options}
+    text_norm = _normalize(text)
+    if text_norm in by_norm:
+        return ParsedResponse(option=by_norm[text_norm])
+
+    match = _BARE_LETTER_RE.match(text)
+    if match:
+        idx = ord(match.group(1).upper()) - ord("A")
+        if 0 <= idx < len(options):
+            return ParsedResponse(option=options[idx])
+        return ParsedResponse()
+
+    labelled = _leading_label_match(text, options)
+    if labelled is not None:
+        return labelled
+
+    lines = [ln for ln in text.splitlines() if ln.strip()]
+    # A parenthesized label after a short lead-in ("I'd say (B) agree",
+    # possibly below a roleplay line like "*thinks*"). Only when the opening
+    # lines carry exactly one distinct label; several means a list or a
+    # comparison, not an answer.
+    lead = "\n".join(lines[:2])
+    labels = list(_PAREN_LABEL_RE.finditer(lead))
+    if labels and len({m.group(1).upper() for m in labels}) == 1:
+        inline = _leading_label_match(lead[labels[0].start() :], options)
+        if inline is not None:
+            return inline
+
+    first_line = lines[0] if lines else ""
+    contained = _containment_match(first_line, options)
+    if contained is not None:
+        return ParsedResponse(option=contained)
+
+    # Whole-response containment only when unambiguous: an explanation that
+    # mentions several options must not be scored as whichever matched first.
+    hits = [opt for opt in options if _containment_match(text, [opt]) is not None]
+    if len(hits) == 1:
+        return ParsedResponse(option=hits[0])
+    return ParsedResponse()
+
+
 def parse_option_response(
     text: str,
     options: list[str],
     *,
     refusal_detector_version: int = REFUSAL_DETECTOR_VERSION,
+    option_parser_version: int = OPTION_PARSER_VERSION,
 ) -> ParsedResponse:
     """Parse a raw model response into an option selection, refusal, or failure.
 
@@ -111,6 +231,9 @@ def parse_option_response(
     answer-initial anchoring + option-echo exemption) or 1 (the legacy
     un-anchored patterns, kept callable so historical runs can be
     reproduced bit-for-bit).
+
+    ``option_parser_version`` selects option matching the same way: 2
+    (default, markdown-aware) or 1 (legacy).
     """
     raw = str(text)
     stripped = raw.strip()
@@ -134,6 +257,9 @@ def parse_option_response(
     if is_refusal:
         return ParsedResponse(refusal=True)
 
+    if option_parser_version >= 2:
+        return _parse_v2_options(stripped, options)
+
     # 3. Bare letter, anchored full-match only.
     match = _BARE_LETTER_RE.match(stripped)
     if match:

{synthbench_eval-0.6.2 → synthbench_eval-0.7.0}/src/synthbench/runner.py

@@ -22,6 +22,7 @@ from synthbench.metrics import (
     extract_human_refusal_rate,
     conditioning_fidelity,
 )
+from synthbench.providers._parsing import OPTION_PARSER_VERSION
 from synthbench.providers.base import Distribution, PersonaSpec, Provider, Response
 from synthbench.stats import bootstrap_ci, question_set_hash
 
@@ -476,6 +477,9 @@ class BenchmarkRunner:
                 # this run's raw responses. Absent on pre-v2 files (= v1).
                 # Never feeds build_config_id, so config_ids are stable.
                 "refusal_detector_version": REFUSAL_DETECTOR_VERSION,
+                # Same contract for option matching (synthbench#352): absent
+                # on pre-v2 files (= v1); never feeds build_config_id.
+                "option_parser_version": OPTION_PARSER_VERSION,
                 **_provider_reproducibility_hashes(self.provider),
             },
             elapsed_seconds=elapsed,

{synthbench_eval-0.6.2 → synthbench_eval-0.7.0}/src/synthbench_eval.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: synthbench-eval
-Version: 0.6.2
+Version: 0.7.0
 Summary: Open benchmark harness for synthetic survey respondent quality
 License-Expression: MIT
 Keywords: benchmark,survey,synthetic-data,llm,evaluation,opinionsqa,globalopinionqa,gss,convergence,althing

{synthbench_eval-0.6.2 → synthbench_eval-0.7.0}/tests/test_provider_parsing.py

@@ -846,6 +846,114 @@ def test_refusal_detector_version_constant():
     assert REFUSAL_DETECTOR_VERSION == 2
 
 
+# ---------------------------------------------------------------------------
+# Option parser v2 (synthbench#352) — markdown-aware matching
+# ---------------------------------------------------------------------------
+
+_OPTS_ALCOHOL = [
+    "Morally acceptable",
+    "Morally unacceptable",
+    "Not a moral issue",
+    "Depends on the situation (VOL)",
+]
+_OPTS_CHINA = [
+    "Next 10 years",
+    "Next 20 years",
+    "Next 50 years",
+    "China will not replace U.S.",
+]
+_OPTS_CONFIDENCE = ["a great deal", "only some", "hardly any", "don't know"]
+
+
+@pytest.mark.parametrize(
+    ("text", "options", "expected_v2", "expected_v1"),
+    [
+        # Markdown-bolded leading label + explanation that mentions another
+        # option: v1 mis-scored via whole-text containment.
+        (
+            "**(D) Depends on the situation.**\n\nA drink with friends is fine. "
+            "For most people it is not a moral issue.",
+            _OPTS_ALCOHOL,
+            "Depends on the situation (VOL)",
+            "Not a moral issue",
+        ),
+        (
+            "**(D) China will not replace the U.S.**\n\nMaybe the gap closes in the next 20 years.",
+            _OPTS_CHINA,
+            "China will not replace U.S.",
+            "Next 20 years",
+        ),
+        # Label after a lead-in, below a roleplay line (from committed runs).
+        (
+            "I'd say **(B) only some confidence**.\n\nThey could do a great deal better.",
+            _OPTS_CONFIDENCE,
+            "only some",
+            "a great deal",
+        ),
+        ("D. Depends", _OPTS_ALCOHOL, "Depends on the situation (VOL)", None),
+        ("Answer: (C)", _OPTS_ALCOHOL, "Not a moral issue", None),
+        (
+            "**Not a moral issue**",
+            _OPTS_ALCOHOL,
+            "Not a moral issue",
+            "Not a moral issue",
+        ),
+        (
+            "## (A) Morally acceptable\nBecause.",
+            _OPTS_ALCOHOL,
+            "Morally acceptable",
+            "Morally acceptable",
+        ),
+        # Unchanged plain forms.
+        ("B", _OPTS_ALCOHOL, "Morally unacceptable", "Morally unacceptable"),
+        (
+            "(B) Morally unacceptable",
+            _OPTS_ALCOHOL,
+            "Morally unacceptable",
+            "Morally unacceptable",
+        ),
+    ],
+)
+def test_option_parser_v2_markdown_answers(text, options, expected_v2, expected_v1):
+    assert parse_option_response(text, options).option == expected_v2
+    assert (
+        parse_option_response(text, options, option_parser_version=1).option
+        == expected_v1
+    )
+
+
+@pytest.mark.parametrize(
+    "text",
+    [
+        "(B) Morally acceptable",  # label and echoed option disagree
+        "(Z) whatever",  # label out of range
+        "I think A.",  # not a label
+        "a. something",  # lowercase bare letter is not a label
+        "Hard to say. (A) and (C) both have merit.",  # several labels
+        # Explanation names two options and no answer line resolves it.
+        "Hard to say.\nSome think it is not a moral issue; others find it morally unacceptable.",
+    ],
+)
+def test_option_parser_v2_refuses_ambiguous_answers(text):
+    assert parse_option_response(text, _OPTS_ALCOHOL) == ParsedResponse()
+
+
+def test_option_parser_v2_unambiguous_whole_text_containment():
+    text = "Hard to say.\nIn the end it is not a moral issue for me."
+    assert parse_option_response(text, _OPTS_ALCOHOL).option == "Not a moral issue"
+
+
+def test_option_parser_v2_keeps_refusal_detection():
+    text = "**I'd rather not answer that.**"
+    assert parse_option_response(text, _OPTS_ALCOHOL) == ParsedResponse(refusal=True)
+
+
+def test_option_parser_version_constant():
+    from synthbench.providers._parsing import OPTION_PARSER_VERSION
+
+    assert OPTION_PARSER_VERSION == 2
+
+
 # ---------------------------------------------------------------------------
 # Structured elicitation (tpl=structured) — schema-forced extraction
 # ---------------------------------------------------------------------------

{synthbench_eval-0.6.2 → synthbench_eval-0.7.0}/tests/test_publish_config_id_consistency.py

@@ -214,6 +214,18 @@ def test_refusal_detector_version_stamp_never_changes_config_id():
         )
 
 
+def test_option_parser_version_stamp_never_changes_config_id():
+    """``config.option_parser_version`` is metadata-only, like the refusal stamp."""
+    for provider in PROVIDER_FIXTURES:
+        base = _result(provider)
+        stamped = _result(provider)
+        stamped["config"]["option_parser_version"] = 2
+        assert (
+            _build_entry(base, rank=1)["config_id"]
+            == _build_entry(stamped, rank=1)["config_id"]
+        ), provider
+
+
 def test_all_committed_config_ids_unchanged_by_detector_stamp():
     """Recompute every committed run's config_id with and without the stamp.
 

{synthbench_eval-0.6.2 → synthbench_eval-0.7.0}/tests/test_runner.py

@@ -698,3 +698,13 @@ async def test_runner_refuses_empty_dataset(mock_provider):
     )
     with pytest.raises(EmptyQuestionSetError, match="loaded 0 questions"):
         await runner.run()
+
+
+async def test_runner_stamps_option_parser_version(mock_dataset, mock_provider):
+    from synthbench.providers._parsing import OPTION_PARSER_VERSION
+
+    runner = BenchmarkRunner(
+        dataset=mock_dataset, provider=mock_provider, samples_per_question=2
+    )
+    result = await runner.run(n=1)
+    assert result.config["option_parser_version"] == OPTION_PARSER_VERSION