synthbench-eval 0.6.2__tar.gz → 0.8.0__tar.gz

{synthbench_eval-0.6.2 → synthbench_eval-0.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: synthbench-eval
-Version: 0.6.2
+Version: 0.8.0
 Summary: Open benchmark harness for synthetic survey respondent quality
 License-Expression: MIT
 Keywords: benchmark,survey,synthetic-data,llm,evaluation,opinionsqa,globalopinionqa,gss,convergence,althing

{synthbench_eval-0.6.2 → synthbench_eval-0.8.0}/pyproject.toml

@@ -8,7 +8,7 @@ build-backend = "setuptools.build_meta"
 name = "synthbench-eval"
 # Placeholder — the publish job stamps the real version from the release tag
 # at build time (see publish-pypi in .github/workflows/auto-tag.yml).
-version = "0.6.2"
+version = "0.8.0"
 description = "Open benchmark harness for synthetic survey respondent quality"
 readme = "README.md"
 license = "MIT"

{synthbench_eval-0.6.2 → synthbench_eval-0.8.0}/src/synthbench/__init__.py

@@ -6,7 +6,7 @@ from synthbench.convergence.baseline import (
     load_convergence_baseline,
 )
 
-__version__ = "0.6.2"
+__version__ = "0.8.0"
 
 __all__ = [
     "BaselineGatedError",

{synthbench_eval-0.6.2 → synthbench_eval-0.8.0}/src/synthbench/metrics/__init__.py

@@ -11,6 +11,7 @@ from synthbench.metrics.refusal import (
     refusal_calibration,
     detect_refusal,
     detect_refusal_v2,
+    detect_refusal_v3,
     extract_human_refusal_rate,
     refusal_rate,
     p_refuse,
@@ -30,6 +31,7 @@ __all__ = [
     "refusal_calibration",
     "detect_refusal",
     "detect_refusal_v2",
+    "detect_refusal_v3",
     "extract_human_refusal_rate",
     "refusal_rate",
     "p_refuse",

{synthbench_eval-0.6.2 → synthbench_eval-0.8.0}/src/synthbench/metrics/refusal.py

@@ -6,13 +6,24 @@ Two detector versions coexist:
   pre-v2 runs). Un-anchored substring patterns; known precision failure:
   chatty in-character answers ("I don't get sick much, I'd say good") match
   ``\\bI don't\\b`` anywhere in the text and are mis-flagged as refusals.
-* :func:`detect_refusal_v2` — **v2** (current default via
-  :data:`REFUSAL_DETECTOR_VERSION`). Patterns must be answer-INITIAL
+* :func:`detect_refusal_v2` — **v2**. Patterns must be answer-INITIAL
   (anchored to the start of the trimmed response, tolerating leading
   pleasantries like "Well," / "Hmm,"), and never fire when the response
   contains a valid option for the current question (option-echo exemption —
   required because some instruments carry substantive options containing
   refusal-ish text, e.g. GSS GOD's "don't know, no way to find out").
+* :func:`detect_refusal_v3` — **v3** (current default via
+  :data:`REFUSAL_DETECTOR_VERSION`, synthbench#356). v2 missed common
+  character breaks from newer models — "I don't have a *personal* opinion
+  …", "I don't have a country …", "I don't think I should pick …", and
+  non-initial declines such as "The premise is outdated … so I can't pick
+  one of these options". Those fell through as parse failures instead of
+  refusals. v3 adds answer-initial "no stake / no standing" openers and
+  an explicit non-selection pattern searched anywhere in the first
+  paragraph. v2's option-echo exemption applies only when exactly one
+  option is echoed (a response weighing both sides chose neither), and
+  v3 adds a single-label exemption: a response carrying exactly one
+  option label (``(B)``) picked an answer, so it is never a refusal.
 
 Runs are stamped with ``config.refusal_detector_version``; the key is
 additive — files without it were parsed under v1. The stamp is metadata
@@ -26,7 +37,7 @@ import re
 
 #: Detector version stamped into run metadata (``config.refusal_detector_version``).
 #: Absent key on a committed run file means v1.
-REFUSAL_DETECTOR_VERSION = 2
+REFUSAL_DETECTOR_VERSION = 3
 
 # Patterns that indicate an LLM-style refusal when explicit refusal field is
 # unavailable. v1 — un-anchored; kept callable for reproducibility.
@@ -159,6 +170,93 @@ def detect_refusal_v2(text: str, options: list[str] | None = None) -> bool:
     return True
 
 
+# ---------------------------------------------------------------------------
+# v3 detector
+# ---------------------------------------------------------------------------
+
+# Additional answer-initial openers: the respondent says it has no opinion,
+# stake, or standing to answer ("I don't have a personal opinion of ...",
+# "I don't have a country ..."), or that answering isn't its place.
+_REFUSAL_PATTERNS_V3_INITIAL = [
+    re.compile(
+        r"\AI (?:do not|don'?t) have (?:a |an |any )?(?:personal |real |genuine |strong |own )?"
+        r"(?:opinions?|views?|stake|preferences?|country|nationality|personal experience|"
+        r"lived experience|political (?:views?|opinions?|affiliation))\b",
+        re.IGNORECASE,
+    ),
+    re.compile(
+        r"\AI (?:do not|don'?t) think (?:I should|it'?s (?:my place|appropriate|right|fair) to|"
+        r"it would be (?:appropriate|right|fair) (?:for me )?to) "
+        r"(?:pick|choose|select|answer|take a side|weigh in|rate)\b",
+        re.IGNORECASE,
+    ),
+    re.compile(
+        r"\AI'?m (?:just )?an? (?:AI|artificial intelligence|(?:large )?language model)\b",
+        re.IGNORECASE,
+    ),
+]
+
+# Explicit non-selection anywhere in the first paragraph ("..., so I can't
+# pick one of these options", "I won't pick (A) or (B)"). Requires a
+# selection verb, so "I can't believe ..." / "I won't lie, ..." never match.
+_NON_SELECTION_RE = re.compile(
+    r"\bI (?:can'?t|cannot|won'?t|will not|shouldn'?t|should not|am not going to|'?m not going to)\s+"
+    r"(?:honestly |really |fairly |just )?(?:pick|choose|select|rate|give (?:a|an|my) (?:rating|answer))\b",
+    re.IGNORECASE,
+)
+
+_OPTION_LABEL_RE = re.compile(r"\(([A-Za-z])\)")
+
+
+def _first_paragraph(text: str) -> str:
+    for block in re.split(r"\n\s*\n", text.strip()):
+        if block.strip():
+            return block
+    return ""
+
+
+def _single_option_label(text: str, options: list[str]) -> bool:
+    """True when *text* names exactly one in-range option label like ``(B)``."""
+    letters = {m.group(1).upper() for m in _OPTION_LABEL_RE.finditer(text)}
+    return len(letters) == 1 and 0 <= ord(next(iter(letters))) - ord("A") < len(options)
+
+
+def detect_refusal_v3(text: str, options: list[str] | None = None) -> bool:
+    """Detect refusal — v3: v2 plus character-break openers and non-selection.
+
+    Classification order:
+
+    1. A response that echoes exactly one declared option (v2's exemption,
+       narrowed: discussing *both* sides is not choosing one) or names
+       exactly one option label (``(B)``) chose an answer: not a refusal.
+    2. Any v2 answer-initial pattern, or a v3 answer-initial "no opinion /
+       no stake / not my place" opener: refusal.
+    3. An explicit non-selection ("I can't pick one of these options",
+       "I won't pick (A) or (B)") anywhere in the first paragraph: refusal.
+
+    Args:
+        text: Raw response text from the provider.
+        options: Declared answer options for the current question.
+
+    Returns:
+        True if the response declines to choose an option.
+    """
+    raw = str(text)
+    stripped = raw.strip(_WRAPPING_CHARS)
+    if not stripped:
+        return False
+    if options:
+        echoed = sum(1 for opt in options if _mentions_option(raw, [opt]))
+        if echoed == 1 or _single_option_label(raw, options):
+            return False
+    opening = _PLEASANTRY_PREFIX_RE.sub("", stripped)
+    if any(p.search(opening) for p in _REFUSAL_PATTERNS_V2):
+        return True
+    if any(p.search(opening) for p in _REFUSAL_PATTERNS_V3_INITIAL):
+        return True
+    return bool(_NON_SELECTION_RE.search(_first_paragraph(stripped)))
+
+
 def refusal_calibration(
     provider_refusal_rates: list[float],
     human_refusal_rates: list[float],

synthbench_eval-0.8.0/src/synthbench/providers/_parsing.py

@@ -0,0 +1,280 @@
+"""Shared option-response parsing for all providers.
+
+Historically every provider carried its own private ``_parse_letter`` copy
+with two systematic bugs:
+
+1. The letter match was un-anchored (``^\\(?([A-Z])\\)?``), so a model that
+   echoed option *text* was parsed by its first character: options
+   ``["Better", "Worse"]`` + response ``"Better"`` parsed as **"Worse"**
+   (``B`` -> index 1).
+2. The fallback was a raw substring test, so ``"Disagree"`` matched
+   ``"Agree"`` first, and refusal text like ``"I cannot answer"`` matched
+   ``"No"`` (``"no" in "cannot"``).
+
+Parse failures were then silently coerced to ``options[0]``, biasing every
+published distribution toward the first option.
+
+This module is the single replacement. Matching order:
+
+1. Exact (normalized) full-string equality against the option text — an
+   unambiguous echo of an option always wins.
+2. Refusal detection (:func:`synthbench.metrics.refusal.detect_refusal`)
+   runs BEFORE any fuzzy option matching, so refusal text can never be
+   substring-matched into a vote.
+3. Bare option letter, anchored as a full match (``"B"``, ``"(c)"``,
+   ``"A."`` — but never the leading character of a longer word).
+4. Word-boundary containment against the option text, longest option
+   first, as a last resort (``"I'd say Disagree"`` -> ``"Disagree"``).
+
+Anything else is a parse failure: ``option is None`` and ``refusal`` is
+False. Callers must NOT substitute a default option.
+
+Parser versions (``config.option_parser_version``; absent key = v1):
+
+* **v1** — the matching order above, verbatim. Kept callable so historical
+  runs re-parse bit-for-bit.
+* **v2** (current, synthbench#352) — newer models answer in markdown prose
+  (``**(D) Depends on the situation.**`` followed by an explanation). v1
+  counted those as parse failures, and its whole-text containment could
+  mis-score them by matching an option mentioned in the explanation. v2:
+
+  - strips markdown emphasis/heading/quote markup before matching;
+  - accepts a *leading* option label (``(D) ...``, ``D. ...``, ``D) ...``)
+    as the answer, unless the text right after it exactly names a
+    different option (contradictory → parse failure);
+  - restricts containment to the first line, falling back to the whole
+    response only when exactly one option is contained in it.
+
+  Refusal detection is unchanged (it is versioned separately).
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+
+from synthbench.metrics.refusal import (
+    REFUSAL_DETECTOR_VERSION,
+    detect_refusal,
+    detect_refusal_v2,
+    detect_refusal_v3,
+)
+
+# Full-match bare letter: optional "(", one ASCII letter, optional ")",
+# optional trailing ".", ")" or ":", nothing else but whitespace.
+_BARE_LETTER_RE = re.compile(r"^\s*\(?([A-Za-z])\)?[.):]?\s*$")
+
+_WS_RE = re.compile(r"\s+")
+
+#: Parser version stamped into run metadata (``config.option_parser_version``).
+#: Absent key on a committed run file means v1.
+OPTION_PARSER_VERSION = 2
+
+# v2: markdown markup to drop before matching — emphasis runs (``**``,
+# ``__``, ``*``), and line-leading heading / blockquote / bullet markers.
+_MD_EMPHASIS_RE = re.compile(r"(\*{1,3}|_{2,3})")
+_MD_LINE_PREFIX_RE = re.compile(
+    r"^[ \t]*(?:#{1,6}[ \t]+|>[ \t]*|[-+][ \t]+)", re.MULTILINE
+)
+
+# v2: leading option label, optionally after an "Answer:" style prefix.
+# Parenthesized letters may be either case; bare letters must be uppercase
+# and delimited (checked in _leading_label_match) so "I think..." or
+# "A lot of people..." never parse as a label.
+_LEADING_LABEL_RE = re.compile(
+    r"^\s*(?:(?:answer|my answer|choice)\s*[:\-]\s*)?"
+    r"(?:\(([A-Za-z])\)|([A-Za-z])[.):](?=\s|$))\s*(.*)",
+    re.IGNORECASE | re.DOTALL,
+)
+_PAREN_LABEL_RE = re.compile(r"\(([A-Za-z])\)")
+
+
+@dataclass(frozen=True)
+class ParsedResponse:
+    """Outcome of parsing one raw model response.
+
+    Exactly one of three states:
+
+    * option selected: ``option`` is the matched option text, ``refusal``
+      is False.
+    * refusal: ``option`` is None, ``refusal`` is True.
+    * parse failure: ``option`` is None, ``refusal`` is False.
+    """
+
+    option: str | None = None
+    refusal: bool = False
+
+    @property
+    def is_parse_failure(self) -> bool:
+        return self.option is None and not self.refusal
+
+
+def _normalize(text: str) -> str:
+    """Case-fold, collapse whitespace, and strip surrounding punctuation."""
+    text = _WS_RE.sub(" ", str(text)).strip().lower()
+    # Strip wrapping punctuation/quotes but keep interior punctuation so
+    # options like "Don't know" still compare correctly.
+    return text.strip("\"'`.,;:!()[]{} ")
+
+
+def _containment_match(text: str, options: list[str]) -> str | None:
+    """Word-boundary containment, longest option first.
+
+    Longest-first ordering prevents "Agree" from shadowing "Disagree"-style
+    superstring options; the word boundary (non-word characters on both
+    sides) prevents "no" from matching inside "cannot".
+    """
+    text_norm = _normalize(text)
+    if not text_norm:
+        return None
+    for opt in sorted(options, key=lambda o: len(_normalize(o)), reverse=True):
+        opt_norm = _normalize(opt)
+        if not opt_norm:
+            continue
+        pattern = r"(?<!\w)" + re.escape(opt_norm) + r"(?!\w)"
+        if re.search(pattern, text_norm):
+            return opt
+    return None
+
+
+def _strip_markdown(text: str) -> str:
+    """Drop markdown emphasis and line-leading markup (v2)."""
+    return _MD_EMPHASIS_RE.sub("", _MD_LINE_PREFIX_RE.sub("", text))
+
+
+def _leading_label_match(text: str, options: list[str]) -> ParsedResponse | None:
+    """Resolve a leading ``(D) ...`` / ``D. ...`` label (v2).
+
+    Returns ``None`` when there is no label; a parse failure when the label
+    is out of range or contradicted by an exact option name right after it.
+    The lowercase bare form (``a. ...``) is not a label — only ``(a)``.
+    """
+    match = _LEADING_LABEL_RE.match(text)
+    if not match:
+        return None
+    paren, bare, rest = match.groups()
+    if bare is not None and not bare.isupper():
+        return None
+    letter = (paren or bare).upper()
+    idx = ord(letter) - ord("A")
+    if not 0 <= idx < len(options):
+        return ParsedResponse()
+    # "(B) Favor" where B is "Oppose": the label and the echoed text
+    # disagree, so neither can be trusted.
+    first_line = rest.strip().split("\n", 1)[0]
+    by_norm = {_normalize(opt): opt for opt in options}
+    echoed = by_norm.get(_normalize(first_line))
+    if echoed is not None and echoed != options[idx]:
+        return ParsedResponse()
+    return ParsedResponse(option=options[idx])
+
+
+def _parse_v2_options(stripped: str, options: list[str]) -> ParsedResponse:
+    """v2 option matching on markdown-stripped text (after refusal checks)."""
+    text = _strip_markdown(stripped).strip()
+    if not text:
+        return ParsedResponse()
+
+    by_norm = {_normalize(opt): opt for opt in options}
+    text_norm = _normalize(text)
+    if text_norm in by_norm:
+        return ParsedResponse(option=by_norm[text_norm])
+
+    match = _BARE_LETTER_RE.match(text)
+    if match:
+        idx = ord(match.group(1).upper()) - ord("A")
+        if 0 <= idx < len(options):
+            return ParsedResponse(option=options[idx])
+        return ParsedResponse()
+
+    labelled = _leading_label_match(text, options)
+    if labelled is not None:
+        return labelled
+
+    lines = [ln for ln in text.splitlines() if ln.strip()]
+    # A parenthesized label after a short lead-in ("I'd say (B) agree",
+    # possibly below a roleplay line like "*thinks*"). Only when the opening
+    # lines carry exactly one distinct label; several means a list or a
+    # comparison, not an answer.
+    lead = "\n".join(lines[:2])
+    labels = list(_PAREN_LABEL_RE.finditer(lead))
+    if labels and len({m.group(1).upper() for m in labels}) == 1:
+        inline = _leading_label_match(lead[labels[0].start() :], options)
+        if inline is not None:
+            return inline
+
+    first_line = lines[0] if lines else ""
+    contained = _containment_match(first_line, options)
+    if contained is not None:
+        return ParsedResponse(option=contained)
+
+    # Whole-response containment only when unambiguous: an explanation that
+    # mentions several options must not be scored as whichever matched first.
+    hits = [opt for opt in options if _containment_match(text, [opt]) is not None]
+    if len(hits) == 1:
+        return ParsedResponse(option=hits[0])
+    return ParsedResponse()
+
+
+def parse_option_response(
+    text: str,
+    options: list[str],
+    *,
+    refusal_detector_version: int = REFUSAL_DETECTOR_VERSION,
+    option_parser_version: int = OPTION_PARSER_VERSION,
+) -> ParsedResponse:
+    """Parse a raw model response into an option selection, refusal, or failure.
+
+    Never falls back to ``options[0]``. See module docstring for the
+    matching order and rationale.
+
+    ``refusal_detector_version`` selects the refusal heuristic: 3 (default,
+    v2 plus character-break openers and explicit non-selection), 2
+    (answer-initial anchoring + option-echo exemption) or 1 (the legacy
+    un-anchored patterns). Older versions stay callable so historical runs
+    can be reproduced bit-for-bit.
+
+    ``option_parser_version`` selects option matching the same way: 2
+    (default, markdown-aware) or 1 (legacy).
+    """
+    raw = str(text)
+    stripped = raw.strip()
+    if not stripped:
+        return ParsedResponse()
+
+    # 1. Exact normalized equality — an unambiguous echo of an option wins
+    # even over refusal-pattern heuristics (an option worded in the first
+    # person, e.g. "I don't know", is a legitimate selection when echoed
+    # verbatim).
+    text_norm = _normalize(stripped)
+    by_norm = {_normalize(opt): opt for opt in options}
+    if text_norm in by_norm:
+        return ParsedResponse(option=by_norm[text_norm])
+
+    # 2. Refusal detection BEFORE fuzzy option matching.
+    if refusal_detector_version >= 3:
+        is_refusal = detect_refusal_v3(stripped, options)
+    elif refusal_detector_version >= 2:
+        is_refusal = detect_refusal_v2(stripped, options)
+    else:
+        is_refusal = detect_refusal(stripped)
+    if is_refusal:
+        return ParsedResponse(refusal=True)
+
+    if option_parser_version >= 2:
+        return _parse_v2_options(stripped, options)
+
+    # 3. Bare letter, anchored full-match only.
+    match = _BARE_LETTER_RE.match(stripped)
+    if match:
+        idx = ord(match.group(1).upper()) - ord("A")
+        if 0 <= idx < len(options):
+            return ParsedResponse(option=options[idx])
+        return ParsedResponse()
+
+    # 4. Word-boundary containment, longest option first.
+    contained = _containment_match(stripped, options)
+    if contained is not None:
+        return ParsedResponse(option=contained)
+
+    return ParsedResponse()

{synthbench_eval-0.6.2 → synthbench_eval-0.8.0}/src/synthbench/runner.py

@@ -22,6 +22,7 @@ from synthbench.metrics import (
     extract_human_refusal_rate,
     conditioning_fidelity,
 )
+from synthbench.providers._parsing import OPTION_PARSER_VERSION
 from synthbench.providers.base import Distribution, PersonaSpec, Provider, Response
 from synthbench.stats import bootstrap_ci, question_set_hash
 
@@ -476,6 +477,9 @@ class BenchmarkRunner:
                 # this run's raw responses. Absent on pre-v2 files (= v1).
                 # Never feeds build_config_id, so config_ids are stable.
                 "refusal_detector_version": REFUSAL_DETECTOR_VERSION,
+                # Same contract for option matching (synthbench#352): absent
+                # on pre-v2 files (= v1); never feeds build_config_id.
+                "option_parser_version": OPTION_PARSER_VERSION,
                 **_provider_reproducibility_hashes(self.provider),
             },
             elapsed_seconds=elapsed,

{synthbench_eval-0.6.2 → synthbench_eval-0.8.0}/src/synthbench_eval.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: synthbench-eval
-Version: 0.6.2
+Version: 0.8.0
 Summary: Open benchmark harness for synthetic survey respondent quality
 License-Expression: MIT
 Keywords: benchmark,survey,synthetic-data,llm,evaluation,opinionsqa,globalopinionqa,gss,convergence,althing