synthbench-eval 0.4.0__py3-none-any.whl

synthbench/convergence/__init__.py

@@ -0,0 +1,67 @@
+"""Bootstrap convergence analysis over aggregate human distributions.
+
+Given a Question with a known ``human_distribution``, this module computes a
+theoretical ~1/sqrt(n) convergence curve by multinomial subsampling:
+
+  curve(n) = distribution over B bootstrap samples of
+             JSD(empirical_sample_of_size_n, full_distribution)
+
+Three layers:
+
+* :mod:`synthbench.convergence.bootstrap` -- multinomial sub-sampling primitives
+* :mod:`synthbench.convergence.curves`    -- curve computation
+  (n -> JSD_mean / JSD_p10 / JSD_p90) across sample sizes
+* :mod:`synthbench.convergence.thresholds` -- "convergence n" finder: smallest
+  n at which the curve stabilizes below epsilon with a flat tail
+
+The output is the theoretical lower bound an idealized i.i.d. sampler from the
+aggregate distribution would achieve. It is NOT an estimate of real-population
+convergence (population heterogeneity requires microdata; see the microdata
+bead). Treat this curve as the floor everything else gets compared to.
+"""
+
+from __future__ import annotations
+
+from synthbench.convergence.baseline import (
+    BaselineGatedError,
+    BaselineUnavailable,
+    load_convergence_baseline,
+)
+from synthbench.convergence.bootstrap import (
+    bootstrap_sample,
+    empirical_distribution,
+)
+from synthbench.convergence.curves import (
+    DEFAULT_BOOTSTRAP_B,
+    DEFAULT_SAMPLE_SIZES,
+    CurvePoint,
+    compute_curve,
+)
+from synthbench.convergence.real_sampling import (
+    compute_real_curve,
+    respondent_distribution,
+    subsample_distribution,
+)
+from synthbench.convergence.thresholds import (
+    DEFAULT_DELTA,
+    DEFAULT_EPSILON,
+    find_convergence_n,
+)
+
+__all__ = [
+    "BaselineGatedError",
+    "BaselineUnavailable",
+    "load_convergence_baseline",
+    "bootstrap_sample",
+    "empirical_distribution",
+    "compute_curve",
+    "CurvePoint",
+    "DEFAULT_SAMPLE_SIZES",
+    "DEFAULT_BOOTSTRAP_B",
+    "compute_real_curve",
+    "respondent_distribution",
+    "subsample_distribution",
+    "find_convergence_n",
+    "DEFAULT_EPSILON",
+    "DEFAULT_DELTA",
+]

synthbench/convergence/baseline.py

@@ -0,0 +1,172 @@
+"""Stable baseline-loader export for synthpanel's ``--calibrate-against`` flag.
+
+synthpanel (>= 0.9) probes ``synthbench`` at runtime for a
+``load_convergence_baseline`` callable when a user passes
+``--calibrate-against DATASET:QUESTION``. This module is the single
+owner of that export.
+
+Policy:
+
+* ``full``-tier datasets (currently ``gss`` and ``ntia``) return the
+  aggregate ``human_distribution`` directly — the underlying license is
+  unambiguously public.
+* ``gated``-tier datasets raise :class:`BaselineGatedError`. Serving
+  those payloads requires hitting the authenticated R2 origin with
+  credentials, which is out of scope for the in-process loader.
+* ``aggregates_only`` / ``citation_only`` datasets raise
+  :class:`BaselineUnavailable` — synthbench has no per-question
+  redistribution rights for them at all.
+* Unknown datasets raise :class:`BaselineUnavailable`; the policy
+  lookup is conservative (defaults to ``aggregates_only``) and a typo
+  should not silently leak or crash downstream.
+
+The return shape is fixed by ``docs/convergence-analysis.md`` — synthpanel
+attaches ``human_distribution`` to every matching question's
+``calibration`` sub-object and computes JSD with its own local metric.
+
+For datasets whose policy permits redistributing the question wording
+(``full`` / ``citation_only``), the payload additionally carries
+``question_text`` and ``options`` so synthpanel can present the *real*
+survey item to its panel instead of fabricating a placeholder prompt.
+These fields are additive; ``human_distribution`` remains the only
+load-bearing key.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from synthbench.datasets import DATASETS
+from synthbench.datasets.policy import policy_for
+
+
+class BaselineUnavailable(LookupError):
+    """Raised when no per-question baseline can be returned for this dataset.
+
+    Covers unknown datasets and the ``aggregates_only`` / ``citation_only``
+    tiers where synthbench ships no per-question artifact at all.
+    """
+
+
+class BaselineGatedError(BaselineUnavailable):
+    """Raised when the dataset is ``gated`` and requires authenticated access.
+
+    The in-process loader cannot vend gated-tier baselines — those payloads
+    live behind the JWT-authenticated R2 origin. synthpanel can surface this
+    to the user as an "authenticate to calibrate against <dataset>" hint.
+    """
+
+
+def load_convergence_baseline(
+    dataset: str,
+    question_key: str | None = None,
+) -> dict[str, Any]:
+    """Return the aggregate baseline payload for one question.
+
+    Args:
+        dataset: Adapter name as registered in
+            :data:`synthbench.datasets.DATASETS` (e.g. ``"gss"``, ``"ntia"``).
+            A filter suffix like ``"gss (2018)"`` is accepted and resolves
+            to the base-name adapter.
+        question_key: Question identifier. Accepts both the canonical
+            :attr:`Question.key` (e.g. ``"GSS_SPKATH"``) and the bare
+            upstream id (``"SPKATH"``). Required.
+
+    Returns:
+        Dict shaped per ``docs/convergence-analysis.md`` with keys:
+        ``dataset``, ``question_key``, ``human_distribution``,
+        ``redistribution_policy``, ``license_url``, ``citation``.
+
+        When the dataset policy permits redistributing question text
+        (``full`` / ``citation_only``), two additive keys are also
+        present: ``question_text`` (the survey item's wording) and
+        ``options`` (the ordered answer labels, aligned with the keys of
+        ``human_distribution``). Callers must treat both as optional —
+        older payloads and policy-withholding datasets omit them.
+
+    Raises:
+        BaselineGatedError: dataset tier is ``gated`` — authenticated
+            access required.
+        BaselineUnavailable: dataset is unknown or its tier is
+            ``aggregates_only`` / ``citation_only``; or the requested
+            question key does not resolve in the adapter.
+        ValueError: ``question_key`` was not provided.
+    """
+    if not question_key:
+        raise ValueError(
+            "question_key is required: load_convergence_baseline needs a "
+            "specific question to return human_distribution for"
+        )
+
+    policy = policy_for(dataset)
+
+    if policy.redistribution_policy == "gated":
+        raise BaselineGatedError(
+            f"dataset {policy.name!r} is gated-tier; baseline requires "
+            "authenticated access to the R2 origin and cannot be loaded "
+            "from the in-process synthbench API"
+        )
+
+    if policy.redistribution_policy != "full":
+        raise BaselineUnavailable(
+            f"dataset {dataset!r} has no per-question baseline available "
+            f"(tier={policy.redistribution_policy!r})"
+        )
+
+    adapter_cls = DATASETS.get(policy.name)
+    if adapter_cls is None:
+        # policy_for() falls back to "aggregates_only" for unknown names,
+        # so we'd have bailed above. This is defensive: a future adapter
+        # whose policy is declared elsewhere could theoretically skip
+        # registration.
+        raise BaselineUnavailable(f"no registered adapter for dataset {dataset!r}")
+
+    question = _resolve_question(adapter_cls(), question_key)
+
+    payload: dict[str, Any] = {
+        "dataset": policy.name,
+        "question_key": question.key,
+        "human_distribution": dict(question.human_distribution),
+        "redistribution_policy": policy.redistribution_policy,
+        "license_url": policy.license_url,
+        "citation": policy.citation,
+    }
+
+    # Additive, policy-gated metadata: the survey item's wording and its
+    # answer labels. synthpanel's calibration panel needs the real question
+    # text (rather than a fabricated placeholder) and needs option labels that
+    # line up with the ``human_distribution`` keys so its pick_one extraction
+    # aligns with ground truth. Only surfaced where the redistribution policy
+    # permits distributing question text (``full`` / ``citation_only``); this
+    # is a strictly narrower disclosure than ``human_distribution``.
+    if policy.allows_question_text:
+        payload["question_text"] = question.text
+        payload["options"] = list(question.options)
+
+    return payload
+
+
+def _resolve_question(dataset, question_key: str):
+    """Locate a Question by key; accept bare or adapter-prefixed forms.
+
+    Mirrors the CLI's :func:`convergence.cli_report._resolve_question`
+    behavior so synthpanel callers can pass either the canonical
+    ``Question.key`` or the bare upstream id.
+    """
+    questions = dataset.load()
+    for q in questions:
+        if q.key == question_key:
+            return q
+    suffixed = [q for q in questions if q.key.endswith(f"_{question_key}")]
+    if len(suffixed) == 1:
+        return suffixed[0]
+    raise BaselineUnavailable(
+        f"no question with key {question_key!r} in dataset {dataset.name!r}"
+    )
+
+
+__all__ = [
+    "load_convergence_baseline",
+    "BaselineUnavailable",
+    "BaselineGatedError",
+]

synthbench/convergence/bootstrap.py

@@ -0,0 +1,60 @@
+"""Multinomial sub-sampling primitives.
+
+The aggregate ``human_distribution`` on a Question is the population proportion
+vector. Drawing n responses from an i.i.d. population with that distribution is
+a multinomial draw. This module provides the narrow primitives -- actual curve
+computation lives in :mod:`synthbench.convergence.curves`.
+"""
+
+from __future__ import annotations
+
+import numpy as np
+
+
+def bootstrap_sample(
+    distribution: dict[str, float],
+    n: int,
+    rng: np.random.Generator | None = None,
+) -> dict[str, int]:
+    """Draw a multinomial sample of size ``n`` from ``distribution``.
+
+    Args:
+        distribution: Option -> probability. Need not be exactly normalized --
+            zero-sum inputs return an empty count vector.
+        n: Number of responses to draw. Must be non-negative.
+        rng: Optional numpy ``Generator`` for reproducibility. If omitted, a
+            fresh default generator is used (non-deterministic).
+
+    Returns:
+        Option -> integer count. Counts sum to ``n`` when the input has positive
+        mass; sum to 0 for an empty / zero-mass distribution.
+    """
+    if n < 0:
+        raise ValueError(f"n must be non-negative, got {n}")
+    if rng is None:
+        rng = np.random.default_rng()
+
+    keys = list(distribution.keys())
+    if not keys or n == 0:
+        return {k: 0 for k in keys}
+
+    probs = np.array([max(0.0, distribution[k]) for k in keys], dtype=np.float64)
+    total = probs.sum()
+    if total <= 0.0:
+        return {k: 0 for k in keys}
+    probs /= total
+
+    counts = rng.multinomial(n, probs)
+    return {k: int(c) for k, c in zip(keys, counts)}
+
+
+def empirical_distribution(sample: dict[str, int]) -> dict[str, float]:
+    """Normalize a count vector into an empirical distribution.
+
+    Zero-total samples return a dict with the same keys but zero probabilities,
+    preserving key alignment for downstream JSD.
+    """
+    total = sum(sample.values())
+    if total <= 0:
+        return {k: 0.0 for k in sample}
+    return {k: v / total for k, v in sample.items()}