althing 1.7.0__py3-none-any.whl

althing/aggregation.py

@@ -0,0 +1,240 @@
+"""Post-run variant aggregation and robustness scoring (sp-5on.16).
+
+Groups panelist results by the ``_variant_of`` metadata field (set by the
+perturbation module, sp-5on.14) and computes two-level nested robustness
+scores: intra-persona agreement across variants (inner) and inter-persona
+agreement across the panel (outer).
+
+Formula: R(F) = (1/N) * sum_i[(1/K_i) * sum_j 1{variant_ij agrees with F}]
+
+Thresholds: R >= 0.8 robust, 0.6-0.8 moderate, 0.4-0.6 sensitive, < 0.4 fragile.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from typing import Any
+
+from althing.orchestrator import PanelistResult
+
+# ---------------------------------------------------------------------------
+# Data models
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class VariantGroup:
+    """Results for a source persona and all its variants."""
+
+    source_name: str
+    original: PanelistResult | None
+    variants: list[PanelistResult] = field(default_factory=list)
+
+    @property
+    def k(self) -> int:
+        """Number of variants."""
+        return len(self.variants)
+
+
+@dataclass(frozen=True)
+class FindingRobustness:
+    """Robustness of responses to a single question across all personas."""
+
+    question_index: int
+    score: float
+    classification: str
+    per_persona: dict[str, float]
+
+
+@dataclass(frozen=True)
+class RobustnessReport:
+    """Complete two-level robustness report."""
+
+    findings: list[FindingRobustness]
+    aggregate_robustness: float
+    fragile_findings: list[FindingRobustness]
+    n_personas: int
+    per_persona_robustness: dict[str, float]
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+
+def _default_extractor(response: dict[str, Any]) -> str:
+    """Extract a comparable string value from a response dict."""
+    val = response.get("response", "")
+    return str(val)
+
+
+def _classify_robustness(r: float) -> str:
+    """Classify a robustness score into a named tier."""
+    if r >= 0.8:
+        return "robust"
+    elif r >= 0.6:
+        return "moderate"
+    elif r >= 0.4:
+        return "sensitive"
+    return "fragile"
+
+
+# ---------------------------------------------------------------------------
+# Grouping
+# ---------------------------------------------------------------------------
+
+
+def aggregate_variants(
+    results: list[PanelistResult],
+    personas: list[dict[str, Any]],
+) -> list[VariantGroup]:
+    """Group panelist results by ``_variant_of`` metadata.
+
+    Matches results to personas by name.  Personas whose dict contains a
+    ``_variant_of`` key are variants; all others are originals.
+
+    Returns one :class:`VariantGroup` per source persona that has at least
+    one variant.  Groups are sorted by source name for deterministic order.
+
+    Args:
+        results: Panelist results from a panel run that included both
+            original personas and their perturbation variants.
+        personas: The persona dicts used in the panel run.  Variant
+            personas must carry the ``_variant_of`` field set by the
+            perturbation module.
+
+    Returns:
+        List of :class:`VariantGroup`, one per source persona with variants.
+    """
+    persona_by_name: dict[str, dict[str, Any]] = {p["name"]: p for p in personas if p.get("name")}
+
+    originals: dict[str, PanelistResult] = {}
+    variant_map: dict[str, list[PanelistResult]] = {}
+
+    for r in results:
+        persona = persona_by_name.get(r.persona_name)
+        if persona and "_variant_of" in persona:
+            source = persona["_variant_of"]
+            variant_map.setdefault(source, []).append(r)
+        else:
+            originals[r.persona_name] = r
+
+    groups: list[VariantGroup] = []
+    for source_name in sorted(set(originals) | set(variant_map)):
+        variants = variant_map.get(source_name, [])
+        if not variants:
+            continue
+        groups.append(
+            VariantGroup(
+                source_name=source_name,
+                original=originals.get(source_name),
+                variants=variants,
+            )
+        )
+    return groups
+
+
+# ---------------------------------------------------------------------------
+# Scoring
+# ---------------------------------------------------------------------------
+
+
+def robustness_report(
+    groups: list[VariantGroup],
+    *,
+    response_extractor: Callable[[dict[str, Any]], str] | None = None,
+) -> RobustnessReport:
+    """Compute two-level nested robustness scores from variant groups.
+
+    For each question, the original persona's response is the reference.
+    Each variant's response is compared for agreement (exact match on the
+    extracted string value).
+
+    **Inner level** (intra-persona): for persona *i* with *K_i* variants,
+    the agreement fraction is ``agreements / K_i``.
+
+    **Outer level** (inter-persona): for question *q*,
+    ``R(q) = (1/N) * sum_i(agreement_fraction_i)``.
+
+    Args:
+        groups: Variant groups from :func:`aggregate_variants`.
+        response_extractor: Extracts a comparable string from a response
+            dict.  Defaults to ``str(response["response"])``.
+
+    Returns:
+        :class:`RobustnessReport` with per-finding and per-persona scores.
+
+    Raises:
+        ValueError: If *groups* is empty or no group has both an original
+            result and at least one variant.
+    """
+    if not groups:
+        raise ValueError("groups must not be empty")
+
+    extract = response_extractor or _default_extractor
+
+    scorable = [g for g in groups if g.original is not None and g.k > 0]
+    if not scorable:
+        raise ValueError("no groups with both an original and variants")
+
+    n_questions = max(len(g.original.responses) for g in scorable)
+
+    findings: list[FindingRobustness] = []
+    per_persona_all: dict[str, list[float]] = {}
+
+    for qi in range(n_questions):
+        per_persona: dict[str, float] = {}
+
+        for group in scorable:
+            if qi >= len(group.original.responses):
+                continue
+            orig_resp = group.original.responses[qi]
+            # No reference answer when the original itself was conditionally skipped.
+            if isinstance(orig_resp, dict) and orig_resp.get("skipped_by_condition"):
+                continue
+
+            ref = extract(orig_resp)
+
+            k = 0
+            agreements = 0
+            for v in group.variants:
+                if qi >= len(v.responses):
+                    continue
+                v_resp = v.responses[qi]
+                # Don't count a variant that skipped this question by condition.
+                if isinstance(v_resp, dict) and v_resp.get("skipped_by_condition"):
+                    continue
+                k += 1
+                if extract(v_resp) == ref:
+                    agreements += 1
+
+            score = agreements / k if k > 0 else 0.0
+            per_persona[group.source_name] = score
+            per_persona_all.setdefault(group.source_name, []).append(score)
+
+        if per_persona:
+            finding_score = sum(per_persona.values()) / len(per_persona)
+            findings.append(
+                FindingRobustness(
+                    question_index=qi,
+                    score=finding_score,
+                    classification=_classify_robustness(finding_score),
+                    per_persona=per_persona,
+                )
+            )
+
+    aggregate = sum(f.score for f in findings) / len(findings) if findings else 0.0
+    fragile = [f for f in findings if f.score < 0.6]
+
+    persona_robustness: dict[str, float] = {}
+    for name, scores in per_persona_all.items():
+        persona_robustness[name] = sum(scores) / len(scores) if scores else 0.0
+
+    return RobustnessReport(
+        findings=findings,
+        aggregate_robustness=aggregate,
+        fragile_findings=fragile,
+        n_personas=len(scorable),
+        per_persona_robustness=persona_robustness,
+    )

althing/analysis/__init__.py

@@ -0,0 +1,46 @@
+"""Panel-result analysis utilities.
+
+Two layers live here:
+
+* Inspection helpers (:mod:`althing.analysis.inspect`) — schema
+  walkers and no-LLM result summaries.
+* Structured-response aggregation (sp-2hpi scaffolding) —
+  :mod:`althing.analysis.distribution` computes per-question
+  distributions from ``response_schema``-typed responses, and
+  :mod:`althing.analysis.subgroup` splits those distributions by
+  persona field.
+
+Heavy statistical analysis that's entangled with the legacy
+narrative-synthesis pipeline continues to live in
+:mod:`althing.analyze`.
+"""
+
+from althing.analysis.distribution import (
+    InvalidResponseSchemaError,
+    coerce_enum_value,
+    coerce_scale_value,
+    coerce_tagged_themes,
+    distribution_for_question,
+)
+from althing.analysis.inspect import (
+    InspectReport,
+    build_inspect_report,
+    format_inspect_text,
+)
+from althing.analysis.subgroup import (
+    UnknownPersonaFieldError,
+    subgroup_breakdown,
+)
+
+__all__ = [
+    "InspectReport",
+    "InvalidResponseSchemaError",
+    "UnknownPersonaFieldError",
+    "build_inspect_report",
+    "coerce_enum_value",
+    "coerce_scale_value",
+    "coerce_tagged_themes",
+    "distribution_for_question",
+    "format_inspect_text",
+    "subgroup_breakdown",
+]

althing/analysis/distribution.py

@@ -0,0 +1,323 @@
+"""Deterministic distributions over structured panelist responses (sp-2hpi).
+
+Given a question's ``response_schema`` and the per-panelist raw responses,
+this module computes a distribution summary without any LLM call:
+
+* ``scale``  — frequency table, mean, median, stdev, min, max
+* ``enum``   — frequency table over the declared options (zero-filled)
+* ``tagged_themes`` — per-tag frequency over the declared taxonomy
+  (zero-filled), with a bucket for off-taxonomy ``other`` themes
+* ``text``   — only ``n`` and ``n_nonempty`` (text carries no
+  distribution on its own; extraction lives upstream)
+
+The output is intentionally plain dicts/lists so the result can be
+serialized directly into the panel result's ``analysis`` section.
+This module is the scaffolding the narrative-over-stats and
+``--analysis-mode`` layers will ride on top of in follow-up work.
+"""
+
+from __future__ import annotations
+
+import math
+from collections import Counter
+from typing import Any
+
+__all__ = [
+    "InvalidResponseSchemaError",
+    "coerce_enum_value",
+    "coerce_scale_value",
+    "coerce_tagged_themes",
+    "distribution_for_question",
+]
+
+
+_RECOGNIZED_TYPES: frozenset[str] = frozenset({"text", "scale", "enum", "tagged_themes"})
+
+
+class InvalidResponseSchemaError(ValueError):
+    """Raised when a response_schema lacks a recognized ``type``.
+
+    Matches the contract in :mod:`althing.instrument`: a ``response_schema``
+    dict without a recognized ``type`` is treated as a legacy inline JSON
+    Schema by the parser. This module only computes distributions for the
+    *semantic* types, so callers must pre-filter.
+    """
+
+
+def distribution_for_question(
+    responses: list[Any],
+    response_schema: dict[str, Any],
+) -> dict[str, Any]:
+    """Compute a distribution summary for one question's responses.
+
+    Args:
+        responses: Raw per-panelist response values. Shape depends on
+            ``response_schema['type']``; unparseable entries are counted
+            as ``n_invalid``, not dropped silently.
+        response_schema: Question-level schema as validated by
+            :func:`althing.instrument._validate_response_schema`.
+
+    Returns:
+        A plain dict with at least ``type``, ``n`` (total responses),
+        ``n_valid``, and ``n_invalid``. Shape beyond that is type-specific
+        — see module docstring.
+
+    Raises:
+        InvalidResponseSchemaError: If ``response_schema`` is missing a
+            recognized ``type``.
+    """
+    t = response_schema.get("type") if isinstance(response_schema, dict) else None
+    if t not in _RECOGNIZED_TYPES:
+        raise InvalidResponseSchemaError(f"response_schema must have type in {sorted(_RECOGNIZED_TYPES)}, got {t!r}")
+
+    n = len(responses)
+    if t == "scale":
+        return _scale_distribution(responses, response_schema, n)
+    if t == "enum":
+        return _enum_distribution(responses, response_schema, n)
+    if t == "tagged_themes":
+        return _tagged_themes_distribution(responses, response_schema, n)
+    return _text_distribution(responses, n)
+
+
+# ---------------------------------------------------------------------------
+# Scale
+# ---------------------------------------------------------------------------
+
+
+def coerce_scale_value(value: Any, lo: int, hi: int) -> int | None:
+    """Return a clamped integer scale value, or ``None`` if uncoerceable.
+
+    Accepts ints, floats (rounded half-to-even per ``round()``),
+    and numeric strings. Values outside [lo, hi] return ``None`` —
+    the caller decides whether to clamp or count as invalid.
+    """
+    if isinstance(value, bool):
+        return None
+    if isinstance(value, int):
+        iv = value
+    elif isinstance(value, float):
+        if math.isnan(value) or math.isinf(value):
+            return None
+        iv = round(value)
+    elif isinstance(value, str):
+        s = value.strip()
+        if not s:
+            return None
+        try:
+            iv = round(float(s))
+        except ValueError:
+            return None
+    else:
+        return None
+    if iv < lo or iv > hi:
+        return None
+    return iv
+
+
+def _scale_distribution(responses: list[Any], schema: dict[str, Any], n: int) -> dict[str, Any]:
+    lo = int(schema["min"])
+    hi = int(schema["max"])
+    coerced = [coerce_scale_value(r, lo, hi) for r in responses]
+    valid = [v for v in coerced if v is not None]
+    n_valid = len(valid)
+    n_invalid = n - n_valid
+
+    # Zero-filled frequency table across the full integer range so callers
+    # can render a histogram without post-processing.
+    freq_counter = Counter(valid)
+    distribution = [{"value": v, "count": freq_counter.get(v, 0)} for v in range(lo, hi + 1)]
+
+    stats: dict[str, Any] = {
+        "min": lo,
+        "max": hi,
+        "n_valid": n_valid,
+        "n_invalid": n_invalid,
+    }
+    if n_valid:
+        mean = sum(valid) / n_valid
+        stats["mean"] = round(mean, 4)
+        stats["median"] = _median(valid)
+        if n_valid > 1:
+            var = sum((v - mean) ** 2 for v in valid) / (n_valid - 1)
+            stats["stdev"] = round(math.sqrt(var), 4)
+        else:
+            stats["stdev"] = 0.0
+        stats["observed_min"] = min(valid)
+        stats["observed_max"] = max(valid)
+
+    return {
+        "type": "scale",
+        "n": n,
+        "n_valid": n_valid,
+        "n_invalid": n_invalid,
+        "distribution": distribution,
+        "stats": stats,
+    }
+
+
+def _median(values: list[int]) -> float:
+    s = sorted(values)
+    m = len(s)
+    mid = m // 2
+    if m % 2:
+        return float(s[mid])
+    return round((s[mid - 1] + s[mid]) / 2, 4)
+
+
+# ---------------------------------------------------------------------------
+# Enum
+# ---------------------------------------------------------------------------
+
+
+def coerce_enum_value(value: Any, options: list[str]) -> str | None:
+    """Match a free response to an enum option (case-insensitive, trimmed).
+
+    Returns the canonical option string as declared in ``options`` when a
+    match is found, or ``None`` otherwise. An exact match always wins over
+    a case-insensitive one.
+    """
+    if not isinstance(value, str):
+        return None
+    s = value.strip()
+    if not s:
+        return None
+    if s in options:
+        return s
+    lowered = s.lower()
+    for opt in options:
+        if opt.lower() == lowered:
+            return opt
+    return None
+
+
+def _enum_distribution(responses: list[Any], schema: dict[str, Any], n: int) -> dict[str, Any]:
+    options: list[str] = list(schema["options"])
+    coerced = [coerce_enum_value(r, options) for r in responses]
+    valid = [v for v in coerced if v is not None]
+    n_valid = len(valid)
+    n_invalid = n - n_valid
+
+    counter = Counter(valid)
+    distribution = [{"option": opt, "count": counter.get(opt, 0)} for opt in options]
+
+    top_option: str | None = None
+    if n_valid:
+        # Stable: options are iterated in declaration order — the first
+        # option that ties for the max count is returned.
+        top_option = max(options, key=lambda o: counter.get(o, 0))
+        if counter.get(top_option, 0) == 0:
+            top_option = None
+
+    return {
+        "type": "enum",
+        "n": n,
+        "n_valid": n_valid,
+        "n_invalid": n_invalid,
+        "options": options,
+        "distribution": distribution,
+        "top_option": top_option,
+    }
+
+
+# ---------------------------------------------------------------------------
+# Tagged themes
+# ---------------------------------------------------------------------------
+
+
+def coerce_tagged_themes(value: Any, taxonomy: list[str], *, multi: bool) -> tuple[list[str], list[str]] | None:
+    """Split a response into (in-taxonomy tags, off-taxonomy tags).
+
+    Accepts a list of strings, a single string (which becomes a 1-element
+    list), or a delimited string (comma / semicolon / pipe / newline). When
+    ``multi`` is False, only the first tag is kept. Returns ``None`` if the
+    input cannot be coerced into any tag list.
+    """
+    if value is None:
+        return None
+    if isinstance(value, str):
+        parts: list[str] = []
+        buf = value
+        for delim in (",", ";", "|", "\n"):
+            buf = buf.replace(delim, "\x00")
+        parts = [p.strip() for p in buf.split("\x00") if p.strip()]
+    elif isinstance(value, (list, tuple)):
+        parts = [str(p).strip() for p in value if str(p).strip()]
+    else:
+        return None
+
+    if not parts:
+        return None
+
+    if not multi:
+        parts = parts[:1]
+
+    lowered = {tag.lower(): tag for tag in taxonomy}
+    in_tax: list[str] = []
+    off_tax: list[str] = []
+    for p in parts:
+        canon = lowered.get(p.lower())
+        if canon is not None:
+            if canon not in in_tax:
+                in_tax.append(canon)
+        else:
+            if p not in off_tax:
+                off_tax.append(p)
+    return in_tax, off_tax
+
+
+def _tagged_themes_distribution(responses: list[Any], schema: dict[str, Any], n: int) -> dict[str, Any]:
+    taxonomy: list[str] = list(schema["taxonomy"])
+    multi: bool = bool(schema.get("multi", False))
+
+    tag_counter: Counter[str] = Counter()
+    off_counter: Counter[str] = Counter()
+    n_valid = 0
+    total_mentions = 0
+    for r in responses:
+        coerced = coerce_tagged_themes(r, taxonomy, multi=multi)
+        if coerced is None:
+            continue
+        in_tax, off_tax = coerced
+        if in_tax or off_tax:
+            n_valid += 1
+            total_mentions += len(in_tax) + len(off_tax)
+        for tag in in_tax:
+            tag_counter[tag] += 1
+        for tag in off_tax:
+            off_counter[tag] += 1
+
+    n_invalid = n - n_valid
+    distribution = [{"theme": tag, "count": tag_counter.get(tag, 0)} for tag in taxonomy]
+    off_taxonomy = [
+        {"theme": tag, "count": count} for tag, count in sorted(off_counter.items(), key=lambda kv: (-kv[1], kv[0]))
+    ]
+
+    return {
+        "type": "tagged_themes",
+        "n": n,
+        "n_valid": n_valid,
+        "n_invalid": n_invalid,
+        "multi": multi,
+        "taxonomy": taxonomy,
+        "distribution": distribution,
+        "off_taxonomy": off_taxonomy,
+        "total_mentions": total_mentions,
+    }
+
+
+# ---------------------------------------------------------------------------
+# Text
+# ---------------------------------------------------------------------------
+
+
+def _text_distribution(responses: list[Any], n: int) -> dict[str, Any]:
+    n_nonempty = sum(
+        1 for r in responses if (isinstance(r, str) and r.strip()) or (r is not None and not isinstance(r, str))
+    )
+    return {
+        "type": "text",
+        "n": n,
+        "n_valid": n_nonempty,
+        "n_invalid": n - n_nonempty,
+    }