eq-layer 0.1.0__py3-none-any.whl

eq_layer/__init__.py

@@ -0,0 +1,68 @@
+__version__ = "0.1.0"
+
+from .affect import AffectAdapter, HeuristicAffect, load_cases
+from .auditor import AuditContext, AuditIssue, ResponseAudit, ResponseAuditor, audit_from_generation_metadata
+from .actions import FactoredAction, RealizationControls, compose_action
+from .intent import HeuristicIntent, IntentAdapter, IntentState
+from .intent_belief import IntentBelief, IntentRiskDecision, decide_intent_action, expected_loss
+from .interaction import InteractionQualityState, RepairState, infer_interaction_quality, infer_repair_state
+from .policies import POLICIES, REGISTRY, SUBTEXTS, AffectState, Policy, Register, Selector
+from .preferences import SessionPreferences, infer_session_preferences
+from .response_experiment import CommandModel, FullEQPipeline, generate_pairs, validate_experiment_cases
+from .steer import Steer, aggregate, score_case
+from .trained_affect import DimensionalAffect, TrainedAffect
+from .trained_intent import TrainedIntent
+from .trained_subtext import DialogueSignals, SubtextDecision, TrainedSubtext
+from .tracker import AnnotationStanceResolver, ConversationTracker, StanceDecision, TrackingResult
+
+__all__ = [
+    "__version__",
+    "AffectAdapter",
+    "AffectState",
+    "audit_from_generation_metadata",
+    "ResponseAuditor",
+    "ResponseAudit",
+    "AuditIssue",
+    "AuditContext",
+    "compose_action",
+    "RealizationControls",
+    "FactoredAction",
+    "HeuristicAffect",
+    "HeuristicIntent",
+    "IntentAdapter",
+    "IntentState",
+    "expected_loss",
+    "decide_intent_action",
+    "IntentRiskDecision",
+    "IntentBelief",
+    "infer_repair_state",
+    "infer_interaction_quality",
+    "RepairState",
+    "InteractionQualityState",
+    "POLICIES",
+    "Policy",
+    "REGISTRY",
+    "Register",
+    "SUBTEXTS",
+    "Selector",
+    "infer_session_preferences",
+    "SessionPreferences",
+    "Steer",
+    "TrainedIntent",
+    "TrainedAffect",
+    "DimensionalAffect",
+    "aggregate",
+    "load_cases",
+    "TrainedSubtext",
+    "DialogueSignals",
+    "SubtextDecision",
+    "ConversationTracker",
+    "TrackingResult",
+    "StanceDecision",
+    "AnnotationStanceResolver",
+    "CommandModel",
+    "FullEQPipeline",
+    "generate_pairs",
+    "validate_experiment_cases",
+    "score_case",
+]

eq_layer/ab_eval.py

@@ -0,0 +1,302 @@
+from __future__ import annotations
+
+import json
+import math
+import random
+from dataclasses import dataclass
+from pathlib import Path
+
+
+DIMENSIONS = (
+    "intent_fidelity",
+    "appropriateness",
+    "actionability",
+    "non_patronizing",
+    "overall",
+)
+
+
+def load_jsonl(path: str | Path) -> list[dict]:
+    with open(path, encoding="utf-8") as fh:
+        return [json.loads(line) for line in fh if line.strip()]
+
+
+def write_jsonl(path: str | Path, rows: list[dict]) -> None:
+    with open(path, "w", encoding="utf-8") as fh:
+        for row in rows:
+            fh.write(json.dumps(row, ensure_ascii=False) + "\n")
+
+
+def prepare_blinded(
+    pairs: list[dict],
+    *,
+    seed: int = 0,
+) -> tuple[list[dict], dict]:
+    """Randomize baseline/EQ sides and return ballot + separate decode key."""
+    rng = random.Random(seed)
+    ballot: list[dict] = []
+    key: dict[str, dict[str, str]] = {}
+
+    seen: set[str] = set()
+    for row in pairs:
+        case_id = str(row["id"])
+        if case_id in seen:
+            raise ValueError(f"Duplicate A/B case id: {case_id}")
+        seen.add(case_id)
+
+        baseline = str(row["baseline"])
+        eq_response = str(row["eq"])
+        if not baseline.strip() or not eq_response.strip():
+            raise ValueError(f"Empty response in A/B case: {case_id}")
+
+        eq_is_a = bool(rng.getrandbits(1))
+        response_a = eq_response if eq_is_a else baseline
+        response_b = baseline if eq_is_a else eq_response
+        key[case_id] = {
+            "A": "eq" if eq_is_a else "baseline",
+            "B": "baseline" if eq_is_a else "eq",
+        }
+        ballot.append(
+            {
+                "id": case_id,
+                "context": row.get("context", []),
+                "response_A": response_a,
+                "response_B": response_b,
+                "ratings": {dimension: "" for dimension in DIMENSIONS},
+            }
+        )
+
+    return ballot, key
+
+
+@dataclass(frozen=True)
+class PreferenceSummary:
+    wins: int
+    losses: int
+    ties: int
+    win_rate_non_tie: float | None
+    wilson_low: float | None
+    wilson_high: float | None
+    exact_p_two_sided: float | None
+
+
+def _wilson_interval(successes: int, total: int, z: float = 1.959963984540054) -> tuple[float, float]:
+    if total <= 0:
+        raise ValueError("Wilson interval requires total > 0.")
+    p = successes / total
+    denom = 1.0 + (z * z) / total
+    centre = (p + (z * z) / (2.0 * total)) / denom
+    margin = (
+        z
+        * math.sqrt((p * (1.0 - p) + (z * z) / (4.0 * total)) / total)
+        / denom
+    )
+    return max(0.0, centre - margin), min(1.0, centre + margin)
+
+
+def _binom_two_sided_half(successes: int, total: int) -> float:
+    """Exact two-sided sign test under p=0.5, summing equally/less likely outcomes."""
+    if total <= 0:
+        raise ValueError("Binomial test requires total > 0.")
+    observed = math.comb(total, successes)
+    numerator = 0
+    for k in range(total + 1):
+        ways = math.comb(total, k)
+        if ways <= observed:
+            numerator += ways
+    return min(1.0, numerator / (2 ** total))
+
+
+def summarize(wins: int, losses: int, ties: int) -> PreferenceSummary:
+    non_ties = wins + losses
+    if non_ties == 0:
+        return PreferenceSummary(
+            wins=wins,
+            losses=losses,
+            ties=ties,
+            win_rate_non_tie=None,
+            wilson_low=None,
+            wilson_high=None,
+            exact_p_two_sided=None,
+        )
+
+    low, high = _wilson_interval(wins, non_ties)
+    return PreferenceSummary(
+        wins=wins,
+        losses=losses,
+        ties=ties,
+        win_rate_non_tie=round(wins / non_ties, 4),
+        wilson_low=round(low, 4),
+        wilson_high=round(high, 4),
+        exact_p_two_sided=round(_binom_two_sided_half(wins, non_ties), 6),
+    )
+
+
+def score_blinded(rated_ballot: list[dict], key: dict) -> dict:
+    counts = {
+        dimension: {"wins": 0, "losses": 0, "ties": 0}
+        for dimension in DIMENSIONS
+    }
+
+    for row in rated_ballot:
+        case_id = str(row["id"])
+        if case_id not in key:
+            raise ValueError(f"Missing decode key for case: {case_id}")
+
+        ratings = row.get("ratings") or {}
+        for dimension in DIMENSIONS:
+            vote = str(ratings.get(dimension, "")).strip().upper()
+            if vote not in {"A", "B", "TIE"}:
+                raise ValueError(
+                    f"{case_id}/{dimension}: expected A, B, or tie; got {vote!r}"
+                )
+
+            if vote == "TIE":
+                counts[dimension]["ties"] += 1
+                continue
+
+            winner = key[case_id][vote]
+            if winner == "eq":
+                counts[dimension]["wins"] += 1
+            elif winner == "baseline":
+                counts[dimension]["losses"] += 1
+            else:
+                raise ValueError(f"Invalid decode key target: {winner}")
+
+    report = {
+        "design": "blind-pairwise-human-evaluation",
+        "unit": "case-level paired preference",
+        "dimensions": {},
+    }
+    for dimension, values in counts.items():
+        summary = summarize(**values)
+        report["dimensions"][dimension] = {
+            "eq_wins": summary.wins,
+            "eq_losses": summary.losses,
+            "ties": summary.ties,
+            "eq_win_rate_non_tie": summary.win_rate_non_tie,
+            "wilson_95_low": summary.wilson_low,
+            "wilson_95_high": summary.wilson_high,
+            "exact_sign_test_p_two_sided": summary.exact_p_two_sided,
+        }
+    return report
+
+
+def _decoded_vote(row: dict, key: dict, dimension: str) -> str:
+    case_id = str(row["id"])
+    if case_id not in key:
+        raise ValueError(f"Missing decode key for case: {case_id}")
+    ratings = row.get("ratings") or {}
+    vote = str(ratings.get(dimension, "")).strip().upper()
+    if vote == "TIE":
+        return "tie"
+    if vote not in {"A", "B"}:
+        raise ValueError(
+            f"{case_id}/{dimension}: expected A, B, or tie; got {vote!r}"
+        )
+    winner = key[case_id][vote]
+    if winner not in {"eq", "baseline"}:
+        raise ValueError(f"Invalid decode key target: {winner}")
+    return winner
+
+
+def _fleiss_kappa(case_votes: list[list[str]]) -> float | None:
+    if not case_votes:
+        return None
+    n_raters = len(case_votes[0])
+    if n_raters < 2 or any(len(votes) != n_raters for votes in case_votes):
+        return None
+
+    categories = ("eq", "baseline", "tie")
+    p_i = []
+    totals = {category: 0 for category in categories}
+
+    for votes in case_votes:
+        counts = {category: votes.count(category) for category in categories}
+        for category in categories:
+            totals[category] += counts[category]
+        agreement = (
+            sum(count * count for count in counts.values()) - n_raters
+        ) / (n_raters * (n_raters - 1))
+        p_i.append(agreement)
+
+    p_bar = sum(p_i) / len(p_i)
+    denom = len(case_votes) * n_raters
+    marginals = {
+        category: totals[category] / denom
+        for category in categories
+    }
+    p_e = sum(value * value for value in marginals.values())
+    if math.isclose(1.0 - p_e, 0.0):
+        return None
+    return round((p_bar - p_e) / (1.0 - p_e), 4)
+
+
+def score_multiple_blinded(
+    rated_ballots: dict[str, list[dict]],
+    key: dict,
+) -> dict:
+    """Aggregate multiple blinded raters without treating rater-votes as cases."""
+    if len(rated_ballots) < 2:
+        raise ValueError("Multi-rater scoring requires at least two raters.")
+
+    case_ids: list[str] | None = None
+    rows_by_rater: dict[str, dict[str, dict]] = {}
+
+    for rater_id, ballot in rated_ballots.items():
+        mapping = {str(row["id"]): row for row in ballot}
+        if len(mapping) != len(ballot):
+            raise ValueError(f"Duplicate case id in rater ballot: {rater_id}")
+        current_ids = sorted(mapping)
+        if case_ids is None:
+            case_ids = current_ids
+        elif current_ids != case_ids:
+            raise ValueError("All raters must score the same case ids.")
+        rows_by_rater[rater_id] = mapping
+
+    assert case_ids is not None
+    report = {
+        "design": "blind-pairwise-multi-rater-human-evaluation",
+        "n_raters": len(rated_ballots),
+        "n_cases": len(case_ids),
+        "dimensions": {},
+        "rater_reports": {},
+    }
+
+    for rater_id, ballot in rated_ballots.items():
+        report["rater_reports"][rater_id] = score_blinded(ballot, key)
+
+    for dimension in DIMENSIONS:
+        majority_wins = 0
+        majority_losses = 0
+        majority_ties = 0
+        per_case_votes: list[list[str]] = []
+
+        for case_id in case_ids:
+            votes = [
+                _decoded_vote(rows_by_rater[rater_id][case_id], key, dimension)
+                for rater_id in sorted(rows_by_rater)
+            ]
+            per_case_votes.append(votes)
+            eq_votes = votes.count("eq")
+            baseline_votes = votes.count("baseline")
+            if eq_votes > baseline_votes:
+                majority_wins += 1
+            elif baseline_votes > eq_votes:
+                majority_losses += 1
+            else:
+                majority_ties += 1
+
+        summary = summarize(majority_wins, majority_losses, majority_ties)
+        report["dimensions"][dimension] = {
+            "case_majority_eq_wins": summary.wins,
+            "case_majority_eq_losses": summary.losses,
+            "case_majority_ties": summary.ties,
+            "case_majority_eq_win_rate_non_tie": summary.win_rate_non_tie,
+            "wilson_95_low": summary.wilson_low,
+            "wilson_95_high": summary.wilson_high,
+            "exact_sign_test_p_two_sided": summary.exact_p_two_sided,
+            "fleiss_kappa": _fleiss_kappa(per_case_votes),
+        }
+
+    return report

eq_layer/actions.py

@@ -0,0 +1,242 @@
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass
+
+from .dialogue_reference import DialogueReferenceState
+from .intent import IntentState
+from .interaction import InteractionQualityState, RepairState
+from .policies import Policy, Register
+
+
+@dataclass(frozen=True)
+class RealizationControls:
+    """Surface-form controls kept separate from task/social/repair decisions."""
+
+    verbosity: str = "normal"
+    directness: str = "normal"
+    warmth: str = "normal"
+    question_budget: int = 0
+    scope_limited: bool = False
+    no_guess: bool = False
+
+
+@dataclass(frozen=True)
+class FactoredAction:
+    """Inspectable response action compiled from intent and dialogue state.
+
+    task_move is driven primarily by the user's explicit goal.
+    social_move and repair_move are driven by conversational state/policy.
+    This prevents affective adaptation from silently replacing the task.
+    """
+
+    task_move: str
+    social_move: str
+    repair_move: str
+    realization: RealizationControls
+    source_policy: str
+
+    def to_dict(self) -> dict:
+        return asdict(self)
+
+
+def _task_move(
+    intent: IntentState | None,
+    policy: Policy,
+    repair: RepairState | None = None,
+    reference: DialogueReferenceState | None = None,
+) -> str:
+    if repair is not None and repair.active:
+        if repair.replacement_explicit:
+            return "resume_prior_task_with_correction"
+        return "clarify_correction"
+
+    if reference is not None and reference.active:
+        if not reference.resolved:
+            return "clarify_reference"
+        return {
+            "continue": "continue_referenced_goal",
+            "execute_reference": "execute_referenced_goal",
+            "revise_reference": "revise_referenced_output",
+        }.get(reference.form, "respond_contextually")
+
+    if intent is not None:
+        if intent.needs_clarification:
+            return "clarify_goal"
+        return {
+            "action_request": "execute_request",
+            "status_check": "report_status",
+            "explanation": "explain",
+            "question": "answer_question",
+            "statement": "respond_contextually",
+            "unknown": "clarify_goal",
+        }.get(intent.kind, "respond_contextually")
+
+    return {
+        "execute_request": "execute_request",
+        "report_status": "report_status",
+        "explain_request": "explain",
+        "clarify_request": "clarify_goal",
+        "direct_no_padding": "answer_question",
+        "boundary": "set_boundary",
+    }.get(policy.name, "respond_contextually")
+
+
+SOCIAL_MOVES = {
+    "mirror_specific": "mirror_specific",
+    "validate_then_redirect": "validate_specific",
+    "deflate_tension": "deflate_tension",
+    "hold": "minimal_presence",
+    "boundary": "boundary_neutral",
+    "hold_sustained_escalation": "low_warmth",
+}
+
+REPAIR_MOVES = {
+    "clarify_request": "clarify_goal",
+    "ask_one_question": "clarify_one",
+    "repair_hold_position": "hold_position",
+    "repair_interrogation": "stop_restatement",
+}
+
+
+def _repair_move(policy: Policy, repair: RepairState | None) -> str:
+    base = REPAIR_MOVES.get(policy.name, "none")
+    if repair is None or not repair.active:
+        return base
+    if repair.replacement_explicit:
+        if repair.repeated:
+            return "stop_restatement_and_apply_correction"
+        return "apply_correction"
+    if repair.repeated:
+        return "stop_restatement_and_repair"
+    if base != "none":
+        return base
+    return "clarify_one"
+
+
+def _realization(
+    policy: Policy,
+    intent: IntentState | None,
+    *,
+    repair_move: str,
+    repair: RepairState | None = None,
+    interaction_quality: InteractionQualityState | None = None,
+    reference: DialogueReferenceState | None = None,
+) -> RealizationControls:
+    by_register = {
+        Register.MIRROR: ("normal", "normal", "normal"),
+        Register.VALIDATE_REDIRECT: ("normal", "high", "normal"),
+        Register.DIRECT: ("normal", "high", "low"),
+        Register.ASK: ("low", "high", "normal"),
+        Register.DEFLATE: ("low", "high", "low"),
+        Register.REPAIR: ("low", "high", "low"),
+        Register.HOLD: ("low", "normal", "low"),
+        Register.BOUNDARY: ("low", "high", "low"),
+    }
+    verbosity, directness, warmth = by_register[policy.register]
+
+    constraints = set(intent.constraints) if intent is not None else set()
+    if "brief" in constraints:
+        verbosity = "low"
+    if "detailed" in constraints:
+        verbosity = "high"
+    if "direct" in constraints:
+        directness = "high"
+
+    # Interaction quality is not affect. Only a strong structural failure
+    # signal changes realization, and it does so conservatively.
+    if interaction_quality is not None and interaction_quality.current <= 0.55:
+        verbosity = "low"
+        directness = "high"
+        warmth = "low"
+
+    explicit_replacement = (
+        repair is not None
+        and repair.active
+        and repair.replacement_explicit
+    )
+    repair_needs_clarification = (
+        repair is not None
+        and repair.active
+        and not repair.replacement_explicit
+    )
+    reference_needs_clarification = (
+        reference is not None
+        and reference.active
+        and not reference.resolved
+    )
+    question_budget = 1 if (
+        explicit_replacement
+        or repair_needs_clarification
+        or reference_needs_clarification
+        or repair_move in {"clarify_goal", "clarify_one"}
+        or (
+            intent is not None
+            and intent.needs_clarification
+            and not explicit_replacement
+        )
+    ) else 0
+
+    no_guess = (
+        "no_guess" in constraints
+        or repair_needs_clarification
+        or reference_needs_clarification
+        or (
+            intent is not None
+            and intent.needs_clarification
+            and not explicit_replacement
+        )
+    )
+
+    # An active correction identifies a repair target, but does not prove the
+    # user's factual stance. Do not promote it to user_right/user_wrong.
+    if repair is not None and repair.active and repair_move == "clarify_one":
+        no_guess = True
+
+    return RealizationControls(
+        verbosity=verbosity,
+        directness=directness,
+        warmth=warmth,
+        question_budget=question_budget,
+        scope_limited="scope_limited" in constraints,
+        no_guess=no_guess,
+    )
+
+
+def compose_action(
+    policy: Policy,
+    intent: IntentState | None = None,
+    *,
+    repair: RepairState | None = None,
+    interaction_quality: InteractionQualityState | None = None,
+    reference: DialogueReferenceState | None = None,
+) -> FactoredAction:
+    """Compile policy + intent + dialogue-health state into orthogonal controls."""
+
+    task_move = _task_move(intent, policy, repair, reference)
+    if policy.name == "boundary":
+        task_move = "set_boundary"
+
+    repair_move = _repair_move(policy, repair)
+    social_move = SOCIAL_MOVES.get(policy.name, "neutral")
+
+    if (
+        interaction_quality is not None
+        and interaction_quality.current <= 0.55
+        and social_move == "neutral"
+    ):
+        social_move = "low_warmth"
+
+    return FactoredAction(
+        task_move=task_move,
+        social_move=social_move,
+        repair_move=repair_move,
+        realization=_realization(
+            policy,
+            intent,
+            repair_move=repair_move,
+            repair=repair,
+            interaction_quality=interaction_quality,
+            reference=reference,
+        ),
+        source_policy=policy.name,
+    )