skill-harness 0.2.1__py3-none-any.whl

skill_harness/__init__.py

@@ -0,0 +1,3 @@
+"""Skill Harness — clause-ablation differential testing for LLM skills."""
+
+__version__ = "0.2.0"

skill_harness/__main__.py

@@ -0,0 +1,6 @@
+"""Module entry point: ``python -m skill_harness`` == the ``skill-harness`` console script."""
+
+from skill_harness.cli.main import cli
+
+if __name__ == "__main__":
+    cli()

skill_harness/ablation/__init__.py

@@ -0,0 +1,15 @@
+"""Ablation runner — Track D.
+
+D.1 provides:
+  - AblationOperator: versioned matched-length neutral substitution (operator.py)
+  - ConditionRenderer: renders Full/Ablated_k/Null conditions (render.py)
+
+D.2 provides:
+  - AblationRunner: deterministic orchestration engine (runner.py)
+  - SubjectClient: subject model wrapper (subject.py)
+  - BetaBinomialAccumulator, StoppingReason: sequential stopping (stopping.py)
+  - NullAccumulator, detect_confounds, ConfoundEvent: confound monitoring (confound.py)
+  - reconcile_run_cost: cost reconciler (reconciler.py)
+
+D.3 will add: CLI wiring in src/skill_harness/cli/main.py
+"""

skill_harness/ablation/confound.py

@@ -0,0 +1,362 @@
+"""Confound monitoring for ablation runs (A11, A45, A46, A47).
+
+Design:
+- Score every sample on ALL registered metric_library axes in-memory.
+- sigma(Null) estimated per (run, axis) at write-time from accumulated Null scores.
+- N_null >= 30 floor: if fewer Null samples, confound detection is disabled for that axis.
+- k = 2.0: emit a confound_events row ONLY when |delta| > k * sigma_Null.
+- Threshold-triggered only -- NO dense per-sample x axis table (A47).
+- Uncalibrated Tier-2 axes excluded from sigma(Null).
+- Write-side assertion: primary_clause_id == ablated_clause_id (A46).
+
+delta_kind values (A11):
+  - 'confound_flagged': movement on a claimed-but-other-clause axis (taints primary verdict)
+  - 'observed_unclaimed_delta': movement on a non-claimed axis (audit only, never aggregated)
+
+This module is PURELY in-memory computation -- storage writes happen in the runner via
+insert_confound_event(). The confound monitor returns events for the runner to persist.
+
+Per A45: confound stays two-table; exclusion is the read-time VIEW (admissible_verdicts),
+never a verdict-row state change. The confound_events row drives the VIEW exclusion.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import statistics
+from collections.abc import Callable
+from dataclasses import dataclass
+from typing import Final
+
+# ---------------------------------------------------------------------------
+# Constants
+# ---------------------------------------------------------------------------
+
+N_NULL_FLOOR: Final[int] = 30
+"""Minimum Null samples before sigma(Null) is reliable enough for confound detection."""
+
+K_THRESHOLD: Final[float] = 2.0
+"""Sigma multiplier for confound threshold (A47)."""
+
+# Standard Tier-1 axis names (matches function names in tier1 modules)
+AXIS_VERBOSITY: Final[str] = "verbosity"
+AXIS_HEDGE_INDEX: Final[str] = "hedge_index"
+AXIS_STRUCTURE_SCORE: Final[str] = "structure_score"
+AXIS_COMPLIANCE_PROXY: Final[str] = "compliance_proxy"
+AXIS_CITATION_PRESENCE_PER_FLAG: Final[str] = "citation_presence_per_flag"
+
+# ---------------------------------------------------------------------------
+# Data types
+# ---------------------------------------------------------------------------
+
+
+@dataclass(frozen=True)
+class ConfoundEvent:
+    """A detected confound event (threshold-triggered, A47).
+
+    Does NOT map 1:1 to a DB row -- the runner calls insert_confound_event().
+    This is the pure-computation output of the confound monitor.
+    """
+
+    primary_clause_id: str
+    """The ablated clause whose verdict is being evaluated (A46: == ablated_clause_id)."""
+
+    affected_clause_id: str | None
+    """The clause that owns the axis that moved (None for unclaimed axes)."""
+
+    axis: str
+    """The metric axis name."""
+
+    delta: float
+    """Observed delta (Full score - Ablated_k score) for this axis."""
+
+    null_sigma: float
+    """Estimated sigma(Null) for this axis at time of detection."""
+
+    k_threshold: float
+    """The k multiplier used (default K_THRESHOLD = 2.0)."""
+
+    delta_kind: str
+    """'confound_flagged' or 'observed_unclaimed_delta'."""
+
+
+# ---------------------------------------------------------------------------
+# Metric scorer registry (injected by runner, not module-level globals)
+# ---------------------------------------------------------------------------
+
+# Type alias for a metric scoring function
+MetricFn = Callable[[str], float]
+
+
+def get_default_tier1_scorers() -> dict[str, MetricFn]:
+    """Return the default set of Tier-1 metric scoring functions.
+
+    Only imports when called (avoids import-time side effects in tests).
+    These are the Tier-1 metrics available per A14/A33.
+    """
+    from skill_harness.oracles.tier1.citation_presence_per_flag import (
+        compute_citation_presence_per_flag,
+    )
+    from skill_harness.oracles.tier1.compliance_proxy import compute_compliance_proxy
+    from skill_harness.oracles.tier1.hedge_index import compute_hedge_index
+    from skill_harness.oracles.tier1.structure_score import compute_structure_score
+    from skill_harness.oracles.tier1.verbosity import count_tokens
+
+    return {
+        AXIS_VERBOSITY: count_tokens,
+        AXIS_HEDGE_INDEX: compute_hedge_index,
+        AXIS_STRUCTURE_SCORE: compute_structure_score,
+        AXIS_COMPLIANCE_PROXY: compute_compliance_proxy,
+        AXIS_CITATION_PRESENCE_PER_FLAG: compute_citation_presence_per_flag,
+    }
+
+
+# ---------------------------------------------------------------------------
+# Score computation
+# ---------------------------------------------------------------------------
+
+
+def score_text_on_axes(text: str, scorers: dict[str, MetricFn]) -> dict[str, float]:
+    """Score a text on all provided metric axes.
+
+    :param text: The output text to score.
+    :param scorers: Dict mapping axis_name -> scoring function.
+    :returns: Dict mapping axis_name -> float score.
+    """
+    import contextlib
+
+    scores: dict[str, float] = {}
+    for axis, fn in scorers.items():
+        with contextlib.suppress(Exception):
+            scores[axis] = float(fn(text))
+    return scores
+
+
+# ---------------------------------------------------------------------------
+# NullAccumulator
+# ---------------------------------------------------------------------------
+
+
+class NullAccumulator:
+    """Collects Null-condition scores for sigma(Null) estimation across all clauses.
+
+    The runner collects Null samples across ALL clauses in a run so that sigma(Null)
+    is estimated with sufficient power (N >= N_NULL_FLOOR per axis, A47).
+    """
+
+    def __init__(
+        self,
+        scorers: dict[str, MetricFn] | None = None,
+        null_floor: int = N_NULL_FLOOR,
+    ) -> None:
+        """
+        :param scorers: Metric scoring functions (injected; defaults to Tier-1 set).
+        :param null_floor: Minimum Null samples per axis before sigma is reliable.
+            Defaults to ``N_NULL_FLOOR`` (30, A47). Lowered ONLY in tests for speed —
+            production code must use the default floor.
+        """
+        self._scorers: dict[str, MetricFn] = (
+            scorers if scorers is not None else get_default_tier1_scorers()
+        )
+        self._null_floor: int = null_floor
+        self._axis_values: dict[str, list[float]] = {}
+
+    @property
+    def null_floor(self) -> int:
+        """The configured Null-sample floor for this accumulator (A47)."""
+        return self._null_floor
+
+    def add(self, text: str) -> None:
+        """Score text on all axes and accumulate for sigma estimation."""
+        scores = score_text_on_axes(text, self._scorers)
+        for axis, value in scores.items():
+            if axis not in self._axis_values:
+                self._axis_values[axis] = []
+            self._axis_values[axis].append(value)
+
+    def sigmas(self) -> dict[str, float]:
+        """Compute sigma per axis. Returns only axes with >= null_floor samples.
+
+        A6: a floor-met, zero-variance axis is returned as sigma=0.0 rather than
+        dropped. Dropping it made "zero variance" and "below floor" both read as
+        "axis absent" to callers, which let confound detection go silently inert
+        for zero-variance axes -- see ``detect_confounds`` for the consuming side.
+        """
+        result: dict[str, float] = {}
+        for axis, values in self._axis_values.items():
+            if len(values) < self._null_floor:
+                continue  # Below floor -- confound detection disabled for this axis
+            with contextlib.suppress(statistics.StatisticsError):
+                result[axis] = statistics.stdev(values)  # may be 0.0 (A6)
+        return result
+
+    def n(self) -> int:
+        """Total Null sample count (max across axes)."""
+        if not self._axis_values:
+            return 0
+        return max(len(v) for v in self._axis_values.values())
+
+    def axis_n(self, axis: str) -> int:
+        """Null sample count for a specific axis."""
+        return len(self._axis_values.get(axis, []))
+
+
+# ---------------------------------------------------------------------------
+# Confound detection (stateless -- uses pre-computed sigma)
+# ---------------------------------------------------------------------------
+
+
+def detect_confounds(
+    full_text: str,
+    ablated_text: str,
+    null_sigmas: dict[str, float],
+    primary_clause_id: str,
+    clause_to_axis_map: dict[str, str],
+    scorers: dict[str, MetricFn] | None = None,
+    k: float = K_THRESHOLD,
+) -> list[ConfoundEvent]:
+    """Score Full vs Ablated_k on all axes and return confound events.
+
+    Stateless: uses pre-computed sigma(Null) from a NullAccumulator.
+    Threshold-triggered only -- no dense per-sample x axis table (A47).
+
+    :param full_text: Full-condition subject output.
+    :param ablated_text: Ablated_k-condition subject output.
+    :param null_sigmas: Pre-computed sigma(Null) per axis.
+    :param primary_clause_id: The ablated clause ID (A46 write-side assertion).
+    :param clause_to_axis_map: Maps clause_id -> claimed axis name.
+    :param scorers: Metric scoring functions (defaults to Tier-1 set).
+    :param k: Sigma multiplier.
+    :returns: List of ConfoundEvent (threshold-triggered only, A47).
+    """
+    if scorers is None:
+        scorers = get_default_tier1_scorers()
+
+    full_scores = score_text_on_axes(full_text, scorers)
+    ablated_scores = score_text_on_axes(ablated_text, scorers)
+
+    # Invert clause_to_axis_map: axis -> clause_id (for delta classification)
+    axis_to_clause: dict[str, str] = {ax: cid for cid, ax in clause_to_axis_map.items()}
+
+    events: list[ConfoundEvent] = []
+    all_axes = set(full_scores.keys()) & set(ablated_scores.keys())
+
+    for axis in sorted(all_axes):  # sorted for determinism
+        sigma = null_sigmas.get(axis)
+        if sigma is None:
+            continue  # Below N_null floor -- genuinely insufficient Null data (A47)
+
+        delta = full_scores[axis] - ablated_scores[axis]
+        # A6: when sigma == 0.0 (floor met, zero variance under Null), k * sigma == 0,
+        # so this only screens out an EXACT-zero delta. Any nonzero delta falls
+        # through and is flagged -- a zero-variance Null gives no safe threshold to
+        # hide real movement behind, so confound monitoring must never go silently
+        # inert for a zero-variance axis.
+        if abs(delta) <= k * sigma:
+            continue  # Within threshold -- not a confound event
+
+        # Determine delta_kind and affected_clause_id (A11)
+        owner_clause = axis_to_clause.get(axis)
+
+        if owner_clause is None:
+            # No clause claims this axis -> observed_unclaimed_delta (audit only)
+            events.append(
+                ConfoundEvent(
+                    primary_clause_id=primary_clause_id,
+                    affected_clause_id=None,
+                    axis=axis,
+                    delta=delta,
+                    null_sigma=sigma,
+                    k_threshold=k,
+                    delta_kind="observed_unclaimed_delta",
+                )
+            )
+        elif owner_clause == primary_clause_id:
+            # Primary clause's own axis moved -- expected, not a confound
+            # (the runner's oracle measures this axis as the primary signal)
+            pass
+        else:
+            # Another clause's axis moved -> confound_flagged (taints primary verdict)
+            events.append(
+                ConfoundEvent(
+                    primary_clause_id=primary_clause_id,
+                    affected_clause_id=owner_clause,
+                    axis=axis,
+                    delta=delta,
+                    null_sigma=sigma,
+                    k_threshold=k,
+                    delta_kind="confound_flagged",
+                )
+            )
+
+    return events
+
+
+# ---------------------------------------------------------------------------
+# Directional observation conversion
+# ---------------------------------------------------------------------------
+
+
+def delta_to_observation(
+    delta: float, comparator: str = "increase", tie_tolerance: float = 0.0
+) -> float:
+    """Convert a metric delta to a Win/Tie/Loss observation (A10 half-update).
+
+    :param delta: Full score - Ablated_k score on the primary axis.
+    :param comparator: The clause's persisted directional claim (schema CHECK:
+        increase|decrease|match). Determines which sign of delta counts as a Win:
+        - 'increase': Win = Full > Ablated (delta > 0). The clause claims the axis
+          rises -- this is the original, pre-A2 behavior and the default.
+        - 'decrease': Win = Full < Ablated (delta < 0). The clause claims the axis
+          falls, so the sign is flipped relative to 'increase' -- an effective
+          decrease-direction clause (e.g. "be concise" on verbosity) must verdict
+          Win, not Loss (A2).
+        - 'match'/anything else: treated as 'increase'. A "preserve" claim is a
+          magnitude-of-change question, not a delta-sign question, and defining that
+          comparison model is out of scope here -- this is a deliberate placeholder,
+          not a considered design decision.
+    :param tie_tolerance: Absolute tolerance for tie classification (0.0 = strict equality).
+    :returns: 1.0 (Win), 0.5 (Tie), 0.0 (Loss).
+    """
+    if abs(delta) <= tie_tolerance:
+        return 0.5
+    if comparator == "decrease":
+        return 1.0 if delta < 0 else 0.0
+    return 1.0 if delta > 0 else 0.0
+
+
+# ---------------------------------------------------------------------------
+# QUAL-1: Length-confound detection for sub-tolerance clauses
+# ---------------------------------------------------------------------------
+
+
+def check_operator_length_tolerance(
+    clause_text: str,
+    placeholder_text: str,
+    tolerance_fraction: float = 0.10,
+    min_tolerance_tokens: int = 2,
+) -> tuple[bool, int, int, int]:
+    """Check whether the AblationOperator met its length tolerance for a clause.
+
+    QUAL-1 carry-forward from D.1: [ABLATED] is 4 tokens, so 1-2-token clauses
+    cannot achieve the +/-2-token minimum tolerance. The runner must detect this
+    and flag the Ablated_k samples as length-confounded (never emit clean-looking
+    verbosity evidence from them).
+
+    :param clause_text: Original clause text.
+    :param placeholder_text: AblationOperator placeholder for this clause.
+    :param tolerance_fraction: Token length tolerance fraction (default 10%).
+    :param min_tolerance_tokens: Minimum tolerance in tokens (default 2).
+    :returns: (within_tolerance, clause_tokens, placeholder_tokens, tolerance)
+    """
+    # F4: delegate to the canonical cl100k_base counter (tier1/verbosity.py)
+    # instead of a third local tiktoken.get_encoding() re-implementation —
+    # this module already imports the same counter inside
+    # get_default_tier1_scorers(); a second copy here was the drift risk.
+    from skill_harness.oracles.tier1.verbosity import count_tokens
+
+    clause_tokens = count_tokens(clause_text)
+    placeholder_tokens = count_tokens(placeholder_text)
+    tolerance = max(min_tolerance_tokens, int(clause_tokens * tolerance_fraction))
+    within = abs(placeholder_tokens - clause_tokens) <= tolerance
+    return within, clause_tokens, placeholder_tokens, tolerance

skill_harness/ablation/operator.py

@@ -0,0 +1,198 @@
+"""AblationOperator — versioned matched-length neutral substitution (A39).
+
+Design:
+- Produces a **deterministic, byte-stable, matched-length, semantically-null**
+  placeholder for a removed clause. NOT deletion (A39: deletion conflates clause
+  axis-effect with prompt coherence / length / format loss).
+- The placeholder token count approximates the removed clause's token count within
+  TOKEN_LENGTH_TOLERANCE_FRACTION (±10% or ±2 tokens, whichever is larger).
+- ``implementation_hash`` is SHA-256 of the stable implementation-configuration
+  canonical string — mirrors the Tier-1 ``metric_versions.implementation_hash``
+  discipline so re-audit is possible if the operator changes version.
+- ``ablation_operator_id`` is a stable version identifier string.
+- NEVER calls a model — the deterministic Python layer constructs the placeholder.
+- Uses ``tiktoken cl100k_base`` (version-pinned, offline) for token counting.
+
+Algorithm:
+  1. Count tokens in the input clause text (offline tiktoken).
+  2. Divide the target token count by the filler unit token count to find how
+     many filler units to repeat.
+  3. Build the placeholder by repeating the filler unit, adding/removing partial
+     units until token count is within tolerance.
+  4. Return the placeholder string.
+
+The filler unit is a semantically null, grammatically neutral phrase that:
+  - Has stable tokenization across tiktoken versions
+  - Produces no meaningful semantic signal to the subject model
+  - Has a known, stable token count (verified in ``FILLER_UNIT_TOKENS``)
+
+Encoding determinism: tiktoken encoding is deterministic; PYTHONHASHSEED=0 is
+not required but is enforced globally by the Tier-1 test suite.
+"""
+
+from __future__ import annotations
+
+import hashlib
+from typing import TYPE_CHECKING, Final
+
+from skill_harness.oracles.tier1.verbosity import ENCODING_NAME, get_encoding
+
+if TYPE_CHECKING:
+    import tiktoken
+
+# ---------------------------------------------------------------------------
+# Constants
+# ---------------------------------------------------------------------------
+
+ABLATION_OPERATOR_VERSION: Final[str] = "neutral-filler-v1"
+"""Stable version identifier for this operator implementation."""
+
+# F-7 (S55 hostile review): ENCODING_NAME is imported from
+# skill_harness.oracles.tier1.verbosity, the single source of truth for "which
+# tiktoken encoding the harness uses" -- this module previously hand-duplicated
+# its own "cl100k_base" literal (_ENCODING_NAME) and its own
+# tiktoken.get_encoding() call, independent of verbosity.py's copy. A name
+# change to the canonical encoding would have silently not propagated here.
+
+# Semantically null filler phrase. Chosen to be:
+# - Grammatically neutral (imperative-ish but content-free)
+# - Stable tokenization in cl100k_base across the 0.7.x series
+# - Not interpretable as an instruction to the subject model
+_FILLER_UNIT: Final[str] = "[ABLATED]"
+"""Base filler atom. Repeated to match target token count."""
+
+# QUAL-2: pin FILLER_UNIT_TOKENS as a module-level constant so it is assertable by callers
+# and not just private instance state. Computed once at module import time via the
+# canonical encoding (F-7: get_encoding(), not a second tiktoken.get_encoding() call).
+# This resolves the dangling docstring reference "verified in ``FILLER_UNIT_TOKENS``" (line 27).
+FILLER_UNIT_TOKENS: Final[int] = len(get_encoding().encode(_FILLER_UNIT))
+"""Stable token count of one _FILLER_UNIT atom in cl100k_base.
+
+Pinned as a module constant (QUAL-2) so callers can assert invariants and the
+docstring reference on line 27 ("verified in ``FILLER_UNIT_TOKENS``") resolves.
+Change this constant whenever _FILLER_UNIT or the canonical encoding changes.
+"""
+
+# Tolerance for matched-length: ±10% of clause token count, minimum ±2 tokens.
+TOKEN_LENGTH_TOLERANCE_FRACTION: Final[float] = 0.10
+
+# Implementation configuration string — hashed to produce implementation_hash.
+# Change this whenever the algorithm, filler text, or encoding changes.
+_IMPL_CONFIG: Final[str] = (
+    f"ablation-operator version={ABLATION_OPERATOR_VERSION} "
+    f"filler={_FILLER_UNIT!r} "
+    f"encoding={ENCODING_NAME} "
+    f"tolerance={TOKEN_LENGTH_TOLERANCE_FRACTION}"
+)
+
+
+# ---------------------------------------------------------------------------
+# AblationOperator
+# ---------------------------------------------------------------------------
+
+
+class AblationOperator:
+    """Versioned matched-length neutral substitution operator.
+
+    Parameters
+    ----------
+    None — all behaviour is fixed by the class constants above.
+
+    Usage
+    -----
+    >>> op = AblationOperator()
+    >>> placeholder = op.render("Always begin your response with 'Certainly!'.")
+    >>> op.ablation_operator_id
+    'neutral-filler-v1'
+    >>> len(op.implementation_hash)
+    64
+    """
+
+    # ------------------------------------------------------------------
+    # Class-level stable attributes (shared across instances)
+    # ------------------------------------------------------------------
+
+    ablation_operator_id: str = ABLATION_OPERATOR_VERSION
+    implementation_hash: str = hashlib.sha256(_IMPL_CONFIG.encode("utf-8")).hexdigest()
+
+    def __init__(self) -> None:
+        # F-7: the canonical encoding (verbosity.get_encoding()), not a second
+        # independent tiktoken.get_encoding() call — see module docstring note.
+        self._enc: tiktoken.Encoding = get_encoding()
+        # Token count of one filler atom — pinned by FILLER_UNIT_TOKENS module constant (QUAL-2).
+        self._filler_unit_tokens: int = FILLER_UNIT_TOKENS
+        assert self._filler_unit_tokens > 0, (
+            f"FILLER_UNIT_TOKENS must be > 0; got {self._filler_unit_tokens!r} "
+            f"(QUAL-2: assertion on module constant)"
+        )
+
+    # ------------------------------------------------------------------
+    # Public API
+    # ------------------------------------------------------------------
+
+    def render(self, clause_text: str) -> str:
+        """Return a deterministic, matched-length placeholder for ``clause_text``.
+
+        The returned placeholder:
+        - Is byte-equal across calls with the same input (deterministic).
+        - Has a token count within TOKEN_LENGTH_TOLERANCE_FRACTION of the
+          clause's token count (±10% or ±2 tokens, whichever is larger).
+        - Does NOT contain the original clause text.
+        - Is semantically null — conveys no axis-relevant signal to the model.
+
+        :param clause_text: The original clause text being ablated.
+        :returns: The placeholder string.
+        """
+        target_tokens = len(self._enc.encode(clause_text))
+        return self._build_placeholder(target_tokens)
+
+    # ------------------------------------------------------------------
+    # Private helpers
+    # ------------------------------------------------------------------
+
+    def _build_placeholder(self, target_tokens: int) -> str:
+        """Build a placeholder string of approximately ``target_tokens`` tokens.
+
+        Algorithm:
+        1. Compute how many whole filler units fit in target_tokens.
+        2. Build an initial placeholder.
+        3. Trim if over-length, pad if under-length — both by adjusting the
+           filler repetition count — until within tolerance.
+
+        The algorithm terminates because:
+        - We start close to the target (whole-unit count).
+        - Adjustment is monotone (add or remove one unit).
+        - Tolerance guarantees a valid range exists.
+
+        For zero-token clauses the placeholder is the single filler unit
+        (we never return an empty string — NOT deletion per A39).
+        """
+        if target_tokens == 0:
+            return _FILLER_UNIT
+
+        filler_count = max(1, round(target_tokens / self._filler_unit_tokens))
+
+        # Separator between units is a space — stable token boundary
+        separator = " "
+
+        def _build(count: int) -> str:
+            return separator.join([_FILLER_UNIT] * count)
+
+        placeholder = _build(filler_count)
+        actual_tokens = len(self._enc.encode(placeholder))
+
+        tolerance = max(2, int(target_tokens * TOKEN_LENGTH_TOLERANCE_FRACTION))
+
+        # Adjust upward if under target
+        while actual_tokens < target_tokens - tolerance:
+            filler_count += 1
+            placeholder = _build(filler_count)
+            actual_tokens = len(self._enc.encode(placeholder))
+
+        # Adjust downward if over target
+        while actual_tokens > target_tokens + tolerance and filler_count > 1:
+            filler_count -= 1
+            placeholder = _build(filler_count)
+            actual_tokens = len(self._enc.encode(placeholder))
+
+        return placeholder