wisent 0.1.1__py3-none-any.whl → 0.5.1__py3-none-any.whl

wisent/core/contrastive_pairs/diagnostics/control_vectors.py

@@ -0,0 +1,169 @@
+"""Diagnostics for steering/control vectors."""
+
+from __future__ import annotations
+
+import statistics
+from dataclasses import dataclass
+from typing import Mapping
+
+import torch
+
+from wisent_guard.core.activations.core.atoms import LayerActivations, RawActivationMap
+
+from .base import DiagnosticsIssue, DiagnosticsReport, MetricReport
+
+__all__ = [
+    "ControlVectorDiagnosticsConfig",
+    "run_control_vector_diagnostics",
+    "run_control_steering_diagnostics",
+]
+
+
+@dataclass(slots=True)
+class ControlVectorDiagnosticsConfig:
+    """Thresholds and options for control vector diagnostics."""
+
+    min_norm: float = 1e-4
+    max_norm: float | None = None
+    zero_value_threshold: float = 1e-8
+    max_zero_fraction: float = 0.999
+    warn_on_missing: bool = True
+
+
+def _to_layer_activations(vectors: LayerActivations | RawActivationMap | Mapping[str, object] | None) -> LayerActivations:
+    if isinstance(vectors, LayerActivations):
+        return vectors
+    data: RawActivationMap = vectors or {}
+    return LayerActivations(data)
+
+
+def run_control_vector_diagnostics(
+    vectors: LayerActivations | RawActivationMap | Mapping[str, object] | None,
+    config: ControlVectorDiagnosticsConfig | None = None,
+) -> DiagnosticsReport:
+    """Evaluate steering/control vectors for basic health metrics."""
+
+    cfg = config or ControlVectorDiagnosticsConfig()
+    activations = _to_layer_activations(vectors)
+
+    issues: list[DiagnosticsIssue] = []
+    norms: list[float] = []
+    zero_fractions: list[float] = []
+    per_layer: dict[str, dict[str, float]] = {}
+
+    for layer, tensor in activations.to_dict().items():
+        if tensor is None:
+            if cfg.warn_on_missing:
+                issues.append(
+                    DiagnosticsIssue(
+                        metric="control_vectors",
+                        severity="warning",
+                        message=f"Layer {layer} has no control vector",
+                        details={"layer": layer},
+                    )
+                )
+            continue
+
+        detached = tensor.detach()
+        if detached.numel() == 0:
+            issues.append(
+                DiagnosticsIssue(
+                    metric="control_vectors",
+                    severity="critical",
+                    message=f"Layer {layer} control vector is empty",
+                    details={"layer": layer},
+                )
+            )
+            continue
+
+        flat = detached.to(dtype=torch.float32, device="cpu").reshape(-1)
+
+        if not torch.isfinite(flat).all():
+            non_finite = (~torch.isfinite(flat)).sum().item()
+            issues.append(
+                DiagnosticsIssue(
+                    metric="control_vectors",
+                    severity="critical",
+                    message=f"Layer {layer} contains non-finite values",
+                    details={"layer": layer, "non_finite_entries": int(non_finite)},
+                )
+            )
+            continue
+
+        norm_value = float(torch.linalg.vector_norm(flat).item())
+        norms.append(norm_value)
+
+        zero_fraction = float((flat.abs() <= cfg.zero_value_threshold).sum().item()) / float(flat.numel())
+        zero_fractions.append(zero_fraction)
+
+        per_layer[layer] = {
+            "norm": norm_value,
+            "zero_fraction": zero_fraction,
+        }
+
+        if norm_value < cfg.min_norm:
+            issues.append(
+                DiagnosticsIssue(
+                    metric="control_vectors",
+                    severity="critical",
+                    message=f"Layer {layer} control vector norm {norm_value:.3e} below minimum {cfg.min_norm}",
+                    details={"layer": layer, "norm": norm_value},
+                )
+            )
+
+        if cfg.max_norm is not None and norm_value > cfg.max_norm:
+            issues.append(
+                DiagnosticsIssue(
+                    metric="control_vectors",
+                    severity="warning",
+                    message=f"Layer {layer} control vector norm {norm_value:.3e} exceeds maximum {cfg.max_norm}",
+                    details={"layer": layer, "norm": norm_value},
+                )
+            )
+
+        if zero_fraction >= cfg.max_zero_fraction:
+            severity = "critical" if zero_fraction >= 1.0 - 1e-9 else "warning"
+            issues.append(
+                DiagnosticsIssue(
+                    metric="control_vectors",
+                    severity=severity,
+                    message=(
+                        f"Layer {layer} control vector is {zero_fraction:.3%} zero-valued, exceeding allowed {cfg.max_zero_fraction:.3%}"
+                    ),
+                    details={"layer": layer, "zero_fraction": zero_fraction},
+                )
+            )
+
+    summary: dict[str, object] = {
+        "evaluated_layers": len(norms),
+        "norm_min": min(norms) if norms else None,
+        "norm_max": max(norms) if norms else None,
+        "norm_mean": statistics.mean(norms) if norms else None,
+        "norm_median": statistics.median(norms) if norms else None,
+        "zero_fraction_max": max(zero_fractions) if zero_fractions else None,
+        "per_layer": per_layer,
+    }
+
+    if not norms and not issues:
+        issues.append(
+            DiagnosticsIssue(
+                metric="control_vectors",
+                severity="critical",
+                message="No control vectors were provided for diagnostics",
+                details={},
+            )
+        )
+
+    report = MetricReport(name="control_vectors", summary=summary, issues=issues)
+    return DiagnosticsReport.from_metrics([report])
+
+def run_control_steering_diagnostics(steering_vectors: list[RawActivationMap] | RawActivationMap | None) -> list[DiagnosticsReport]:
+    if steering_vectors is None:
+        return [DiagnosticsReport.from_metrics([])]
+
+    if not isinstance(steering_vectors, list):
+        steering_vectors = [steering_vectors]
+
+    # Run diagnostics for each steering vector
+    reports = [run_control_vector_diagnostics(vec) for vec in steering_vectors]
+    return reports

wisent/core/contrastive_pairs/diagnostics/coverage.py

@@ -0,0 +1,79 @@
+"""Coverage and diversity diagnostics for contrastive pairs."""
+
+from __future__ import annotations
+
+from statistics import mean
+from typing import Iterable, List
+
+from .base import DiagnosticsConfig, DiagnosticsIssue, MetricReport
+
+
+def compute_coverage_metrics(pairs: Iterable, config: DiagnosticsConfig) -> MetricReport:
+    """Assess dataset coverage such as prompt diversity and response length."""
+
+    pairs_list = list(pairs)
+
+    if not pairs_list:
+        return MetricReport(name="coverage", summary={"total_pairs": 0}, issues=[])
+
+    unique_prompts = {getattr(pair, "prompt", "").strip().lower() for pair in pairs_list}
+    prompt_ratio = len(unique_prompts) / len(pairs_list)
+
+    pos_lengths: List[int] = []
+    neg_lengths: List[int] = []
+    labels = set()
+
+    for pair in pairs_list:
+        pos_text = getattr(pair.positive_response, "model_response", "")
+        neg_text = getattr(pair.negative_response, "model_response", "")
+        pos_lengths.append(len(pos_text))
+        neg_lengths.append(len(neg_text))
+
+        if pair.label:
+            labels.add(pair.label)
+
+    avg_positive_length = mean(pos_lengths) if pos_lengths else 0.0
+    avg_negative_length = mean(neg_lengths) if neg_lengths else 0.0
+
+    issues: List[DiagnosticsIssue] = []
+
+    if prompt_ratio < config.min_unique_prompt_ratio:
+        issues.append(
+            DiagnosticsIssue(
+                metric="coverage",
+                severity="warning",
+                message="Prompt diversity below configured ratio.",
+                pair_index=None,
+                details={
+                    "ratio": prompt_ratio,
+                    "threshold": config.min_unique_prompt_ratio,
+                    "unique_prompts": len(unique_prompts),
+                    "total_pairs": len(pairs_list),
+                },
+            )
+        )
+
+    if avg_positive_length < config.min_average_length or avg_negative_length < config.min_average_length:
+        issues.append(
+            DiagnosticsIssue(
+                metric="coverage",
+                severity="warning",
+                message="Average response length below minimum threshold.",
+                pair_index=None,
+                details={
+                    "avg_positive_length": avg_positive_length,
+                    "avg_negative_length": avg_negative_length,
+                    "threshold": config.min_average_length,
+                },
+            )
+        )
+
+    summary = {
+        "total_pairs": len(pairs_list),
+        "unique_prompt_ratio": prompt_ratio,
+        "avg_positive_length": avg_positive_length,
+        "avg_negative_length": avg_negative_length,
+        "label_coverage": len(labels),
+    }
+
+    return MetricReport(name="coverage", summary=summary, issues=issues)

wisent/core/contrastive_pairs/diagnostics/divergence.py

@@ -0,0 +1,98 @@
+"""Divergence diagnostics for contrastive pairs."""
+
+from __future__ import annotations
+
+from difflib import SequenceMatcher
+from statistics import mean
+from typing import Iterable, List
+
+from .base import DiagnosticsConfig, DiagnosticsIssue, MetricReport
+
+
+def _normalize_text(text: str) -> str:
+    return " ".join(text.strip().lower().split())
+
+
+def compute_divergence_metrics(pairs: Iterable, config: DiagnosticsConfig) -> MetricReport:
+    """Evaluate textual divergence between positive and negative responses."""
+
+    pairs_list = list(pairs)
+
+    divergences: List[float] = []
+    issues: List[DiagnosticsIssue] = []
+
+    if not pairs_list:
+        return MetricReport(
+            name="divergence",
+            summary={
+                "mean_divergence": 0.0,
+                "min_divergence": 0.0,
+                "max_divergence": 0.0,
+                "low_divergence_fraction": 0.0,
+            },
+            issues=[],
+        )
+
+    for idx, pair in enumerate(pairs_list):
+        positive = getattr(pair.positive_response, "model_response", "")
+        negative = getattr(pair.negative_response, "model_response", "")
+
+        norm_pos = _normalize_text(positive)
+        norm_neg = _normalize_text(negative)
+
+        if not norm_pos or not norm_neg:
+            issues.append(
+                DiagnosticsIssue(
+                    metric="divergence",
+                    severity="critical",
+                    message="Missing positive or negative response text.",
+                    pair_index=idx,
+                    details={"positive": bool(norm_pos), "negative": bool(norm_neg)},
+                )
+            )
+            divergences.append(0.0)
+            continue
+
+        similarity = SequenceMatcher(None, norm_pos, norm_neg).ratio()
+        divergence = 1.0 - similarity
+        divergences.append(divergence)
+
+        if divergence < config.min_divergence:
+            issues.append(
+                DiagnosticsIssue(
+                    metric="divergence",
+                    severity="warning",
+                    message="Positive and negative responses are highly similar.",
+                    pair_index=idx,
+                    details={"divergence": divergence, "similarity": similarity},
+                )
+            )
+
+    low_divergence_fraction = 0.0
+    low_divergence_count = sum(1 for value in divergences if value < config.min_divergence)
+    low_divergence_fraction = low_divergence_count / len(divergences)
+
+    if low_divergence_fraction > config.max_low_divergence_fraction:
+        issues.append(
+            DiagnosticsIssue(
+                metric="divergence",
+                severity="critical",
+                message="Too many pairs fall below divergence threshold.",
+                pair_index=None,
+                details={
+                    "fraction": low_divergence_fraction,
+                    "threshold": config.max_low_divergence_fraction,
+                    "count": low_divergence_count,
+                    "total": len(divergences),
+                },
+            )
+        )
+
+    summary = {
+        "mean_divergence": mean(divergences) if divergences else 0.0,
+        "min_divergence": min(divergences) if divergences else 0.0,
+        "max_divergence": max(divergences) if divergences else 0.0,
+        "low_divergence_fraction": low_divergence_fraction,
+    }
+
+    return MetricReport(name="divergence", summary=summary, issues=issues)

wisent/core/contrastive_pairs/diagnostics/duplicates.py

@@ -0,0 +1,116 @@
+"""Duplicate detection diagnostics for contrastive pairs."""
+
+from __future__ import annotations
+
+from collections import Counter, defaultdict
+from difflib import SequenceMatcher
+from typing import Dict, Iterable, List
+
+from .base import DiagnosticsConfig, DiagnosticsIssue, MetricReport
+
+
+def _norm(text: str) -> str:
+    return " ".join(text.strip().lower().split())
+
+
+def compute_duplicate_metrics(pairs: Iterable, config: DiagnosticsConfig) -> MetricReport:
+    """Detect exact and near duplicates across prompts and responses."""
+
+    pairs_list = list(pairs)
+
+    prompt_counter: Counter[str] = Counter()
+    positive_counter: Counter[str] = Counter()
+    negative_counter: Counter[str] = Counter()
+    indexed_prompts: Dict[str, List[int]] = defaultdict(list)
+
+    for idx, pair in enumerate(pairs_list):
+        prompt = _norm(getattr(pair, "prompt", ""))
+        pos = _norm(getattr(pair.positive_response, "model_response", ""))
+        neg = _norm(getattr(pair.negative_response, "model_response", ""))
+
+        if prompt:
+            prompt_counter[prompt] += 1
+            indexed_prompts[prompt].append(idx)
+        if pos:
+            positive_counter[pos] += 1
+        if neg:
+            negative_counter[neg] += 1
+
+    total_pairs = len(pairs_list)
+    issues: List[DiagnosticsIssue] = []
+
+    if total_pairs == 0:
+        return MetricReport(name="duplicates", summary={"total_pairs": 0}, issues=[])
+
+    def _collect_exact(counter: Counter[str], label: str) -> List[DiagnosticsIssue]:
+        duplicates: List[DiagnosticsIssue] = []
+        for value, count in counter.items():
+            if count > 1:
+                duplicates.append(
+                    DiagnosticsIssue(
+                        metric="duplicates",
+                        severity="warning",
+                        message=f"Exact duplicate detected in {label}.",
+                        pair_index=None,
+                        details={"value": value, "count": count, "field": label},
+                    )
+                )
+        return duplicates
+
+    issues.extend(_collect_exact(prompt_counter, "prompt"))
+    issues.extend(_collect_exact(positive_counter, "positive_response"))
+    issues.extend(_collect_exact(negative_counter, "negative_response"))
+
+    exact_duplicate_fraction = sum(max(0, count - 1) for count in prompt_counter.values()) / total_pairs
+    if exact_duplicate_fraction > config.max_exact_duplicate_fraction:
+        issues.append(
+            DiagnosticsIssue(
+                metric="duplicates",
+                severity="critical",
+                message="Too many exact duplicate prompts detected.",
+                pair_index=None,
+                details={
+                    "fraction": exact_duplicate_fraction,
+                    "threshold": config.max_exact_duplicate_fraction,
+                    "duplicates": [
+                        {"prompt": prompt, "count": count}
+                        for prompt, count in prompt_counter.items()
+                        if count > 1
+                    ],
+                },
+            )
+        )
+
+    near_duplicate_pairs: List[tuple[int, int, float]] = []
+    prompt_items = list(prompt_counter.keys())
+    for i, prompt_a in enumerate(prompt_items):
+        for prompt_b in prompt_items[i + 1 :]:
+            similarity = SequenceMatcher(None, prompt_a, prompt_b).ratio()
+            if similarity >= config.near_duplicate_prompt_threshold:
+                indices_a = indexed_prompts[prompt_a]
+                indices_b = indexed_prompts[prompt_b]
+                near_duplicate_pairs.append((indices_a[0], indices_b[0], similarity))
+                issues.append(
+                    DiagnosticsIssue(
+                        metric="duplicates",
+                        severity="warning",
+                        message="Near-duplicate prompts detected.",
+                        pair_index=None,
+                        details={
+                            "prompt_a": prompt_a,
+                            "prompt_b": prompt_b,
+                            "similarity": similarity,
+                            "a_indices": indices_a,
+                            "b_indices": indices_b,
+                        },
+                    )
+                )
+
+    summary = {
+        "total_pairs": total_pairs,
+        "exact_duplicate_fraction": exact_duplicate_fraction,
+        "unique_prompts": len(prompt_counter),
+        "near_duplicate_count": len(near_duplicate_pairs),
+    }
+
+    return MetricReport(name="duplicates", summary=summary, issues=issues)