brix-protocol 0.1.0__py3-none-any.whl

brix/__init__.py

@@ -0,0 +1,41 @@
+"""BRIX — Runtime Reliability Infrastructure for LLM Pipelines.
+
+BRIX wraps any LLM client and enforces deterministic reliability rules
+defined in a declarative uncertainty.yaml specification, while measuring
+the Balance Index across all interactions.
+"""
+
+from brix.core.exceptions import (
+    BrixError,
+    CircuitBreakerError,
+    ClassifierError,
+    RegistryError,
+    SamplerError,
+    SpecValidationError,
+)
+from brix.core.result import ActionTaken, StructuredResult, UncertaintyType
+from brix.core.router import BrixRouter
+from brix.llm.mock import MockLLMClient
+from brix.llm.protocol import LLMClient
+from brix.spec.loader import load_spec, load_spec_from_dict
+from brix.spec.models import SpecModel
+
+__all__ = [
+    "ActionTaken",
+    "BrixError",
+    "BrixRouter",
+    "CircuitBreakerError",
+    "ClassifierError",
+    "LLMClient",
+    "MockLLMClient",
+    "RegistryError",
+    "SamplerError",
+    "SpecModel",
+    "SpecValidationError",
+    "StructuredResult",
+    "UncertaintyType",
+    "load_spec",
+    "load_spec_from_dict",
+]
+
+__version__ = "0.1.0"

brix/actions/executor.py

@@ -0,0 +1,168 @@
+"""Action executor — dispatches response strategies per uncertainty type.
+
+Each uncertainty type produces a meaningfully different response:
+  EPISTEMIC      → force retrieval augmentation
+  CONTRADICTORY  → explicit conflict resolution
+  OPEN_ENDED     → distribution of outcomes
+  CERTAIN        → passthrough (no intervention)
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from brix.core.result import ActionTaken, UncertaintyType
+from brix.llm.protocol import LLMClient
+from brix.spec.models import SpecModel, UncertaintyTypeDef
+
+
+@dataclass(frozen=True, slots=True)
+class ActionResult:
+    """Result of action execution."""
+
+    action_taken: ActionTaken
+    response: str
+    intervention_necessary: bool
+    cost_tokens_extra: int
+
+
+class ActionExecutor:
+    """Executes the appropriate response strategy for each uncertainty type.
+
+    Each type produces a meaningfully different response, not just a
+    different label on the same output.
+    """
+
+    def __init__(self, spec: SpecModel, llm_client: LLMClient) -> None:
+        self._llm = llm_client
+        self._type_configs: dict[str, UncertaintyTypeDef] = {
+            t.name: t for t in spec.uncertainty_types
+        }
+
+    async def execute(
+        self,
+        uncertainty_type: UncertaintyType,
+        samples: list[str],
+        query: str,
+        force_retrieval: bool = False,
+    ) -> ActionResult:
+        """Execute the action for the classified uncertainty type.
+
+        Args:
+            uncertainty_type: The classified uncertainty type.
+            samples: Collected response samples.
+            query: Original user query.
+            force_retrieval: Whether to force retrieval (from CB hit).
+
+        Returns:
+            ActionResult with the final response and action metadata.
+        """
+        if uncertainty_type == UncertaintyType.CERTAIN and not force_retrieval:
+            return ActionResult(
+                action_taken=ActionTaken.NONE,
+                response=samples[0] if samples else "",
+                intervention_necessary=False,
+                cost_tokens_extra=0,
+            )
+
+        if uncertainty_type == UncertaintyType.EPISTEMIC or force_retrieval:
+            return await self._handle_epistemic(samples, query)
+
+        if uncertainty_type == UncertaintyType.CONTRADICTORY:
+            return await self._handle_contradictory(samples, query)
+
+        if uncertainty_type == UncertaintyType.OPEN_ENDED:
+            return await self._handle_open_ended(samples, query)
+
+        # Fallback: treat as epistemic
+        return await self._handle_epistemic(samples, query)
+
+    async def _handle_epistemic(
+        self, samples: list[str], query: str
+    ) -> ActionResult:
+        """Handle epistemic uncertainty — signal retrieval augmentation needed."""
+        config = self._type_configs.get("epistemic")
+        template = config.action_config.message_template if config else ""
+
+        # Build a retrieval-signaling response from the samples
+        sample_summary = samples[0] if samples else "No response available."
+        response = (
+            f"{template.strip()}\n\n"
+            f"Based on initial analysis: {sample_summary}\n\n"
+            f"[RETRIEVAL_NEEDED] This response requires verification through "
+            f"retrieval augmentation. The query '{query}' touches on knowledge "
+            f"that may not be reliably represented in the model's training data."
+        )
+
+        return ActionResult(
+            action_taken=ActionTaken.FORCE_RETRIEVAL,
+            response=response,
+            intervention_necessary=True,
+            cost_tokens_extra=self._estimate_extra_tokens(samples),
+        )
+
+    async def _handle_contradictory(
+        self, samples: list[str], query: str
+    ) -> ActionResult:
+        """Handle contradictory uncertainty — explicit conflict resolution."""
+        config = self._type_configs.get("contradictory")
+        template = config.action_config.message_template if config else ""
+
+        # Build conflict resolution from divergent samples
+        conflict_parts: list[str] = []
+        for i, sample in enumerate(samples, 1):
+            conflict_parts.append(f"Position {i}: {sample.strip()}")
+
+        conflicts = "\n\n".join(conflict_parts)
+        response = (
+            f"{template.strip()}\n\n"
+            f"Multiple responses to '{query}' produced conflicting information:\n\n"
+            f"{conflicts}\n\n"
+            f"[CONFLICT_DETECTED] These positions contain material differences "
+            f"that require resolution. The correct answer may depend on specific "
+            f"context, jurisdiction, or conditions not specified in the query."
+        )
+
+        return ActionResult(
+            action_taken=ActionTaken.CONFLICT_RESOLUTION,
+            response=response,
+            intervention_necessary=True,
+            cost_tokens_extra=self._estimate_extra_tokens(samples),
+        )
+
+    async def _handle_open_ended(
+        self, samples: list[str], query: str
+    ) -> ActionResult:
+        """Handle open-ended uncertainty — distribution of outcomes."""
+        config = self._type_configs.get("open_ended")
+        template = config.action_config.message_template if config else ""
+
+        # Build distribution response from varied samples
+        perspective_parts: list[str] = []
+        for i, sample in enumerate(samples, 1):
+            perspective_parts.append(f"Perspective {i}: {sample.strip()}")
+
+        perspectives = "\n\n".join(perspective_parts)
+        response = (
+            f"{template.strip()}\n\n"
+            f"The query '{query}' has multiple valid answers:\n\n"
+            f"{perspectives}\n\n"
+            f"[MULTIPLE_PERSPECTIVES] This question does not have a single "
+            f"deterministically correct answer. The above perspectives represent "
+            f"the distribution of plausible responses."
+        )
+
+        return ActionResult(
+            action_taken=ActionTaken.DISTRIBUTION_RESPONSE,
+            response=response,
+            intervention_necessary=True,
+            cost_tokens_extra=self._estimate_extra_tokens(samples),
+        )
+
+    @staticmethod
+    def _estimate_extra_tokens(samples: list[str]) -> int:
+        """Estimate extra token cost from additional samples."""
+        if len(samples) <= 1:
+            return 0
+        extra_chars = sum(len(s) for s in samples[1:])
+        return extra_chars // 4  # rough token estimate

brix/analysis/classifier.py

@@ -0,0 +1,106 @@
+"""Uncertainty type classifier using semantic consistency and refusal signals.
+
+Classification thresholds:
+  - consistency > 0.90, no refusal signals     → CERTAIN
+  - consistency > 0.90, refusal in ≥2 samples  → EPISTEMIC
+  - consistency < 0.45                         → CONTRADICTORY
+  - 0.45 ≤ consistency < 0.70, variance > 0.15 → OPEN_ENDED
+  - all other cases                            → EPISTEMIC (safe fallback)
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from brix.analysis.consistency import ConsistencyResult, SemanticConsistencyAnalyzer
+from brix.analysis.refusal import count_refusals
+from brix.core.result import UncertaintyType
+
+
+@dataclass(frozen=True, slots=True)
+class ClassificationResult:
+    """Result of uncertainty type classification."""
+
+    uncertainty_type: UncertaintyType
+    subtype: str
+    mean_consistency: float
+    variance: float
+    refusal_count: int
+
+
+class UncertaintyClassifier:
+    """Classifies uncertainty type from response samples.
+
+    Uses the SemanticConsistencyAnalyzer for embedding-based similarity
+    and refusal detection for behavioral signals. Applies fixed thresholds
+    to produce a deterministic classification given the same inputs.
+    """
+
+    def __init__(self, analyzer: SemanticConsistencyAnalyzer) -> None:
+        self._analyzer = analyzer
+
+    def classify(self, samples: list[str]) -> ClassificationResult:
+        """Classify the uncertainty type from collected samples.
+
+        Args:
+            samples: List of LLM response texts.
+
+        Returns:
+            ClassificationResult with the determined uncertainty type.
+        """
+        # Single sample → CERTAIN (low-risk passthrough)
+        if len(samples) <= 1:
+            return ClassificationResult(
+                uncertainty_type=UncertaintyType.CERTAIN,
+                subtype="single_sample",
+                mean_consistency=1.0,
+                variance=0.0,
+                refusal_count=0,
+            )
+
+        consistency = self._analyzer.analyze(samples)
+        refusal_count = count_refusals(samples)
+
+        uncertainty_type, subtype = self._apply_thresholds(
+            consistency, refusal_count
+        )
+
+        return ClassificationResult(
+            uncertainty_type=uncertainty_type,
+            subtype=subtype,
+            mean_consistency=consistency.mean_similarity,
+            variance=consistency.variance,
+            refusal_count=refusal_count,
+        )
+
+    def _apply_thresholds(
+        self,
+        consistency: ConsistencyResult,
+        refusal_count: int,
+    ) -> tuple[UncertaintyType, str]:
+        """Apply classification thresholds to consistency and refusal data.
+
+        Returns:
+            Tuple of (UncertaintyType, subtype_string).
+        """
+        sim = consistency.mean_similarity
+        var = consistency.variance
+
+        # High consistency, no refusals → CERTAIN
+        if sim > 0.90 and refusal_count == 0:
+            return UncertaintyType.CERTAIN, "high_consistency_no_refusal"
+
+        # High consistency, but refusals present → EPISTEMIC
+        if sim > 0.90 and refusal_count >= 2:
+            return UncertaintyType.EPISTEMIC, "high_consistency_with_refusals"
+
+        # Very low consistency → CONTRADICTORY
+        if sim < 0.45:
+            return UncertaintyType.CONTRADICTORY, "low_consistency"
+
+        # Moderate consistency with high variance → OPEN_ENDED
+        if 0.45 <= sim < 0.70 and var > 0.15:
+            return UncertaintyType.OPEN_ENDED, "moderate_consistency_high_variance"
+
+        # Safe fallback → EPISTEMIC
+        return UncertaintyType.EPISTEMIC, "fallback"

brix/analysis/consistency.py

@@ -0,0 +1,87 @@
+"""Semantic consistency analyzer using sentence-transformers.
+
+Uses the all-MiniLM-L6-v2 model to compute pairwise cosine similarity
+between collected response samples. The model is loaded ONCE at
+initialization and never reloaded per request.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+import numpy as np
+
+
+@dataclass(frozen=True, slots=True)
+class ConsistencyResult:
+    """Result of semantic consistency analysis."""
+
+    mean_similarity: float
+    variance: float
+    pairwise_similarities: list[float]
+
+
+class SemanticConsistencyAnalyzer:
+    """Computes pairwise semantic similarity between response samples.
+
+    The sentence-transformers model is loaded exactly once during __init__
+    and reused for all subsequent calls. No per-request model loading.
+    """
+
+    def __init__(self, model_name: str = "all-MiniLM-L6-v2") -> None:
+        from sentence_transformers import SentenceTransformer
+
+        self._model: Any = SentenceTransformer(model_name)
+
+    def analyze(self, samples: list[str]) -> ConsistencyResult:
+        """Compute pairwise cosine similarity between all samples.
+
+        Args:
+            samples: List of response texts (must have at least 2).
+
+        Returns:
+            ConsistencyResult with mean similarity, variance, and all
+            pairwise similarity values.
+        """
+        if len(samples) < 2:
+            return ConsistencyResult(
+                mean_similarity=1.0,
+                variance=0.0,
+                pairwise_similarities=[1.0],
+            )
+
+        # Encode all samples at once (batch operation)
+        embeddings = self._model.encode(samples, convert_to_numpy=True)
+        embeddings = np.array(embeddings)
+
+        # Normalize for cosine similarity
+        norms = np.linalg.norm(embeddings, axis=1, keepdims=True)
+        norms = np.where(norms == 0, 1.0, norms)
+        normalized = embeddings / norms
+
+        # Compute pairwise cosine similarity matrix
+        sim_matrix = np.dot(normalized, normalized.T)
+
+        # Extract upper triangle (excluding diagonal)
+        n = len(samples)
+        pairwise: list[float] = []
+        for i in range(n):
+            for j in range(i + 1, n):
+                pairwise.append(float(sim_matrix[i, j]))
+
+        if not pairwise:
+            return ConsistencyResult(
+                mean_similarity=1.0,
+                variance=0.0,
+                pairwise_similarities=[1.0],
+            )
+
+        mean_sim = float(np.mean(pairwise))
+        var_sim = float(np.var(pairwise))
+
+        return ConsistencyResult(
+            mean_similarity=mean_sim,
+            variance=var_sim,
+            pairwise_similarities=pairwise,
+        )

brix/analysis/refusal.py

@@ -0,0 +1,70 @@
+"""Refusal detection heuristics for LLM response samples.
+
+Detects common patterns indicating the model is refusing to answer,
+acknowledging limitations, or deferring to human expertise.
+"""
+
+from __future__ import annotations
+
+# Common refusal/deferral phrases found in LLM responses
+_REFUSAL_PATTERNS: list[str] = [
+    "i cannot",
+    "i can't",
+    "i'm not able to",
+    "i am not able to",
+    "i'm unable to",
+    "i am unable to",
+    "i don't have",
+    "i do not have",
+    "i shouldn't",
+    "i should not",
+    "consult a professional",
+    "consult a doctor",
+    "consult a lawyer",
+    "consult an attorney",
+    "consult a financial advisor",
+    "seek professional",
+    "seek medical",
+    "seek legal",
+    "not qualified to",
+    "beyond my capabilities",
+    "outside my expertise",
+    "i'm not a doctor",
+    "i'm not a lawyer",
+    "i'm not a financial",
+    "not medical advice",
+    "not legal advice",
+    "not financial advice",
+    "please consult",
+    "please seek",
+    "i must decline",
+    "i have to decline",
+    "i apologize, but i cannot",
+    "as an ai",
+    "as a language model",
+]
+
+
+def detect_refusal(text: str) -> bool:
+    """Check if a single response contains refusal language.
+
+    Args:
+        text: A single LLM response text.
+
+    Returns:
+        True if refusal language is detected.
+    """
+    lowered = text.lower()
+    return any(pattern in lowered for pattern in _REFUSAL_PATTERNS)
+
+
+def count_refusals(samples: list[str]) -> int:
+    """Count how many samples in a list contain refusal language.
+
+    Args:
+        samples: List of LLM response texts.
+
+    Returns:
+        Number of samples containing refusal patterns.
+    """
+    return sum(1 for s in samples if detect_refusal(s))

brix/balance/tracker.py

@@ -0,0 +1,165 @@
+"""Balance Index tracker — TP/FN/TN/FP counters and harmonic mean computation.
+
+The Balance Index is defined as:
+  Balance Index = 2 * (R * U) / (R + U)
+  where R = TP / (TP + FN), U = TN / (TN + FP)
+
+Supports both heuristic defaults and explicit feedback from the caller.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from uuid import UUID
+
+
+@dataclass
+class BalanceState:
+    """Current state of the balance tracker counters."""
+
+    tp: int = 0  # Risky query correctly intercepted
+    fn: int = 0  # Risky query passed without intervention
+    tn: int = 0  # Safe query correctly passed
+    fp: int = 0  # Safe query incorrectly intercepted
+
+
+class BalanceTracker:
+    """Tracks reliability and utility metrics across a session.
+
+    Computes the running Balance Index after every request. Supports
+    both heuristic auto-classification and explicit feedback via
+    the feedback() method.
+    """
+
+    def __init__(self) -> None:
+        self._state = BalanceState()
+        self._pending: dict[UUID, _PendingDecision] = {}
+
+    @property
+    def state(self) -> BalanceState:
+        """Current counter state."""
+        return self._state
+
+    def record_decision(
+        self,
+        decision_id: UUID,
+        intervention_necessary: bool,
+        circuit_breaker_hit: bool,
+        risk_score: float,
+    ) -> tuple[bool, bool, float]:
+        """Record a decision using heuristic classification.
+
+        Heuristic rules:
+        - CB hit + intervention → TP (assumed risky, correctly caught)
+        - High risk (>0.70) + intervention → TP
+        - Medium risk + intervention → TP (conservative)
+        - Low risk + no intervention → TN (safe, correctly passed)
+        - Low risk + intervention → FP (safe, incorrectly caught)
+        - High risk + no intervention → FN (risky, missed)
+
+        Also stores the decision for possible later feedback override.
+
+        Args:
+            decision_id: Unique ID for this decision.
+            intervention_necessary: Whether intervention was applied.
+            circuit_breaker_hit: Whether a CB fired.
+            risk_score: Computed risk score.
+
+        Returns:
+            Tuple of (reliability_signal, utility_signal, balance_index).
+        """
+        is_risky = circuit_breaker_hit or risk_score > 0.40
+
+        if is_risky and intervention_necessary:
+            self._state.tp += 1
+            reliability_signal = True
+            utility_signal = True
+        elif is_risky and not intervention_necessary:
+            self._state.fn += 1
+            reliability_signal = False
+            utility_signal = True
+        elif not is_risky and not intervention_necessary:
+            self._state.tn += 1
+            reliability_signal = True
+            utility_signal = True
+        else:  # not risky, but intervention happened
+            self._state.fp += 1
+            reliability_signal = True
+            utility_signal = False
+
+        # Store for potential feedback override
+        self._pending[decision_id] = _PendingDecision(
+            is_risky=is_risky,
+            intervention_applied=intervention_necessary,
+        )
+
+        balance = self.compute_balance_index()
+        return reliability_signal, utility_signal, balance
+
+    def feedback(self, decision_id: UUID, was_intervention_necessary: bool) -> None:
+        """Provide ground-truth feedback to correct heuristic classification.
+
+        Args:
+            decision_id: The decision to correct.
+            was_intervention_necessary: True if intervention was actually needed.
+        """
+        pending = self._pending.pop(decision_id, None)
+        if pending is None:
+            return
+
+        # Reverse the heuristic classification
+        if pending.is_risky and pending.intervention_applied:
+            self._state.tp -= 1
+        elif pending.is_risky and not pending.intervention_applied:
+            self._state.fn -= 1
+        elif not pending.is_risky and not pending.intervention_applied:
+            self._state.tn -= 1
+        else:
+            self._state.fp -= 1
+
+        # Apply ground-truth classification
+        actually_risky = was_intervention_necessary
+        if actually_risky and pending.intervention_applied:
+            self._state.tp += 1
+        elif actually_risky and not pending.intervention_applied:
+            self._state.fn += 1
+        elif not actually_risky and not pending.intervention_applied:
+            self._state.tn += 1
+        else:
+            self._state.fp += 1
+
+    def compute_balance_index(self) -> float:
+        """Compute the current Balance Index.
+
+        Balance Index = 2 * R * U / (R + U)
+        where R = TP / (TP + FN), U = TN / (TN + FP)
+
+        Returns 0.0 if insufficient data to compute.
+        """
+        r = self._reliability_score()
+        u = self._utility_score()
+        if r + u == 0:
+            return 0.0
+        return 2.0 * r * u / (r + u)
+
+    def _reliability_score(self) -> float:
+        """R = TP / (TP + FN). Returns 0.0 if no relevant data."""
+        total = self._state.tp + self._state.fn
+        if total == 0:
+            return 0.0
+        return self._state.tp / total
+
+    def _utility_score(self) -> float:
+        """U = TN / (TN + FP). Returns 0.0 if no relevant data."""
+        total = self._state.tn + self._state.fp
+        if total == 0:
+            return 0.0
+        return self._state.tn / total
+
+
+@dataclass(frozen=True, slots=True)
+class _PendingDecision:
+    """Internal record of a decision awaiting potential feedback."""
+
+    is_risky: bool
+    intervention_applied: bool