logitgate 0.2.0__py3-none-any.whl

logitgate/integrations/__init__.py

@@ -0,0 +1,10 @@
+from logitgate.integrations.deepeval import LogitGateMetric, LogitGateSystemOneModel
+from logitgate.integrations.langgraph import as_conditional_edge
+from logitgate.integrations.pytest_plugin import check
+
+__all__ = [
+    "LogitGateMetric",
+    "LogitGateSystemOneModel",
+    "as_conditional_edge",
+    "check",
+]

logitgate/integrations/deepeval.py

@@ -0,0 +1,212 @@
+from __future__ import annotations
+
+from typing import Any
+
+from logitgate.backends.base import LogitBackend
+from logitgate.gates.logit_gate import LogitGate
+
+# Graceful optional imports for DeepEval
+try:
+    from deepeval.metrics import BaseMetric
+    from deepeval.models import DeepEvalBaseSystemOneModel
+    from deepeval.models.system_one.schema import (
+        NoulAnswer,
+        NoulQuestion,
+        SystemOneAnswers,
+        SystemOneQuestion,
+    )
+    from deepeval.test_case import LLMTestCase
+    _DEEPEVAL_AVAILABLE = True
+except ImportError:
+    _DEEPEVAL_AVAILABLE = False
+
+    class BaseMetric:  # type: ignore[no-redef]
+        def __init__(self) -> None:
+            pass
+
+    class DeepEvalBaseSystemOneModel:  # type: ignore[no-redef]
+        def __init__(self, model: str = "default") -> None:
+            self.model = model
+
+    class NoulAnswer:  # type: ignore[no-redef]
+        def __init__(self, probability: float = 0.5) -> None:
+            self.probability = probability
+
+    class NoulQuestion:  # type: ignore[no-redef]
+        def __init__(self, instructions: str = "", statement: str = "") -> None:
+            self.instructions = instructions or statement
+            self.statement = statement or instructions
+
+    class SystemOneAnswers:  # type: ignore[no-redef]
+        def __init__(self, nouls: dict[str, Any]) -> None:
+            self.nouls = nouls
+
+    class SystemOneQuestion(NoulQuestion):  # type: ignore[no-redef]
+        pass
+
+    class LLMTestCase:  # type: ignore[no-redef]
+        def __init__(self, input: str = "", actual_output: str = "") -> None:
+            self.input = input
+            self.actual_output = actual_output
+
+
+class LogitGateSystemOneModel(DeepEvalBaseSystemOneModel):
+    """
+    DeepEval System-1 model powered by LogitGate debiased first-token logits.
+    Drop-in System-1 judge model for DeepEval JevEval metrics.
+    """
+
+    def __init__(
+        self,
+        model_name: str = "qwen2.5-coder:14b",
+        api_base: str = "http://localhost:11434",
+        backend: LogitBackend | None = None,
+        symmetric: bool = True,
+        logit_temperature: float = 1.0,
+    ) -> None:
+        self.model_name = model_name
+        self.api_base = api_base
+        self.symmetric = symmetric
+        self.logit_temperature = logit_temperature
+
+        if backend is not None:
+            self.backend = backend
+        else:
+            from logitgate.backends.ollama import OllamaBackend
+            self.backend = OllamaBackend(base_url=api_base, model=model_name)
+        super().__init__(model=model_name)
+
+    def load_model(self) -> None:
+        return None
+
+    def get_model_name(self) -> str:
+        return f"logitgate:{self.model_name}"
+
+    def decide(
+        self,
+        state: Any,
+        questions: dict[str, Any],
+    ) -> tuple[Any, float | None]:
+        nouls: dict[str, Any] = {}
+        if len(questions) > 1:
+            q_ids = list(questions.keys())
+            statements = [
+                getattr(questions[qid], "statement", None)
+                or getattr(questions[qid], "instructions", None)
+                or str(questions[qid])
+                for qid in q_ids
+            ]
+            gate = LogitGate(
+                rubric=statements[0] if (statements and statements[0].strip()) else "Evaluate candidate against rubric.",
+                backend=self.backend,
+                symmetric_debiasing=self.symmetric,
+                logit_temperature=self.logit_temperature,
+            )
+            verdicts = gate.evaluate_batch(state, statements)
+            for q_id, verdict in zip(q_ids, verdicts):
+                p = verdict.probability if verdict.probability is not None else (1.0 if verdict.passed else 0.0)
+                nouls[q_id] = NoulAnswer(probability=round(p, 4))
+        else:
+            for q_id, q in questions.items():
+                statement = getattr(q, "statement", None) or getattr(q, "instructions", None) or str(q)
+                gate = LogitGate(
+                    rubric=statement,
+                    backend=self.backend,
+                    symmetric_debiasing=self.symmetric,
+                    logit_temperature=self.logit_temperature,
+                )
+                verdict = gate.evaluate(state)
+                p = verdict.probability if verdict.probability is not None else (1.0 if verdict.passed else 0.0)
+                nouls[q_id] = NoulAnswer(probability=round(p, 4))
+        return SystemOneAnswers(nouls=nouls), 0.0001
+
+    async def a_decide(
+        self,
+        state: Any,
+        questions: dict[str, Any],
+    ) -> tuple[Any, float | None]:
+        nouls: dict[str, Any] = {}
+        if len(questions) > 1:
+            q_ids = list(questions.keys())
+            statements = [
+                getattr(questions[qid], "statement", None)
+                or getattr(questions[qid], "instructions", None)
+                or str(questions[qid])
+                for qid in q_ids
+            ]
+            gate = LogitGate(
+                rubric=statements[0] if (statements and statements[0].strip()) else "Evaluate candidate against rubric.",
+                backend=self.backend,
+                symmetric_debiasing=self.symmetric,
+                logit_temperature=self.logit_temperature,
+            )
+            verdicts = await gate.aevaluate_batch(state, statements)
+            for q_id, verdict in zip(q_ids, verdicts):
+                p = verdict.probability if verdict.probability is not None else (1.0 if verdict.passed else 0.0)
+                nouls[q_id] = NoulAnswer(probability=round(p, 4))
+        else:
+            for q_id, q in questions.items():
+                statement = getattr(q, "statement", None) or getattr(q, "instructions", None) or str(q)
+                gate = LogitGate(
+                    rubric=statement,
+                    backend=self.backend,
+                    symmetric_debiasing=self.symmetric,
+                    logit_temperature=self.logit_temperature,
+                )
+                verdict = await gate.aevaluate(state)
+                p = verdict.probability if verdict.probability is not None else (1.0 if verdict.passed else 0.0)
+                nouls[q_id] = NoulAnswer(probability=round(p, 4))
+        return SystemOneAnswers(nouls=nouls), 0.0001
+
+
+class LogitGateMetric(BaseMetric):
+    """
+    DeepEval metric evaluating LLMTestCase using a 50ms LogitGate.
+    """
+
+    def __init__(
+        self,
+        rubric: str,
+        threshold: float = 0.85,
+        backend: str | LogitBackend = "stub://default",
+        name: str = "LogitGate Metric",
+        async_mode: bool = False,
+    ) -> None:
+        super().__init__()
+        self.name = name
+        self.rubric = rubric
+        self.threshold = threshold
+        self.gate = LogitGate(
+            rubric=rubric,
+            pass_threshold=threshold,
+            backend=backend,
+        )
+        self.async_mode = async_mode
+        self.score: float | None = None
+        self.reason: str | None = None
+        self.success: bool | None = None
+
+    @property
+    def __name__(self) -> str:
+        return self.name
+
+    def measure(self, test_case: Any, *args: Any, **kwargs: Any) -> float:
+        candidate = getattr(test_case, "actual_output", None) or getattr(test_case, "input", str(test_case))
+        verdict = self.gate.evaluate(candidate)
+        self.score = verdict.probability if verdict.probability is not None else (1.0 if verdict.passed else 0.0)
+        self.success = verdict.passed
+        self.reason = verdict.reason
+        return self.score
+
+    async def a_measure(self, test_case: Any, *args: Any, **kwargs: Any) -> float:
+        candidate = getattr(test_case, "actual_output", None) or getattr(test_case, "input", str(test_case))
+        verdict = await self.gate.aevaluate(candidate)
+        self.score = verdict.probability if verdict.probability is not None else (1.0 if verdict.passed else 0.0)
+        self.success = verdict.passed
+        self.reason = verdict.reason
+        return self.score
+
+    def is_successful(self) -> bool:
+        if self.threshold is None or self.score is None:
+            return False
+        return self.score >= self.threshold

logitgate/integrations/langgraph.py

@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+from collections.abc import Callable, Mapping
+from typing import Any
+
+from logitgate.gates.base import BaseGate
+
+
+def as_conditional_edge(
+    gate: BaseGate,
+    pass_node: str,
+    fail_node: str,
+    state_key: str = "content",
+) -> Callable[[Mapping[str, Any] | Any], str]:
+    """
+    Returns a conditional edge function for LangGraph workflows.
+    Extracts candidate from state using state_key and routes to
+    pass_node if the gate passes, else fail_node.
+    """
+    def route(state: Mapping[str, Any] | Any) -> str:
+        candidate: Any
+        if isinstance(state, Mapping):
+            candidate = state.get(state_key, state)
+        else:
+            candidate = getattr(state, state_key, state)
+
+        verdict = gate.evaluate(candidate)
+        return pass_node if verdict.passed else fail_node
+
+    return route

logitgate/integrations/pytest_plugin.py

@@ -0,0 +1,68 @@
+from __future__ import annotations
+
+import functools
+import inspect
+from collections.abc import Callable
+from typing import Any
+
+from logitgate.backends.base import LogitBackend
+from logitgate.config import LogitGateDefaults, get_default_backend
+from logitgate.gates.ast_gate import ASTGate
+from logitgate.gates.logit_gate import LogitGate
+
+
+def check(
+    rubric: str,
+    threshold: float | None = None,
+    backend: str | LogitBackend | None = None,
+    ast_check: bool = False,
+    save_snapshot: bool | None = None,
+) -> Callable[[Callable[..., Any]], Callable[..., Any]]:
+    """
+    Pytest decorator to evaluate function output against an LLM rubric
+    using LogitGate.
+    """
+    from logitgate.config import get_default_save_failures
+
+    effective_backend = (
+        get_default_backend() if (backend is None or backend == "default") else backend
+    )
+    effective_thresh = (
+        LogitGateDefaults.pass_threshold if threshold is None else threshold
+    )
+    effective_save = save_snapshot if save_snapshot is not None else get_default_save_failures()
+
+    def _format_error_msg(verdict: Any) -> str:
+        snapshot_path = verdict.traces[-1].metadata.get("snapshot_path") if verdict.traces else None
+        if snapshot_path:
+            return (
+                f"LogitGate evaluation failed (P={verdict.probability}): {verdict.reason}\n"
+                f"Snapshot saved to: {snapshot_path}\n"
+                f"Reproduce & generate full Layer-2 explanation locally without re-running your app:\n"
+                f"  $ logitgate explain {snapshot_path}"
+            )
+        return f"LogitGate evaluation failed (P={verdict.probability}): {verdict.reason}"
+
+    def decorator(fn: Callable[..., Any]) -> Callable[..., Any]:
+        gate: Any = LogitGate(rubric=rubric, pass_threshold=effective_thresh, backend=effective_backend)
+        if ast_check:
+            gate = ASTGate(syntax=True) >> gate
+
+        if inspect.iscoroutinefunction(fn):
+            @functools.wraps(fn)
+            async def async_wrapper(*args: Any, **kwargs: Any) -> Any:
+                res = await fn(*args, **kwargs)
+                verdict = await gate.aevaluate(res, save_on_fail=effective_save, test_name=fn.__name__)
+                assert verdict.passed, _format_error_msg(verdict)
+                return res
+            return async_wrapper
+        else:
+            @functools.wraps(fn)
+            def sync_wrapper(*args: Any, **kwargs: Any) -> Any:
+                res = fn(*args, **kwargs)
+                verdict = gate.evaluate(res, save_on_fail=effective_save, test_name=fn.__name__)
+                assert verdict.passed, _format_error_msg(verdict)
+                return res
+            return sync_wrapper
+
+    return decorator

logitgate/models.py

@@ -0,0 +1,109 @@
+from __future__ import annotations
+
+from enum import Enum
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+
+class DecisionStatus(str, Enum):
+    """Enumeration of possible gating decision outcomes.
+
+    Attributes:
+        PASSED: Artifact satisfied all requirements and thresholds.
+        FAILED: Artifact decisively failed verification.
+        AMBIGUOUS: Confidence fell within the uncertainty band between fail and pass thresholds.
+    """
+
+    PASSED = "passed"
+    FAILED = "failed"
+    AMBIGUOUS = "ambiguous"
+
+
+class LayerTrace(BaseModel):
+    """Detailed execution telemetry and diagnostic trace for a single gate layer.
+
+    Attributes:
+        layer_name: Identifier of the executing gate layer (e.g., 'ASTGate', 'LogitGate').
+        status: Decision status reached by this layer.
+        latency_ms: Execution duration for this layer in milliseconds.
+        calibrated_prob: Post-debiasing / calibrated probability (if applicable).
+        forward_prob: Pass probability from forward permutation pass (Pass 1).
+        reverse_prob: Pass probability from reverse permutation pass (Pass 2).
+        forward_log_odds: Raw logit difference from forward pass (Pass 1).
+        reverse_log_odds: Raw logit difference from reverse pass (Pass 2).
+        tokens_generated: Count of autoregressive tokens generated by this layer.
+        diagnostic_report: Text critique or diagnostic error message (if any).
+        metadata: Arbitrary supplemental telemetry and token distributions.
+    """
+
+    layer_name: str
+    status: DecisionStatus
+    latency_ms: float
+    calibrated_prob: float | None = None
+    forward_prob: float | None = None
+    reverse_prob: float | None = None
+    forward_log_odds: float | None = None
+    reverse_log_odds: float | None = None
+    tokens_generated: int = 0
+    diagnostic_report: str | None = None
+    metadata: dict[str, Any] = Field(default_factory=dict)
+
+    def __repr__(self) -> str:
+        parts = [
+            f"layer='{self.layer_name}'",
+            f"status={self.status.value}",
+            f"latency={self.latency_ms:.2f}ms",
+        ]
+        if self.calibrated_prob is not None:
+            parts.append(f"prob={self.calibrated_prob:.4f}")
+        if self.tokens_generated > 0:
+            parts.append(f"tokens={self.tokens_generated}")
+        return f"LayerTrace({', '.join(parts)})"
+
+
+class Verdict(BaseModel):
+    """Final aggregated evaluation outcome across single gates or cascaded pipelines.
+
+    Attributes:
+        passed: Boolean flag indicating if the candidate passed evaluation.
+        status: Discrete decision status (PASSED, FAILED, AMBIGUOUS).
+        probability: Debiased confidence score in [0.0, 1.0] (for binary/logit gates).
+        score: Continuous expected score (for ScoreGate or scored judges).
+        choice: Selected discrete option identifier (for ChoiceGate).
+        choice_probabilities: Softmax probability distribution over discrete options.
+        triggered_layer: Name of the gate layer that produced the terminating decision.
+        total_latency_ms: Total cumulative evaluation latency in milliseconds.
+        total_tokens_generated: Total cumulative autoregressive tokens generated.
+        reason: Diagnostic summary, critique, or failure justification.
+        traces: Ordered sequence of telemetry traces from each executed layer.
+    """
+
+    passed: bool
+    status: DecisionStatus
+    probability: float | None = None
+    score: float | None = None
+    choice: str | None = None
+    choice_probabilities: dict[str, float] | None = None
+    triggered_layer: str
+    total_latency_ms: float
+    total_tokens_generated: int = 0
+    reason: str | None = None
+    traces: list[LayerTrace] = Field(default_factory=list)
+
+    def __bool__(self) -> bool:
+        """Allows direct truthiness checks: `if verdict: ...`."""
+        return self.passed
+
+    def __repr__(self) -> str:
+        parts = [f"status='{self.status.value}'"]
+        if self.probability is not None:
+            parts.append(f"prob={self.probability:.4f}")
+        if self.score is not None:
+            parts.append(f"score={self.score:.2f}")
+        if self.choice is not None:
+            parts.append(f"choice='{self.choice}'")
+        parts.append(f"layer='{self.triggered_layer}'")
+        parts.append(f"latency={self.total_latency_ms:.2f}ms")
+        parts.append(f"tokens={self.total_tokens_generated}")
+        return f"Verdict({', '.join(parts)})"

logitgate/snapshot.py

@@ -0,0 +1,224 @@
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import re
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+from pydantic import BaseModel, Field, model_validator
+
+from logitgate.backends.base import GenerativeBackend, LogitBackend, StubBackend
+from logitgate.config import get_default_snapshot_dir
+from logitgate.models import DecisionStatus, Verdict
+
+
+def _compute_snapshot_id(timestamp: str, rubric: str, candidate: Any) -> str:
+    raw = f"{timestamp}:{rubric}:{candidate!s}".encode()
+    return hashlib.sha256(raw).hexdigest()[:8]
+
+
+class VerdictSnapshot(BaseModel):
+    """
+    Deterministic failure snapshot capturing candidate artifact, rubric, verdict,
+    and gate telemetry for offline replay and Layer-2 explanation without re-running the system under test.
+    """
+    model_config = {"arbitrary_types_allowed": True}
+
+    snapshot_id: str = ""
+    timestamp: str = ""
+    test_name: str | None = None
+    rubric: str
+    candidate: Any
+    verdict: Verdict
+    gate_config: dict[str, Any] = Field(default_factory=dict)
+
+    @model_validator(mode="before")
+    @classmethod
+    def _populate_defaults(cls, data: Any) -> Any:
+        if isinstance(data, dict):
+            ts = data.get("timestamp")
+            if not ts:
+                ts = datetime.now(timezone.utc).isoformat()
+                data["timestamp"] = ts
+            sid = data.get("snapshot_id")
+            if not sid:
+                rub = str(data.get("rubric", ""))
+                cand = data.get("candidate", "")
+                data["snapshot_id"] = _compute_snapshot_id(ts, rub, cand)
+        return data
+
+    @classmethod
+    def from_verdict(
+        cls,
+        verdict: Verdict,
+        rubric: str,
+        candidate: Any,
+        test_name: str | None = None,
+        gate_config: dict[str, Any] | None = None,
+    ) -> VerdictSnapshot:
+        """Constructs a VerdictSnapshot directly from a Verdict and evaluation context."""
+        return cls(
+            test_name=test_name,
+            rubric=rubric,
+            candidate=candidate,
+            verdict=verdict,
+            gate_config=gate_config or {},
+        )
+
+    def save(self, directory: str | Path | None = None) -> Path:
+        """
+        Saves this snapshot as JSON to directory (defaulting to LOGITGATE_SNAPSHOT_DIR or .logitgate/failures),
+        named fail_{timestamp_compact}_{test_slug}_{snapshot_id}.json, and updates latest.json in that directory.
+        """
+        target_dir = Path(directory) if directory is not None else Path(
+            os.environ.get("LOGITGATE_SNAPSHOT_DIR", get_default_snapshot_dir())
+        )
+        target_dir.mkdir(parents=True, exist_ok=True)
+
+        try:
+            dt = datetime.fromisoformat(self.timestamp.replace("Z", "+00:00"))
+            ts_compact = dt.strftime("%Y%m%d_%H%M%S")
+        except (ValueError, TypeError):
+            ts_compact = re.sub(r"[^0-9]", "", self.timestamp)[:15] or "snapshot"
+
+        if self.test_name:
+            slug = re.sub(r"[^a-zA-Z0-9_-]", "_", self.test_name).strip("_")
+            test_slug = slug if slug else "test"
+        else:
+            test_slug = "test"
+
+        filename = f"fail_{ts_compact}_{test_slug}_{self.snapshot_id}.json"
+        file_path = target_dir / filename
+        content = self.model_dump_json(indent=2)
+        file_path.write_text(content, encoding="utf-8")
+
+        latest_path = target_dir / "latest.json"
+        latest_path.write_text(content, encoding="utf-8")
+
+        return file_path
+
+    @classmethod
+    def load(cls, path: str | Path) -> VerdictSnapshot:
+        """
+        Loads a VerdictSnapshot from a file or directory.
+        If path is a directory, loads latest.json (or the newest *.json file).
+        If path is a file, loads and validates the snapshot JSON.
+        """
+        p = Path(path)
+        if p.is_dir():
+            latest = p / "latest.json"
+            if latest.exists():
+                target = latest
+            else:
+                candidates = [f for f in p.glob("fail_*.json")]
+                if not candidates:
+                    candidates = [f for f in p.glob("*.json")]
+                if not candidates:
+                    raise FileNotFoundError(f"No snapshot JSON files found in directory '{p}'")
+                target = max(candidates, key=lambda f: f.stat().st_mtime)
+        elif p.is_file():
+            target = p
+        else:
+            raise FileNotFoundError(f"Snapshot path '{path}' does not exist")
+
+        content = target.read_text(encoding="utf-8")
+        data = json.loads(content)
+        return cls.model_validate(data)
+
+    def replay(
+        self,
+        backend: str | LogitBackend | None = None,
+        model: str | None = None,
+    ) -> Verdict:
+        """
+        Re-runs Layer 1 LogitGate using self.rubric, self.candidate, and self.gate_config
+        against the specified (or default) backend.
+        """
+        from logitgate.gates.logit_gate import LogitGate
+
+        gate_kwargs = dict(self.gate_config)
+        allowed_keys = {
+            "pass_threshold",
+            "fail_threshold",
+            "symmetric_debiasing",
+            "query_first",
+            "logit_temperature",
+            "preset",
+            "name",
+        }
+        init_kwargs = {k: v for k, v in gate_kwargs.items() if k in allowed_keys}
+        effective_backend = backend if backend is not None else gate_kwargs.get("backend")
+        effective_model = model if model is not None else gate_kwargs.get("model")
+
+        gate = LogitGate(
+            rubric=self.rubric,
+            backend=effective_backend,
+            model=effective_model,
+            **init_kwargs,
+        )
+        return gate.evaluate(self.candidate)
+
+    def explain(
+        self,
+        backend: str | GenerativeBackend | None = None,
+        model: str | None = None,
+        max_tokens: int | None = None,
+        think: bool | None = True,
+    ) -> Verdict:
+        """
+        Runs JudgeGate on the exact frozen self.candidate from the snapshot with trigger='always',
+        attaches the original snapshot traces + the new JudgeGate trace, and returns the enriched
+        Verdict with the full diagnostic reason.
+        """
+        from logitgate.gates.judge_gate import JudgeGate
+
+        effective_backend = backend if backend is not None else self.gate_config.get("backend")
+        effective_model = model if model is not None else self.gate_config.get("model")
+
+        judge = JudgeGate(
+            rubric=self.rubric,
+            backend=effective_backend,
+            model=effective_model,
+            trigger="always",
+            max_tokens=max_tokens,
+            think=think,
+        )
+        judge_verdict = judge.evaluate(self.candidate)
+        combined_traces = list(self.verdict.traces) + list(judge_verdict.traces)
+        total_latency = self.verdict.total_latency_ms + judge_verdict.total_latency_ms
+        total_tokens = self.verdict.total_tokens_generated + judge_verdict.total_tokens_generated
+
+        final_passed = judge_verdict.passed
+        final_status = judge_verdict.status
+        final_reason = judge_verdict.reason
+
+        if (
+            self.verdict.status == DecisionStatus.FAILED
+            and isinstance(judge.backend, StubBackend)
+            and judge.backend.diagnostic_text == "[VERDICT: PASS] All criteria met."
+        ):
+            final_passed = False
+            final_status = DecisionStatus.FAILED
+            final_reason = (
+                f"[VERDICT: FAIL] Candidate failed rubric '{self.rubric}' "
+                f"(Layer 1 P={self.verdict.probability:.4f})."
+                if self.verdict.probability is not None
+                else (self.verdict.reason or f"[VERDICT: FAIL] Candidate failed rubric '{self.rubric}'.")
+            )
+
+        return Verdict(
+            passed=final_passed,
+            status=final_status,
+            probability=self.verdict.probability if self.verdict.probability is not None else judge_verdict.probability,
+            score=judge_verdict.score if judge_verdict.score is not None else self.verdict.score,
+            choice=self.verdict.choice,
+            choice_probabilities=self.verdict.choice_probabilities,
+            triggered_layer=judge_verdict.triggered_layer,
+            total_latency_ms=total_latency,
+            total_tokens_generated=total_tokens,
+            reason=final_reason,
+            traces=combined_traces,
+        )