logitgate 0.2.0__py3-none-any.whl

logitgate/cli.py

@@ -0,0 +1,264 @@
+from __future__ import annotations
+
+import argparse
+import sys
+from collections.abc import Sequence
+
+from logitgate.cascade import Cascade
+from logitgate.gates.ast_gate import ASTGate
+from logitgate.gates.logit_gate import LogitGate
+
+
+def create_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="logitgate",
+        description="LogitGate CLI: Calibrated first-token logit evaluation and cascaded routing.",
+    )
+    subparsers = parser.add_subparsers(dest="command", help="Available commands")
+
+    eval_parser = subparsers.add_parser(
+        "eval",
+        help="Evaluate candidate artifact against a rubric",
+    )
+    eval_parser.add_argument(
+        "--rubric",
+        required=True,
+        help="Evaluation rubric or question to verify against candidate",
+    )
+    eval_parser.add_argument(
+        "--candidate",
+        type=str,
+        default=None,
+        help="Candidate text/artifact to evaluate",
+    )
+    eval_parser.add_argument(
+        "--file",
+        type=str,
+        default=None,
+        help="Path to file containing candidate text to evaluate",
+    )
+    eval_parser.add_argument(
+        "--backend",
+        type=str,
+        default=None,
+        help="Backend specifier (defaults to LOGITGATE_BACKEND or stub://default)",
+    )
+    eval_parser.add_argument(
+        "--model",
+        type=str,
+        default=None,
+        help="Optional model name for backend",
+    )
+    eval_parser.add_argument(
+        "--threshold",
+        type=float,
+        default=None,
+        help="Pass probability threshold (defaults to LOGITGATE_PASS_THRESHOLD or 0.85)",
+    )
+    eval_parser.add_argument(
+        "--fail-threshold",
+        type=float,
+        default=None,
+        help="Fail probability threshold (defaults to LOGITGATE_FAIL_THRESHOLD or 0.20)",
+    )
+    eval_parser.add_argument(
+        "--ast-syntax",
+        action="store_true",
+        help="Enable Layer 0 AST syntax validation before logit evaluation",
+    )
+    eval_parser.add_argument(
+        "--json",
+        action="store_true",
+        help="Output verdict and traces in JSON format",
+    )
+
+    cal_parser = subparsers.add_parser(
+        "calibrate",
+        help="Run micro-calibration dataset to fit and cache optimal logit temperature",
+    )
+    cal_parser.add_argument(
+        "--backend",
+        type=str,
+        default=None,
+        help="Backend specifier (defaults to LOGITGATE_BACKEND or stub://default)",
+    )
+    cal_parser.add_argument(
+        "--model",
+        type=str,
+        default=None,
+        help="Optional model name for backend",
+    )
+    cal_parser.add_argument(
+        "--cache-path",
+        type=str,
+        default=None,
+        help="Optional custom path to calibration cache JSON file",
+    )
+
+    explain_parser = subparsers.add_parser(
+        "explain",
+        help="Replay frozen failure snapshot and generate full Layer-2 diagnostic explanation",
+    )
+    explain_parser.add_argument(
+        "snapshot_path",
+        nargs="?",
+        default=".logitgate/failures",
+        help="Path to snapshot JSON file or directory containing latest.json (default: .logitgate/failures)",
+    )
+    explain_parser.add_argument(
+        "--backend",
+        type=str,
+        default=None,
+        help="Generative backend specifier (e.g. stub://..., ollama://...)",
+    )
+    explain_parser.add_argument(
+        "--model",
+        type=str,
+        default=None,
+        help="Optional model name for backend",
+    )
+    explain_parser.add_argument(
+        "--max-tokens",
+        type=int,
+        default=None,
+        help="Maximum tokens for generative explanation",
+    )
+    explain_parser.add_argument(
+        "--no-think",
+        action="store_true",
+        help="Disable Chain-of-Thought reasoning tokens in generative explanation",
+    )
+    explain_parser.add_argument(
+        "--json",
+        action="store_true",
+        help="Output enriched verdict and traces in JSON format",
+    )
+
+    return parser
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    parser = create_parser()
+    args = parser.parse_args(argv)
+
+    if args.command == "calibrate":
+        from pathlib import Path
+
+        from logitgate.backends.base import resolve_logit_backend
+        from logitgate.debiasing.permutation import auto_calibrate_backend
+
+        backend = resolve_logit_backend(args.backend, model=args.model)
+        cache_path = Path(args.cache_path) if args.cache_path else None
+        effective_model = args.model or getattr(backend, "model", "default")
+
+        print(f"Calibrating model '{effective_model}' on backend {backend.__class__.__name__}...")
+        result = auto_calibrate_backend(
+            backend,
+            model_key=effective_model,
+            save_cache=True,
+            cache_path=cache_path,
+        )
+        print(f"Optimal Temperature: {result.optimal_temperature:.4f}")
+        print(f"Brier Score: {result.brier_score_before:.6f} -> {result.brier_score_after:.6f}")
+        print(f"ECE: {result.ece_before:.6f} -> {result.ece_after:.6f}")
+        print("Calibration successfully saved to cache.")
+        return 0
+
+    if args.command == "explain":
+        from logitgate.snapshot import VerdictSnapshot
+        try:
+            snapshot = VerdictSnapshot.load(args.snapshot_path)
+        except (OSError, ValueError, KeyError) as e:
+            sys.stderr.write(f"Error loading snapshot from '{args.snapshot_path}': {e}\n")
+            return 1
+
+        think = not args.no_think
+        verdict = snapshot.explain(
+            backend=args.backend,
+            model=args.model,
+            max_tokens=args.max_tokens,
+            think=think,
+        )
+
+        if args.json:
+            print(verdict.model_dump_json(indent=2))
+        else:
+            print("=" * 60)
+            print("LOGITGATE FAILURE REPLAY & EXPLANATION")
+            print("=" * 60)
+            print(f"Snapshot ID:     {snapshot.snapshot_id}")
+            if snapshot.test_name:
+                print(f"Test Name:       {snapshot.test_name}")
+            print(f"Timestamp:       {snapshot.timestamp}")
+            print(f"Rubric:          {snapshot.rubric}")
+            orig_prob = snapshot.verdict.probability
+            prob_str = f"{orig_prob:.4f}" if orig_prob is not None else "N/A"
+            print(f"Original Prob:   {prob_str}")
+            print(f"Triggered Layer: {snapshot.verdict.triggered_layer}")
+            print("-" * 60)
+            print(f"Replay Verdict:  {verdict.status.value.upper()}")
+            print(f"Replay Latency:  {verdict.total_latency_ms:.2f}ms")
+            print(f"Tokens Spent:    {verdict.total_tokens_generated}")
+            print("-" * 60)
+            print("Diagnostic Critique:")
+            print(verdict.reason or "(No reason provided)")
+            print("=" * 60)
+        return 0
+
+    if args.command != "eval":
+        parser.print_help()
+        return 1
+
+    candidate: str | None = None
+    if args.file:
+        try:
+            with open(args.file, "r", encoding="utf-8") as f:
+                candidate = f.read()
+        except OSError as e:
+            sys.stderr.write(f"Error reading file '{args.file}': {e}\n")
+            return 1
+    elif args.candidate is not None:
+        candidate = args.candidate
+    else:
+        if not sys.stdin.isatty():
+            candidate = sys.stdin.read()
+        else:
+            sys.stderr.write("Error: Either --candidate or --file must be specified.\n")
+            return 1
+
+    logit_gate = LogitGate(
+        rubric=args.rubric,
+        backend=args.backend,
+        model=args.model,
+        pass_threshold=args.threshold,
+        fail_threshold=args.fail_threshold,
+    )
+
+    evaluator: Cascade | LogitGate
+    if args.ast_syntax:
+        evaluator = Cascade([
+            ASTGate(syntax=True),
+            logit_gate,
+        ])
+    else:
+        evaluator = logit_gate
+
+    verdict = evaluator.evaluate(candidate)
+
+    if args.json:
+        print(verdict.model_dump_json(indent=2))
+    else:
+        verdict_str = "PASSED" if verdict.passed else "FAILED"
+        print(f"Verdict: {verdict_str} (status: {verdict.status.value})")
+        if verdict.probability is not None:
+            print(f"Confidence: {verdict.probability:.4f}")
+        print(f"Triggered Layer: {verdict.triggered_layer}")
+        print(f"Total Latency: {verdict.total_latency_ms:.2f}ms")
+        if verdict.reason:
+            print(f"Reason: {verdict.reason}")
+
+    return 0 if verdict.passed else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

logitgate/config.py

@@ -0,0 +1,172 @@
+from __future__ import annotations
+
+import os
+
+
+class _ConfigState:
+    def __init__(self) -> None:
+        self.backend: str | None = None
+        self.model: str | None = None
+        self.pass_threshold: float | None = None
+        self.fail_threshold: float | None = None
+        self.logit_temperature: float | None = None
+        self.num_ctx: int | None = None
+        self.save_failures: bool | None = None
+        self.snapshot_dir: str | None = None
+
+    def reset(self) -> None:
+        self.backend = None
+        self.model = None
+        self.pass_threshold = None
+        self.fail_threshold = None
+        self.logit_temperature = None
+        self.num_ctx = None
+        self.save_failures = None
+        self.snapshot_dir = None
+
+
+_state = _ConfigState()
+
+
+class _DefaultsMeta(type):
+    @property
+    def backend(cls) -> str:
+        if _state.backend is not None:
+            return _state.backend
+        return os.environ.get("LOGITGATE_BACKEND", "stub://default")
+
+    @backend.setter
+    def backend(cls, val: str | None) -> None:
+        _state.backend = val
+
+    @property
+    def model(cls) -> str | None:
+        if _state.model is not None:
+            return _state.model
+        return os.environ.get("LOGITGATE_MODEL", None)
+
+    @model.setter
+    def model(cls, val: str | None) -> None:
+        _state.model = val
+
+    @property
+    def pass_threshold(cls) -> float:
+        if _state.pass_threshold is not None:
+            return _state.pass_threshold
+        val = os.environ.get("LOGITGATE_PASS_THRESHOLD")
+        return float(val) if val is not None else 0.85
+
+    @pass_threshold.setter
+    def pass_threshold(cls, val: float | None) -> None:
+        _state.pass_threshold = val
+
+    @property
+    def fail_threshold(cls) -> float:
+        if _state.fail_threshold is not None:
+            return _state.fail_threshold
+        val = os.environ.get("LOGITGATE_FAIL_THRESHOLD")
+        return float(val) if val is not None else 0.20
+
+    @fail_threshold.setter
+    def fail_threshold(cls, val: float | None) -> None:
+        _state.fail_threshold = val
+
+    @property
+    def logit_temperature(cls) -> float:
+        if _state.logit_temperature is not None:
+            return _state.logit_temperature
+        val = os.environ.get("LOGITGATE_TEMPERATURE")
+        return float(val) if val is not None else 1.0
+
+    @logit_temperature.setter
+    def logit_temperature(cls, val: float | None) -> None:
+        _state.logit_temperature = val
+
+    @property
+    def num_ctx(cls) -> int | None:
+        if _state.num_ctx is not None:
+            return _state.num_ctx
+        val = os.environ.get("LOGITGATE_NUM_CTX")
+        return int(val) if val is not None else None
+
+    @num_ctx.setter
+    def num_ctx(cls, val: int | None) -> None:
+        _state.num_ctx = val
+
+    @property
+    def save_failures(cls) -> bool:
+        if _state.save_failures is not None:
+            return _state.save_failures
+        return os.environ.get("LOGITGATE_SAVE_FAILURES", "0").lower() in ("1", "true", "yes")
+
+    @save_failures.setter
+    def save_failures(cls, val: bool | None) -> None:
+        _state.save_failures = val
+
+    @property
+    def snapshot_dir(cls) -> str:
+        if _state.snapshot_dir is not None:
+            return _state.snapshot_dir
+        return os.environ.get("LOGITGATE_SNAPSHOT_DIR", ".logitgate/failures")
+
+    @snapshot_dir.setter
+    def snapshot_dir(cls, val: str | None) -> None:
+        _state.snapshot_dir = val
+
+
+class LogitGateDefaults(metaclass=_DefaultsMeta):
+    """Global configuration defaults for LogitGate."""
+
+
+def configure(
+    backend: str | None = None,
+    model: str | None = None,
+    pass_threshold: float | None = None,
+    fail_threshold: float | None = None,
+    logit_temperature: float | None = None,
+    num_ctx: int | None = None,
+    save_failures: bool | None = None,
+    snapshot_dir: str | None = None,
+) -> None:
+    """Configures global defaults for LogitGate."""
+    if backend is not None:
+        _state.backend = backend
+    if model is not None:
+        _state.model = model
+    if pass_threshold is not None:
+        _state.pass_threshold = pass_threshold
+    if fail_threshold is not None:
+        _state.fail_threshold = fail_threshold
+    if logit_temperature is not None:
+        _state.logit_temperature = logit_temperature
+    if num_ctx is not None:
+        _state.num_ctx = num_ctx
+    if save_failures is not None:
+        _state.save_failures = save_failures
+    if snapshot_dir is not None:
+        _state.snapshot_dir = snapshot_dir
+
+
+def reset_config() -> None:
+    """Resets global configuration to environment variables and defaults."""
+    _state.reset()
+
+
+def get_default_backend() -> str:
+    """Resolves default backend string via configure() or LOGITGATE_BACKEND."""
+    return LogitGateDefaults.backend
+
+
+def get_default_model() -> str | None:
+    """Resolves default model string via configure() or LOGITGATE_MODEL."""
+    return LogitGateDefaults.model
+
+
+def get_default_save_failures() -> bool:
+    """Resolves default failure saving flag via configure() or LOGITGATE_SAVE_FAILURES."""
+    return LogitGateDefaults.save_failures
+
+
+def get_default_snapshot_dir() -> str:
+    """Resolves default snapshot directory via configure() or LOGITGATE_SNAPSHOT_DIR."""
+    return LogitGateDefaults.snapshot_dir

logitgate/debiasing/__init__.py

@@ -0,0 +1,35 @@
+from logitgate.debiasing.inversion import (
+    compile_batch_binary_prompt,
+    compile_binary_prompt,
+    compile_choice_prompt,
+    compile_score_prompt,
+)
+from logitgate.debiasing.permutation import (
+    KNOWN_MODEL_TEMPERATURES,
+    CalibrationResult,
+    auto_calibrate_backend,
+    combine_symmetric_log_odds,
+    fit_calibration,
+    get_model_temperature,
+)
+from logitgate.debiasing.softmax import (
+    compute_binary_log_odds,
+    compute_normalized_probabilities,
+    extract_token_logprob,
+)
+
+__all__ = [
+    "KNOWN_MODEL_TEMPERATURES",
+    "CalibrationResult",
+    "auto_calibrate_backend",
+    "combine_symmetric_log_odds",
+    "compile_batch_binary_prompt",
+    "compile_binary_prompt",
+    "compile_choice_prompt",
+    "compile_score_prompt",
+    "compute_binary_log_odds",
+    "compute_normalized_probabilities",
+    "extract_token_logprob",
+    "fit_calibration",
+    "get_model_temperature",
+]

logitgate/debiasing/inversion.py

@@ -0,0 +1,180 @@
+from __future__ import annotations
+
+from collections.abc import Mapping, Sequence
+from typing import Any
+
+
+def compile_binary_prompt(
+    rubric: str,
+    candidate: Any,
+    pass_token: str = "A",
+    fail_token: str = "B",
+    query_first: bool = True,
+    pass_description: str = "The candidate artifact complies fully with the rubric.",
+    fail_description: str = "The candidate artifact violates the rubric or has defects.",
+) -> str:
+    """
+    Compiles a prefix-cache-friendly binary evaluation prompt.
+    When query_first=True, the rubric and candidate artifact form a 100% byte-identical
+    static prefix across both Pass 1 and Pass 2, allowing KV-cache hit rates > 99%.
+    """
+    cand_str = str(candidate).strip()
+    rubric_str = rubric.strip()
+
+    if query_first:
+        prefix = f"[EVALUATION RUBRIC]\n{rubric_str}\n\n[CANDIDATE ARTIFACT]\n{cand_str}\n\n"
+    else:
+        prefix = f"[CANDIDATE ARTIFACT]\n{cand_str}\n\n[EVALUATION RUBRIC]\n{rubric_str}\n\n"
+
+    t_pass = pass_token.strip().upper()
+    t_fail = fail_token.strip().upper()
+
+    mapping = {
+        t_pass: pass_description,
+        t_fail: fail_description,
+    }
+    sorted_tokens = sorted(mapping.keys())
+    option_lines = "\n".join(f"{tok}: {mapping[tok]}" for tok in sorted_tokens)
+    tokens_str = f"{sorted_tokens[0]} or {sorted_tokens[1]}" if len(sorted_tokens) == 2 else " or ".join(sorted_tokens)
+
+    suffix = (
+        f"[DECISION MAPPING]\n"
+        f"{option_lines}\n"
+        f"Respond with a single character ({tokens_str}).\n"
+        f"Verdict:"
+    )
+    return prefix + suffix
+
+
+def compile_batch_binary_prompt(
+    rubrics: Sequence[str],
+    candidate: Any,
+    rubric_index: int,
+    pass_token: str = "A",
+    fail_token: str = "B",
+    query_first: bool = False,
+    pass_description: str = "The candidate artifact complies fully with the rubric.",
+    fail_description: str = "The candidate artifact violates the rubric or has defects.",
+) -> str:
+    """
+    Compiles an isolated, prefix-cache-friendly binary evaluation prompt for an individual rubric
+    within a multi-rubric evaluation set.
+
+    Architecture:
+    The candidate artifact forms the shared static prefix:
+        [CANDIDATE ARTIFACT]
+        {candidate}
+    This prefix is 100% byte-identical across ALL K rubrics and across both forward (Pass 1) and
+    reverse (Pass 2) passes, allowing the inference server's KV-cache to retain the candidate tokens
+    across all queries.
+
+    Crucially, each rubric is evaluated in complete isolation: no other rubrics are loaded into the
+    prompt context. This prevents cross-rubric attention leakage and context bloat, scaling
+    consistently whether evaluating 2 rubrics or 500 rubrics.
+    """
+    if rubric_index < 0 or rubric_index >= len(rubrics):
+        raise IndexError(f"rubric_index {rubric_index} out of bounds for rubrics list of size {len(rubrics)}")
+
+    cand_str = str(candidate).strip()
+    target_rubric = rubrics[rubric_index].strip()
+
+    t_pass = pass_token.strip().upper()
+    t_fail = fail_token.strip().upper()
+
+    mapping = {
+        t_pass: pass_description,
+        t_fail: fail_description,
+    }
+    sorted_tokens = sorted(mapping.keys())
+    option_lines = "\n".join(f"{tok}: {mapping[tok]}" for tok in sorted_tokens)
+    tokens_str = f"{sorted_tokens[0]} or {sorted_tokens[1]}" if len(sorted_tokens) == 2 else " or ".join(sorted_tokens)
+
+    if not query_first:
+        prefix = f"[CANDIDATE ARTIFACT]\n{cand_str}\n\n"
+        suffix = (
+            f"[EVALUATION RUBRIC]\n"
+            f"{target_rubric}\n\n"
+            f"[DECISION MAPPING]\n"
+            f"{option_lines}\n"
+            f"Respond with a single character ({tokens_str}).\n"
+            f"Verdict:"
+        )
+    else:
+        prefix = f"[EVALUATION RUBRIC]\n{target_rubric}\n\n[CANDIDATE ARTIFACT]\n{cand_str}\n\n"
+        suffix = (
+            f"[DECISION MAPPING]\n"
+            f"{option_lines}\n"
+            f"Respond with a single character ({tokens_str}).\n"
+            f"Verdict:"
+        )
+
+    return prefix + suffix
+
+
+def compile_choice_prompt(
+    question: str,
+    candidate: Any,
+    options: Mapping[str, str],
+    query_first: bool = True,
+) -> str:
+    """
+    Compiles a discrete multi-choice prompt with identical static prefix.
+    """
+    cand_str = str(candidate).strip()
+    q_str = question.strip()
+
+    if query_first:
+        prefix = f"[QUESTION]\n{q_str}\n\n[CANDIDATE ARTIFACT]\n{cand_str}\n\n"
+    else:
+        prefix = f"[CANDIDATE ARTIFACT]\n{cand_str}\n\n[QUESTION]\n{q_str}\n\n"
+
+    option_lines = "\n".join(f"{k}: {v}" for k, v in options.items())
+    keys = list(options.keys())
+    keys_str = " or ".join(keys)
+
+    suffix = (
+        f"[DECISION MAPPING]\n"
+        f"{option_lines}\n"
+        f"Respond with a single character ({keys_str}).\n"
+        f"Choice:"
+    )
+    return prefix + suffix
+
+
+def compile_score_prompt(
+    rubric: str,
+    candidate: Any,
+    min_score: int = 1,
+    max_score: int = 5,
+    query_first: bool = True,
+    inverted: bool = False,
+) -> str:
+    """
+    Compiles a bounded integer scoring prompt with identical static prefix.
+    When inverted=True, min_score corresponds to highest quality and max_score
+    corresponds to lowest quality, enabling symmetric score debiasing with
+    byte-identical prefix caching.
+    """
+    cand_str = str(candidate).strip()
+    rubric_str = rubric.strip()
+
+    if query_first:
+        prefix = f"[EVALUATION RUBRIC]\n{rubric_str}\n\n[CANDIDATE ARTIFACT]\n{cand_str}\n\n"
+    else:
+        prefix = f"[CANDIDATE ARTIFACT]\n{cand_str}\n\n[EVALUATION RUBRIC]\n{rubric_str}\n\n"
+
+    if inverted:
+        scale_desc = (
+            f"Rate the candidate artifact on an inverted scale where {min_score} is highest quality "
+            f"and {max_score} is lowest quality based on the rubric.\n"
+        )
+    else:
+        scale_desc = f"Rate the candidate artifact from {min_score} to {max_score} based on the rubric.\n"
+
+    suffix = (
+        f"[SCORING SCALE]\n"
+        f"{scale_desc}"
+        f"Respond with a single integer ({min_score}-{max_score}).\n"
+        f"Score:"
+    )
+    return prefix + suffix