invarlock 0.2.0__py3-none-any.whl

invarlock/guards_ref/__init__.py

@@ -0,0 +1,15 @@
+"""Reference decision kernels for guards (pure, side-effect-free).
+
+Exposes small math-first helpers used by property/differential tests.
+"""
+
+from .rmt_ref import rmt_decide
+from .spectral_ref import bh_select, spectral_decide
+from .variance_ref import variance_decide
+
+__all__ = [
+    "bh_select",
+    "spectral_decide",
+    "rmt_decide",
+    "variance_decide",
+]

invarlock/guards_ref/rmt_ref.py

@@ -0,0 +1,40 @@
+from __future__ import annotations
+
+import math
+from collections.abc import Mapping
+
+
+def rmt_decide(
+    bare_by_family: Mapping[str, int],
+    guarded_by_family: Mapping[str, int],
+    epsilon_by_family: Mapping[str, float],
+) -> dict[str, object]:
+    """
+    Reference epsilon-rule decision for RMT.
+
+    Allowed excess A_f = ceil(epsilon_f * max(1, b_f)).
+    PASS iff for all families Δ_f <= A_f and sum Δ_f <= sum A_f.
+    """
+    families = set(bare_by_family) | set(guarded_by_family) | set(epsilon_by_family)
+    delta_by_family: dict[str, int] = {}
+    allowed_by_family: dict[str, int] = {}
+    sum_delta = 0
+    sum_allowed = 0
+    for f in families:
+        b = int(bare_by_family.get(f, 0) or 0)
+        g = int(guarded_by_family.get(f, 0) or 0)
+        eps = float(epsilon_by_family.get(f, 0.0) or 0.0)
+        d = g - b
+        a = int(math.ceil(eps * max(1, b)))
+        delta_by_family[f] = d
+        allowed_by_family[f] = a
+        sum_delta += d
+        sum_allowed += a
+    ok = all(delta_by_family[f] <= allowed_by_family[f] for f in families) and (
+        sum_delta <= sum_allowed
+    )
+    return {
+        "pass": ok,
+        "delta_by_family": delta_by_family,
+        "allowed_by_family": allowed_by_family,
+    }

invarlock/guards_ref/spectral_ref.py

@@ -0,0 +1,135 @@
+from __future__ import annotations
+
+import math
+from collections.abc import Mapping
+
+
+def bh_select(pvals: list[float], alpha: float) -> list[bool]:
+    """
+    Benjamini–Hochberg procedure. Returns boolean mask of rejections in input order.
+
+    Preconditions: 0 < alpha <= 1; p in [0,1] or NaN (NaN => reject=False).
+    """
+    n = len(pvals)
+    if n == 0:
+        return []
+    alpha = float(alpha)
+    if not (0.0 < alpha <= 1.0):
+        # Treat invalid alpha as no rejections to be conservative
+        return [False] * n
+
+    # Sort by p-value ascending while remembering original indices
+    order = sorted(
+        range(n), key=lambda i: (float("inf") if not _finite01(pvals[i]) else pvals[i])
+    )
+    rejs_sorted = [False] * n
+    max_k = 0
+    for rank, idx in enumerate(order, start=1):
+        p = pvals[idx]
+        if not _finite01(p):
+            continue
+        threshold = (alpha * rank) / n
+        if p <= threshold:
+            max_k = rank
+    # Mark as rejected those with p <= (alpha * max_k / n)
+    if max_k > 0:
+        cutoff = (alpha * max_k) / n
+        for idx in order:
+            p = pvals[idx]
+            if _finite01(p) and p <= cutoff:
+                rejs_sorted[idx] = True
+    # Return in original order
+    return rejs_sorted
+
+
+def spectral_decide(
+    sigma_by_name: Mapping[str, float],
+    default_denom_by_name: Mapping[str, float],
+    family_of_name: Mapping[str, str],
+    deadband: float,
+    caps_by_family: Mapping[str, float],
+    mtest: Mapping[str, object] | None = None,
+) -> dict[str, object]:
+    """
+    Pure decision kernel for spectral guard.
+
+    - z_i = ((sigma_i / denom_i) - 1) / max(deadband, eps)
+    - p_i = Phi(|z_i|) under standard normal tail (two-sided conservative mapping)
+    - Multiple testing per method; then cap by family kappa.
+    """
+    eps = 1e-12
+    dead = max(float(deadband or 0.0), 0.0)
+
+    names = list(sigma_by_name.keys())
+    z_by_name: dict[str, float] = {}
+    for name in names:
+        s = float(sigma_by_name.get(name, 0.0) or 0.0)
+        d = float(default_denom_by_name.get(name, 1.0) or 1.0)
+        d = d if d > 0.0 else 1.0
+        rel = (s / d) - 1.0
+        z = 0.0
+        if abs(rel) > dead:
+            z = rel / max(dead, eps)
+        z_by_name[name] = z
+
+    # Map to two-sided p-values via complementary error function (normal)
+    # p = 2 * (1 - Phi(|z|)) = erfc(|z| / sqrt(2))
+    try:
+        import math as _m
+
+        def _p(z: float) -> float:
+            return float(_m.erfc(abs(z) / math.sqrt(2.0)))
+    except Exception:
+
+        def _p(z: float) -> float:  # pragma: no cover
+            return 1.0
+
+    pvals = [_p(z_by_name[n]) for n in names]
+    method_obj = (mtest or {}).get("method", "bh")
+    method = str(method_obj).lower()
+    alpha_obj = (mtest or {}).get("alpha", 0.05)
+    try:
+        alpha = float(alpha_obj)  # type: ignore[arg-type]
+    except Exception:
+        alpha = 0.05
+    if method in {"bh", "benjamini-hochberg", "benjamini_hochberg"}:
+        rejects = bh_select(pvals, alpha)
+    elif method in {"bonferroni"}:
+        cutoff = alpha / max(1, len(pvals))
+        rejects = [bool(p <= cutoff) if _finite01(p) else False for p in pvals]
+    else:
+        # Unknown method: conservative
+        rejects = [False] * len(pvals)
+
+    # Apply per-family caps (kappa) after selection: greedily keep top-|z| per family up to kappa
+    fam_map = {n: str(family_of_name.get(n, "other")) for n in names}
+    selected: list[str] = []
+    per_family_counts: dict[str, int] = {}
+    # Sort by |z| descending; pick among rejected set
+    for name in sorted(names, key=lambda n: abs(z_by_name[n]), reverse=True):
+        if not rejects[names.index(name)]:
+            continue
+        fam = fam_map[name]
+        kappa = float(caps_by_family.get(fam, float("inf")) or float("inf"))
+        curr = per_family_counts.get(fam, 0)
+        if curr < int(math.ceil(kappa)):
+            per_family_counts[fam] = curr + 1
+            selected.append(name)
+
+    return {
+        "pass": len(selected) == 0,
+        "selected": selected,
+        "z_by_name": z_by_name,
+        "per_family_counts": per_family_counts,
+    }
+
+
+def _finite01(p: float) -> bool:
+    try:
+        return (
+            (isinstance(p, int | float))
+            and math.isfinite(p)
+            and (0.0 <= float(p) <= 1.0)
+        )
+    except Exception:
+        return False

invarlock/guards_ref/variance_ref.py

@@ -0,0 +1,60 @@
+from __future__ import annotations
+
+
+def variance_decide(
+    mean_delta: float,
+    ci: tuple[float, float] | list[float],
+    direction: str,  # "lower" or "higher" is better
+    min_effect: float,
+    predictive_one_sided: bool,
+) -> dict[str, object]:
+    """
+    Reference predictive gate decision.
+
+    For direction=="lower", negative deltas are improvements (Δ<0 better).
+    For direction=="higher", flip sign so that improvements are treated consistently.
+    """
+    if not (isinstance(ci, tuple | list) and len(ci) == 2):
+        return {"evaluated": False, "pass": True, "reason": "ci_unavailable"}
+    lo, hi = float(ci[0]), float(ci[1])
+    mu = float(mean_delta)
+    me = float(min_effect or 0.0)
+
+    dir_norm = (direction or "lower").strip().lower()
+    # Normalize to "lower is better" frame
+    if dir_norm == "higher":
+        mu = -mu
+        lo, hi = -hi, -lo
+
+    # One-sided vs two-sided enablement semantics
+    if predictive_one_sided:
+        # Production parity: evaluate with one-sided criteria (no strict 0-exclusion required)
+        evaluated = True
+        if mu >= 0.0:
+            return {
+                "evaluated": evaluated,
+                "pass": False,
+                "reason": "mean_not_negative",
+            }
+        if me > 0.0 and (-mu) < me:
+            return {
+                "evaluated": evaluated,
+                "pass": False,
+                "reason": "gain_below_threshold",
+            }
+        if lo >= 0.0:
+            return {"evaluated": evaluated, "pass": False, "reason": "ci_contains_zero"}
+        return {"evaluated": evaluated, "pass": True, "reason": "ci_gain_met"}
+
+    # Two-sided enablement requires strict exclusion of 0 and sufficient effect
+    evaluated = (lo <= hi) and (abs(mu) >= me) and not (lo <= 0.0 <= hi)
+    if not evaluated:
+        return {"evaluated": False, "pass": True, "reason": "not_evaluated"}
+
+    # Two-sided: require CI strictly below zero with gain >= min_effect
+    if hi >= 0.0:
+        return {"evaluated": True, "pass": False, "reason": "ci_contains_zero"}
+    gain_lower = -hi
+    if gain_lower < me:
+        return {"evaluated": True, "pass": False, "reason": "gain_below_threshold"}
+    return {"evaluated": True, "pass": True, "reason": "ci_gain_met"}

invarlock/model_profile.py

@@ -0,0 +1,353 @@
+from __future__ import annotations
+
+import hashlib
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from typing import Any
+
+try:
+    from transformers import AutoTokenizer, GPT2Tokenizer
+    from transformers.tokenization_utils_base import PreTrainedTokenizerBase
+except Exception:  # pragma: no cover - exercised only when transformers is absent
+    AutoTokenizer = None  # type: ignore[assignment]
+    GPT2Tokenizer = None  # type: ignore[assignment]
+
+    class PreTrainedTokenizerBase:  # type: ignore[no-redef]
+        """Lightweight stub used when transformers is not installed."""
+
+        def __call__(self, *args: Any, **kwargs: Any) -> dict[str, Any]:
+            raise RuntimeError(
+                "Tokenization requires the 'transformers' extra. "
+                "Install it with: pip install 'invarlock[adapters]'."
+            )
+
+
+TokenizerFactory = Callable[[], tuple[PreTrainedTokenizerBase, str]]
+
+
+def _hash_tokenizer(tokenizer: PreTrainedTokenizerBase) -> str:
+    try:
+        if hasattr(tokenizer, "get_vocab"):
+            vocab_mapping = tokenizer.get_vocab()
+        else:
+            vocab_mapping = getattr(tokenizer, "vocab", {})
+        if hasattr(vocab_mapping, "items"):
+            vocab_items = list(vocab_mapping.items())
+        else:
+            vocab_items = []
+    except Exception:
+        vocab_items = []
+
+    hasher = hashlib.blake2s(digest_size=16)
+    try:
+        for token, idx in sorted(vocab_items, key=lambda x: x[0]):
+            token_str = token if isinstance(token, str) else str(token)
+            hasher.update(token_str.encode("utf-8", "ignore"))
+            try:
+                hasher.update(int(idx).to_bytes(4, "little", signed=False))
+            except Exception:
+                hasher.update(str(idx).encode("utf-8", "ignore"))
+    except Exception:
+        return "unknown"
+
+    hasher.update(tokenizer.__class__.__name__.encode("utf-8", "ignore"))
+    name_path = getattr(tokenizer, "name_or_path", "")
+    hasher.update(str(name_path).encode("utf-8", "ignore"))
+    return hasher.hexdigest()
+
+
+@dataclass(frozen=True)
+class ModelProfile:
+    """Captured capabilities for a recognised model family."""
+
+    family: str
+    default_loss: str
+    make_tokenizer: TokenizerFactory
+    default_metric: str = "ppl_causal"
+    # Must correspond to a registered provider in invarlock.eval.data.get_provider
+    default_provider: str = "wikitext2"
+    module_selectors: dict[str, list[str]] = field(default_factory=dict)
+    invariants: tuple[str, ...] = ()
+    cert_lints: tuple[dict[str, str], ...] = ()
+
+
+def _bert_selectors() -> dict[str, list[str]]:
+    return {
+        "attention": [
+            "attention.self.query",
+            "attention.self.key",
+            "attention.self.value",
+            "attention.output.dense",
+        ],
+        "ffn": [
+            "intermediate.dense",
+            "output.dense",
+        ],
+    }
+
+
+def _gpt2_selectors() -> dict[str, list[str]]:
+    return {
+        "attention": [
+            "attn.c_attn",
+            "attn.c_proj",
+        ],
+        "ffn": [
+            "mlp.c_fc",
+            "mlp.c_proj",
+        ],
+    }
+
+
+def _llama_selectors() -> dict[str, list[str]]:
+    return {
+        "attention": [
+            "self_attn.q_proj",
+            "self_attn.k_proj",
+            "self_attn.v_proj",
+            "self_attn.o_proj",
+        ],
+        "ffn": [
+            "mlp.up_proj",
+            "mlp.down_proj",
+            "mlp.gate_proj",
+        ],
+    }
+
+
+def _unknown_selectors() -> dict[str, list[str]]:
+    return {
+        "attention": ["attention"],
+        "ffn": [],
+    }
+
+
+def _make_bert_tokenizer(model_id: str):
+    def factory() -> tuple[PreTrainedTokenizerBase, str]:
+        if AutoTokenizer is None:
+            raise RuntimeError(
+                "BERT tokenizers require the 'transformers' extra. "
+                "Install it with: pip install 'invarlock[adapters]'."
+            )
+        # Prefer offline/local cache first to respect network guard
+        tokenizer: PreTrainedTokenizerBase | None = None
+        try:
+            tokenizer = AutoTokenizer.from_pretrained(model_id, local_files_only=True)
+        except Exception:
+            # Try a common local BERT if specific model is not cached
+            try:
+                tokenizer = AutoTokenizer.from_pretrained(
+                    "bert-base-uncased", local_files_only=True
+                )
+            except Exception:
+                # If network is permitted, attempt remote fetch; otherwise propagate
+                try:
+                    tokenizer = AutoTokenizer.from_pretrained(model_id)
+                except Exception:
+                    tokenizer = None
+        if tokenizer is None:
+            raise RuntimeError(
+                "Unable to load a BERT tokenizer locally. Set INVARLOCK_ALLOW_NETWORK=1 "
+                "to allow fetching from the Hugging Face Hub, or pre-cache a BERT tokenizer."
+            )
+        if getattr(tokenizer, "mask_token", None) is None:
+            raise ValueError(
+                f"Tokenizer for '{model_id}' does not expose [MASK]; cannot run MLM evaluation."
+            )
+        if getattr(tokenizer, "pad_token", None) is None:
+            for candidate in (
+                getattr(tokenizer, "sep_token", None),
+                getattr(tokenizer, "cls_token", None),
+            ):
+                if candidate is not None:
+                    tokenizer.pad_token = candidate
+                    break
+        hash_value = _hash_tokenizer(tokenizer)
+        return tokenizer, hash_value
+
+    return factory
+
+
+def _make_gpt2_tokenizer(model_id: str):
+    def factory() -> tuple[PreTrainedTokenizerBase, str]:
+        if GPT2Tokenizer is None:
+            raise RuntimeError(
+                "GPT-2 tokenizers require the 'transformers' extra. "
+                "Install it with: pip install 'invarlock[adapters]'."
+            )
+        tokenizer = GPT2Tokenizer.from_pretrained("gpt2")
+        if tokenizer.pad_token is None:
+            tokenizer.pad_token = tokenizer.eos_token
+        hash_value = _hash_tokenizer(tokenizer)
+        return tokenizer, hash_value
+
+    return factory
+
+
+def _make_llama_tokenizer(model_id: str):
+    def factory() -> tuple[PreTrainedTokenizerBase, str]:
+        if AutoTokenizer is None and GPT2Tokenizer is None:
+            raise RuntimeError(
+                "LLaMA-style tokenizers require the 'transformers' extra. "
+                "Install it with: pip install 'invarlock[adapters]'."
+            )
+        # Try offline-first to respect InvarLock network guard; fall back to a
+        # local GPT-2 tokenizer if the model assets are not cached or network
+        # access is denied.
+        try:
+            tokenizer = AutoTokenizer.from_pretrained(model_id, local_files_only=True)
+        except Exception:
+            try:
+                tokenizer = AutoTokenizer.from_pretrained(model_id)
+            except Exception:
+                tokenizer = GPT2Tokenizer.from_pretrained("gpt2")
+        # Ensure padding/bos tokens are configured so downstream encoding
+        # yields stable non-zero ids and a valid attention mask regardless of
+        # environment defaults or tokenizer variants.
+        # Prefer EOS as pad token when no explicit pad token is defined.
+        if getattr(tokenizer, "pad_token", None) is None:
+            eos_token = getattr(tokenizer, "eos_token", None)
+            if eos_token is not None:
+                tokenizer.pad_token = eos_token
+        # Some LLaMA tokenizers default to not adding a BOS token on encode;
+        # enable it to guarantee at least one non-pad, non-zero token id.
+        if hasattr(tokenizer, "add_bos_token"):
+            try:
+                tokenizer.add_bos_token = True
+            except Exception:
+                pass
+        if getattr(tokenizer, "pad_token", None) is None:
+            raise ValueError(
+                f"Tokenizer for '{model_id}' does not define a pad token and no EOS fallback is available."
+            )
+        hash_value = _hash_tokenizer(tokenizer)
+        return tokenizer, hash_value
+
+    return factory
+
+
+def _make_unknown_tokenizer(model_id: str):
+    def factory() -> tuple[PreTrainedTokenizerBase, str]:
+        if AutoTokenizer is None and GPT2Tokenizer is None:
+            raise RuntimeError(
+                "Text tokenization requires the 'transformers' extra. "
+                "Install it with: pip install 'invarlock[adapters]'."
+            )
+        # Unknown families: try local-only first, then remote, then degrade to GPT-2
+        try:
+            tokenizer: PreTrainedTokenizerBase = AutoTokenizer.from_pretrained(
+                model_id, local_files_only=True
+            )
+        except Exception:
+            try:
+                tokenizer = AutoTokenizer.from_pretrained(model_id)
+            except Exception:
+                tokenizer = GPT2Tokenizer.from_pretrained("gpt2")
+        if getattr(tokenizer, "pad_token", None) is None:
+            eos_token = getattr(tokenizer, "eos_token", None)
+            if eos_token is not None:
+                tokenizer.pad_token = eos_token
+        hash_value = _hash_tokenizer(tokenizer)
+        return tokenizer, hash_value
+
+    return factory
+
+
+def detect_model_profile(model_id: str, adapter: str | None = None) -> ModelProfile:
+    """
+    Infer the model family and provide profile metadata used for evaluation.
+    """
+
+    adapter_lower = (adapter or "").lower()
+    model_lower = (model_id or "").lower()
+
+    if any(
+        keyword in adapter_lower for keyword in ("bert", "roberta", "deberta")
+    ) or any(keyword in model_lower for keyword in ("bert", "roberta", "deberta")):
+        return ModelProfile(
+            family="bert",
+            default_loss="mlm",
+            make_tokenizer=_make_bert_tokenizer(model_id),
+            default_metric="ppl_mlm",
+            default_provider="hf_text",
+            module_selectors=_bert_selectors(),
+            invariants=("mlm_mask_alignment",),
+            cert_lints=(
+                {
+                    "type": "equals",
+                    "path": "metrics.loss_type",
+                    "value": "mlm",
+                    "message": "BERT cert must record MLM loss type.",
+                },
+                {
+                    "type": "gte",
+                    "path": "metrics.masked_tokens_total",
+                    "value": "1",
+                    "message": "BERT cert must report masked tokens.",
+                },
+            ),
+        )
+
+    if any(keyword in adapter_lower for keyword in ("llama", "mistral", "qwen")) or any(
+        keyword in model_lower for keyword in ("llama", "mistral", "qwen")
+    ):
+        return ModelProfile(
+            family="llama",
+            default_loss="causal",
+            make_tokenizer=_make_llama_tokenizer(model_id),
+            default_metric="ppl_causal",
+            default_provider="wikitext2",
+            module_selectors=_llama_selectors(),
+            invariants=("rope_rotary_embedding",),
+            cert_lints=(
+                {
+                    "type": "equals",
+                    "path": "metrics.loss_type",
+                    "value": "causal",
+                    "message": "LLaMA cert should report causal loss.",
+                },
+            ),
+        )
+
+    if any(
+        keyword in adapter_lower for keyword in ("gpt", "neox", "opt", "phi")
+    ) or any(keyword in model_lower for keyword in ("gpt", "neox", "opt", "phi")):
+        return ModelProfile(
+            family="gpt2",
+            default_loss="causal",
+            make_tokenizer=_make_gpt2_tokenizer(model_id),
+            default_metric="ppl_causal",
+            default_provider="wikitext2",
+            module_selectors=_gpt2_selectors(),
+            invariants=("causal_masking",),
+            cert_lints=(
+                {
+                    "type": "equals",
+                    "path": "metrics.loss_type",
+                    "value": "causal",
+                    "message": "GPT-style cert should record causal loss.",
+                },
+            ),
+        )
+
+    return ModelProfile(
+        family="unknown",
+        default_loss="causal",
+        make_tokenizer=_make_unknown_tokenizer(model_id),
+        default_metric="ppl_causal",
+        default_provider="wikitext2",
+        module_selectors=_unknown_selectors(),
+        invariants=(),
+        cert_lints=(),
+    )
+
+
+def resolve_tokenizer(profile: ModelProfile) -> tuple[PreTrainedTokenizerBase, str]:
+    """
+    Instantiate a tokenizer for the given profile and return it with its hash.
+    """
+
+    tokenizer, hash_value = profile.make_tokenizer()
+    if not isinstance(hash_value, str) or not hash_value:
+        hash_value = _hash_tokenizer(tokenizer)
+    return tokenizer, hash_value