eigencore 0.4.0__py3-none-any.whl

eigencore/engine/layer_skip.py

@@ -0,0 +1,268 @@
+"""
+Dynamic layer skipping — decides which transformer layers to skip based on
+per-layer sparsity profiles and input complexity.
+
+Transformer models have high layer redundancy: many middle layers produce
+near-identical residual stream updates for routine tokens. Skipping them
+on CPU saves wall-clock time proportional to the skip rate with minimal
+quality loss (typically < 2% perplexity increase at 20-30% skip rate).
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import Enum, auto
+
+import numpy as np
+
+
+class SkipStrategy(Enum):
+    NONE = auto()
+    STATIC = auto()
+    ADAPTIVE = auto()
+    CALIBRATED = auto()
+
+
+@dataclass
+class LayerProfile:
+    index: int
+    importance: float
+    avg_sparsity: float
+    skip_safe: bool
+    samples_seen: int = 0
+
+
+@dataclass
+class SkipPlan:
+    total_layers: int
+    layers_to_run: list[int]
+    layers_to_skip: list[int]
+    strategy: SkipStrategy
+    expected_speedup: float
+    expected_quality_loss: float
+
+    @property
+    def skip_rate(self) -> float:
+        if self.total_layers == 0:
+            return 0.0
+        return len(self.layers_to_skip) / self.total_layers
+
+    def summary(self) -> str:
+        return (
+            f"Skip {len(self.layers_to_skip)}/{self.total_layers} layers "
+            f"({self.skip_rate:.0%}) | ~{self.expected_speedup:.1f}x speedup | "
+            f"~{self.expected_quality_loss:.1%} quality loss"
+        )
+
+
+class LayerSkipScheduler:
+    """
+    Determines which transformer layers to skip during inference.
+
+    Three strategies:
+    1. STATIC — skip fixed layers based on position heuristics (no calibration needed)
+    2. ADAPTIVE — adjust skip decisions per-token based on running sparsity stats
+    3. CALIBRATED — use pre-computed per-layer importance scores from a calibration run
+
+    The key insight: in most transformer models, the first ~15% and last ~15% of
+    layers are critical (embedding projection + output head). The middle layers
+    show high redundancy, especially for simple/repetitive tokens.
+    """
+
+    CRITICAL_HEAD_RATIO = 0.15
+    CRITICAL_TAIL_RATIO = 0.15
+
+    def __init__(
+        self,
+        num_layers: int,
+        strategy: SkipStrategy = SkipStrategy.STATIC,
+        max_skip_rate: float = 0.3,
+        quality_threshold: float = 0.02,
+    ):
+        self.num_layers = num_layers
+        self.strategy = strategy
+        self.max_skip_rate = max_skip_rate
+        self.quality_threshold = quality_threshold
+        self._layer_profiles: list[LayerProfile] = []
+        self._calibrated = False
+
+    def plan(self, complexity: float = 0.5) -> SkipPlan:
+        """
+        Generate a skip plan for the current inference pass.
+
+        complexity: 0.0 (trivial repetition) to 1.0 (novel/complex content).
+        Lower complexity → more aggressive skipping.
+        """
+        if self.num_layers <= 4:
+            return self._no_skip_plan()
+
+        if self.strategy == SkipStrategy.NONE:
+            return self._no_skip_plan()
+        elif self.strategy == SkipStrategy.STATIC:
+            return self._static_plan(complexity)
+        elif self.strategy == SkipStrategy.ADAPTIVE:
+            return self._adaptive_plan(complexity)
+        elif self.strategy == SkipStrategy.CALIBRATED:
+            if not self._calibrated:
+                return self._static_plan(complexity)
+            return self._calibrated_plan(complexity)
+
+        return self._no_skip_plan()
+
+    def calibrate(self, layer_importances: list[float]) -> None:
+        """
+        Load per-layer importance scores from a calibration run.
+        Importance is typically measured as the mean L2 norm of each layer's
+        residual stream contribution over a calibration dataset.
+        """
+        if len(layer_importances) != self.num_layers:
+            raise ValueError(
+                f"Expected {self.num_layers} importance scores, got {len(layer_importances)}"
+            )
+
+        self._layer_profiles = [
+            LayerProfile(
+                index=i,
+                importance=imp,
+                avg_sparsity=0.0,
+                skip_safe=(imp < np.median(layer_importances)),
+            )
+            for i, imp in enumerate(layer_importances)
+        ]
+        self._calibrated = True
+
+    def update_sparsity(self, layer_index: int, sparsity: float) -> None:
+        """Update running sparsity stats for adaptive strategy."""
+        if layer_index >= len(self._layer_profiles):
+            while len(self._layer_profiles) <= layer_index:
+                self._layer_profiles.append(
+                    LayerProfile(
+                        index=len(self._layer_profiles),
+                        importance=1.0,
+                        avg_sparsity=0.0,
+                        skip_safe=False,
+                    )
+                )
+
+        profile = self._layer_profiles[layer_index]
+        n = profile.samples_seen
+        profile.avg_sparsity = (profile.avg_sparsity * n + sparsity) / (n + 1)
+        profile.samples_seen = n + 1
+
+        if profile.avg_sparsity > 0.7 and profile.samples_seen >= 10:
+            profile.skip_safe = True
+
+    def _no_skip_plan(self) -> SkipPlan:
+        return SkipPlan(
+            total_layers=self.num_layers,
+            layers_to_run=list(range(self.num_layers)),
+            layers_to_skip=[],
+            strategy=SkipStrategy.NONE,
+            expected_speedup=1.0,
+            expected_quality_loss=0.0,
+        )
+
+    def _static_plan(self, complexity: float) -> SkipPlan:
+        """Skip middle layers at fixed intervals based on complexity."""
+        head = max(1, int(self.num_layers * self.CRITICAL_HEAD_RATIO))
+        tail = max(1, int(self.num_layers * self.CRITICAL_TAIL_RATIO))
+        tail_start = self.num_layers - tail
+
+        middle_layers = list(range(head, tail_start))
+        if not middle_layers:
+            return self._no_skip_plan()
+
+        adjusted_skip = self.max_skip_rate * (1.0 - complexity)
+        max_skippable = max(1, int(len(middle_layers) * adjusted_skip))
+
+        step = max(1, len(middle_layers) // max_skippable) if max_skippable > 0 else 1
+        skip_set = set(middle_layers[::step][:max_skippable])
+
+        layers_to_run = [i for i in range(self.num_layers) if i not in skip_set]
+        layers_to_skip = sorted(skip_set)
+
+        skip_rate = len(layers_to_skip) / self.num_layers
+        speedup = 1.0 / (1.0 - skip_rate) if skip_rate < 1.0 else 1.0
+        quality_loss = skip_rate * 0.06
+
+        return SkipPlan(
+            total_layers=self.num_layers,
+            layers_to_run=layers_to_run,
+            layers_to_skip=layers_to_skip,
+            strategy=SkipStrategy.STATIC,
+            expected_speedup=speedup,
+            expected_quality_loss=quality_loss,
+        )
+
+    def _adaptive_plan(self, complexity: float) -> SkipPlan:
+        """Skip layers with high observed sparsity."""
+        if not self._layer_profiles:
+            return self._static_plan(complexity)
+
+        head = max(1, int(self.num_layers * self.CRITICAL_HEAD_RATIO))
+        tail = max(1, int(self.num_layers * self.CRITICAL_TAIL_RATIO))
+        tail_start = self.num_layers - tail
+
+        skip_set = set()
+        max_total_skip = int(self.num_layers * self.max_skip_rate * (1.0 - complexity))
+
+        candidates = [
+            p for p in self._layer_profiles if p.skip_safe and head <= p.index < tail_start
+        ]
+        candidates.sort(key=lambda p: p.avg_sparsity, reverse=True)
+
+        for profile in candidates[:max_total_skip]:
+            skip_set.add(profile.index)
+
+        layers_to_run = [i for i in range(self.num_layers) if i not in skip_set]
+        layers_to_skip = sorted(skip_set)
+
+        skip_rate = len(layers_to_skip) / self.num_layers
+        speedup = 1.0 / (1.0 - skip_rate) if skip_rate < 1.0 else 1.0
+        quality_loss = skip_rate * 0.04
+
+        return SkipPlan(
+            total_layers=self.num_layers,
+            layers_to_run=layers_to_run,
+            layers_to_skip=layers_to_skip,
+            strategy=SkipStrategy.ADAPTIVE,
+            expected_speedup=speedup,
+            expected_quality_loss=quality_loss,
+        )
+
+    def _calibrated_plan(self, complexity: float) -> SkipPlan:
+        """Skip layers ranked by calibrated importance scores."""
+        head = max(1, int(self.num_layers * self.CRITICAL_HEAD_RATIO))
+        tail = max(1, int(self.num_layers * self.CRITICAL_TAIL_RATIO))
+        tail_start = self.num_layers - tail
+
+        middle_profiles = [p for p in self._layer_profiles if head <= p.index < tail_start]
+        middle_profiles.sort(key=lambda p: p.importance)
+
+        max_total_skip = int(self.num_layers * self.max_skip_rate * (1.0 - complexity))
+        skip_set = set()
+        cumulative_loss = 0.0
+
+        for profile in middle_profiles:
+            estimated_loss = (1.0 - profile.importance) * 0.01
+            if cumulative_loss + estimated_loss > self.quality_threshold:
+                break
+            if len(skip_set) >= max_total_skip:
+                break
+            skip_set.add(profile.index)
+            cumulative_loss += estimated_loss
+
+        layers_to_run = [i for i in range(self.num_layers) if i not in skip_set]
+        layers_to_skip = sorted(skip_set)
+
+        skip_rate = len(layers_to_skip) / self.num_layers
+        speedup = 1.0 / (1.0 - skip_rate) if skip_rate < 1.0 else 1.0
+
+        return SkipPlan(
+            total_layers=self.num_layers,
+            layers_to_run=layers_to_run,
+            layers_to_skip=layers_to_skip,
+            strategy=SkipStrategy.CALIBRATED,
+            expected_speedup=speedup,
+            expected_quality_loss=cumulative_loss,
+        )

eigencore/engine/prompt_lookup.py

@@ -0,0 +1,39 @@
+"""
+Prompt-lookup decoding (PLD) - Stage C, behind a flag, default OFF.
+
+PLD (prompt lookup / SpecInfer-style): when the model is about to emit a span
+that already appears verbatim in the prompt (common in code, quotes,
+boilerplate), copy it directly from the prompt instead of running the model -
+zero backend work for those tokens.
+
+This module is a pure, backend-independent matcher: given the prompt text and
+the tokens generated so far, find the longest prompt substring continuing the
+generation. The engine uses it only when prompt_lookup=True.
+
+Correctness rule: a copied span must be byte-identical to the prompt span -
+no fuzzy matching, no paraphrase - so output quality is unaffected by
+construction. Speedup is measured, never assumed.
+"""
+
+from __future__ import annotations
+
+
+def longest_prompt_continuation(prompt: str, generated: str, max_span: int = 512) -> str:
+    """Longest prompt text that continues the current generation.
+
+    Finds the longest suffix of `generated` that occurs in `prompt` with at
+    least one more prompt character after it, and returns the prompt text
+    from that point (up to max_span chars). Empty string when nothing matches.
+    """
+    if not prompt or not generated:
+        return ""
+    gen = generated[-max_span:]
+    for size in range(min(len(gen), max_span), 0, -1):
+        suffix = gen[-size:]
+        idx = prompt.find(suffix)
+        while idx != -1:
+            end = idx + size
+            if end < len(prompt):
+                return prompt[end : end + max_span]
+            idx = prompt.find(suffix, idx + 1)
+    return ""

eigencore/engine/pruning.py

@@ -0,0 +1,116 @@
+"""
+Layer-pruning experiment — Stage E slice 2 (ShortGPT-style), REPORTED-ONLY.
+
+ShortGPT (Men et al., 2024) scores each transformer layer by Block Influence:
+the 1 - cosine similarity between the layer's input and output hidden states,
+averaged over calibration samples. Low-BI layers are candidates for removal.
+
+This module computes BI scores, builds a prune plan, and applies the KILL
+CRITERION from the experiment plan: if the measured quality delta (e.g.
+perplexity increase) exceeds `max_quality_delta`, the experiment is aborted
+and NO layers are pruned. Nothing here is wired into inference; the backend
+always runs all layers today.
+
+No performance or quality claim is made without the owner's calibration +
+perplexity runs (FINAL_REPORT §9 item 10).
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+import numpy as np
+
+
+def block_influence(layer_in: np.ndarray, layer_out: np.ndarray, eps: float = 1e-8) -> float:
+    """BI for one layer on one sample: 1 - cos(x_in, x_out).
+
+    `layer_in`/`layer_out`: (hidden,) vectors. Returns 0 when either is ~zero.
+    """
+    x = np.asarray(layer_in, dtype=np.float64).reshape(-1)
+    y = np.asarray(layer_out, dtype=np.float64).reshape(-1)
+    if x.shape != y.shape or x.size == 0:
+        raise ValueError("layer_in/layer_out must be same-shape non-empty vectors")
+    nx = float(np.linalg.norm(x))
+    ny = float(np.linalg.norm(y))
+    if nx < eps or ny < eps:
+        return 0.0
+    cos = float(np.dot(x, y) / (nx * ny))
+    return 1.0 - max(-1.0, min(1.0, cos))
+
+
+def layer_bi_scores(
+    hidden_states: np.ndarray,
+) -> np.ndarray:
+    """Mean BI per layer over samples.
+
+    `hidden_states`: (num_layers + 1, num_samples, hidden) — activations at each
+    layer boundary (0 = model input, L = model output). Layer i's BI uses
+    boundary i (input) and i+1 (output). Returns (num_layers,) float64.
+    """
+    hs = np.asarray(hidden_states, dtype=np.float64)
+    if hs.ndim != 3 or hs.shape[0] < 2:
+        raise ValueError("expected (layers+1, samples, hidden)")
+    num_layers = hs.shape[0] - 1
+    scores = np.zeros(num_layers)
+    for i in range(num_layers):
+        x = hs[i]  # (samples, hidden)
+        y = hs[i + 1]
+        nx = np.linalg.norm(x, axis=1)
+        ny = np.linalg.norm(y, axis=1)
+        denom = nx * ny
+        cos = np.where(denom > 1e-16, (x * y).sum(axis=1) / np.maximum(denom, 1e-16), 0.0)
+        cos = np.clip(cos, -1.0, 1.0)
+        scores[i] = float(np.mean(1.0 - cos))
+    return scores
+
+
+@dataclass
+class PrunePlan:
+    layers_to_prune: list[int]
+    bi_scores: np.ndarray
+    max_quality_delta_allowed: float
+    killed: bool = False
+    kill_reason: str = ""
+    notes: list[str] = field(default_factory=list)
+
+
+def build_prune_plan(
+    bi_scores: np.ndarray,
+    num_layers: int,
+    max_prune_fraction: float = 0.3,
+    protect_head: float = 0.15,
+    protect_tail: float = 0.15,
+    max_quality_delta_allowed: float = 0.02,
+    measured_quality_delta: float | None = None,
+) -> PrunePlan:
+    """Rank layers by BI ascending and pick the lowest-BI middle layers.
+
+    First/last `protect_*` fraction of layers are never pruned (embedding/head
+    criticality). Kill criterion: if `measured_quality_delta` is provided and
+    exceeds `max_quality_delta_allowed`, the plan is killed (no pruning).
+    """
+    scores = np.asarray(bi_scores, dtype=np.float64)
+    if scores.shape[0] != num_layers:
+        raise ValueError("bi_scores length must equal num_layers")
+    if measured_quality_delta is not None and measured_quality_delta > max_quality_delta_allowed:
+        return PrunePlan(
+            layers_to_prune=[],
+            bi_scores=scores,
+            max_quality_delta_allowed=max_quality_delta_allowed,
+            killed=True,
+            kill_reason=(
+                f"quality delta {measured_quality_delta:.4f} > allowed "
+                f"{max_quality_delta_allowed:.4f} — kill criterion hit, no pruning"
+            ),
+        )
+    n_protect_head = int(num_layers * protect_head)
+    n_protect_tail = int(num_layers * protect_tail)
+    candidates = list(range(n_protect_head, num_layers - n_protect_tail))
+    max_prune = int(num_layers * max_prune_fraction)
+    candidates.sort(key=lambda i: scores[i])
+    return PrunePlan(
+        layers_to_prune=sorted(candidates[:max_prune]),
+        bi_scores=scores,
+        max_quality_delta_allowed=max_quality_delta_allowed,
+    )

eigencore/engine/sparse_inference.py

@@ -0,0 +1,218 @@
+"""
+Sparsity-aware inference — exploits the 70-90% activation sparsity in dense
+transformer models to skip near-zero neuron computations on CPU.
+
+GPU warp divergence makes sparse execution inefficient on GPUs (all 32 threads
+in a warp must execute the same instruction). CPUs have fine-grained branch
+prediction — skipping a zero multiply is a correctly-predicted branch that
+costs nearly nothing. This is the core CPU advantage EigenCore exploits.
+
+This module provides:
+1. SparsityPredictor: learns per-layer sparsity patterns from observation
+2. SparseExecutionPlan: decides which neurons to skip for a given input
+3. SparsityCache: caches sparsity masks across similar inputs for reuse
+"""
+
+from __future__ import annotations
+
+import hashlib
+from collections import OrderedDict
+from dataclasses import dataclass
+
+import numpy as np
+
+
+@dataclass
+class NeuronMask:
+    layer_index: int
+    total_neurons: int
+    active_indices: np.ndarray
+    sparsity: float
+
+    @property
+    def active_count(self) -> int:
+        return len(self.active_indices)
+
+    @property
+    def skip_count(self) -> int:
+        return self.total_neurons - self.active_count
+
+
+@dataclass
+class SparseExecutionPlan:
+    masks: list[NeuronMask]
+    total_neurons: int
+    total_active: int
+    overall_sparsity: float
+    estimated_flop_savings: float
+    estimated_speedup: float
+
+    def summary(self) -> str:
+        return (
+            f"Sparse plan: {self.overall_sparsity:.0%} sparsity across "
+            f"{len(self.masks)} layers | {self.estimated_speedup:.1f}x speedup | "
+            f"{self.estimated_flop_savings:.0%} FLOPs saved"
+        )
+
+
+class SparsityPredictor:
+    """
+    Learns and predicts per-layer activation sparsity patterns.
+
+    Tracks running statistics of which neurons activate (|value| > threshold)
+    across multiple inputs. After enough observations, it can predict which
+    neurons are likely inactive for a new input, enabling preemptive skipping.
+    """
+
+    def __init__(
+        self,
+        num_layers: int,
+        neurons_per_layer: int,
+        threshold: float = 0.01,
+        warmup_samples: int = 16,
+    ):
+        self.num_layers = num_layers
+        self.neurons_per_layer = neurons_per_layer
+        self.threshold = threshold
+        self.warmup_samples = warmup_samples
+        self._activation_counts = np.zeros((num_layers, neurons_per_layer), dtype=np.int32)
+        self._total_samples = 0
+        self._layer_sparsity_history: list[list[float]] = [[] for _ in range(num_layers)]
+
+    @property
+    def is_warmed_up(self) -> bool:
+        return self._total_samples >= self.warmup_samples
+
+    def observe(self, layer_index: int, activations: np.ndarray) -> float:
+        """
+        Record which neurons activated for this input.
+        Returns the observed sparsity for this layer.
+        """
+        if layer_index >= self.num_layers:
+            raise ValueError(f"Layer {layer_index} >= num_layers {self.num_layers}")
+
+        abs_acts = np.abs(activations)
+        active_mask = abs_acts >= self.threshold
+        active_count = int(np.sum(active_mask))
+        sparsity = 1.0 - (active_count / len(activations)) if len(activations) > 0 else 0.0
+
+        n = min(len(activations), self.neurons_per_layer)
+        self._activation_counts[layer_index, :n] += active_mask[:n].astype(np.int32)
+        self._total_samples += 1
+        self._layer_sparsity_history[layer_index].append(sparsity)
+
+        return sparsity
+
+    def predict_mask(self, layer_index: int, aggressiveness: float = 0.5) -> NeuronMask:
+        """
+        Predict which neurons will be active for the next input.
+
+        aggressiveness: 0.0 (conservative, skip only always-zero neurons)
+                       to 1.0 (aggressive, skip anything below median activation rate)
+        """
+        if not self.is_warmed_up:
+            return NeuronMask(
+                layer_index=layer_index,
+                total_neurons=self.neurons_per_layer,
+                active_indices=np.arange(self.neurons_per_layer),
+                sparsity=0.0,
+            )
+
+        activation_rates = self._activation_counts[layer_index] / self._total_samples
+        cutoff = aggressiveness * np.median(activation_rates)
+        active_indices = np.where(activation_rates > cutoff)[0]
+
+        sparsity = 1.0 - (len(active_indices) / self.neurons_per_layer)
+
+        return NeuronMask(
+            layer_index=layer_index,
+            total_neurons=self.neurons_per_layer,
+            active_indices=active_indices,
+            sparsity=sparsity,
+        )
+
+    def create_execution_plan(self, aggressiveness: float = 0.5) -> SparseExecutionPlan:
+        """Generate a full execution plan across all layers."""
+        masks = []
+        total_neurons = 0
+        total_active = 0
+
+        for layer_idx in range(self.num_layers):
+            mask = self.predict_mask(layer_idx, aggressiveness)
+            masks.append(mask)
+            total_neurons += mask.total_neurons
+            total_active += mask.active_count
+
+        overall_sparsity = 1.0 - (total_active / total_neurons) if total_neurons > 0 else 0.0
+        flop_savings = overall_sparsity
+        speedup = 1.0 / (1.0 - overall_sparsity) if overall_sparsity < 1.0 else 1.0
+        speedup = min(speedup, 5.0)
+
+        return SparseExecutionPlan(
+            masks=masks,
+            total_neurons=total_neurons,
+            total_active=total_active,
+            overall_sparsity=overall_sparsity,
+            estimated_flop_savings=flop_savings,
+            estimated_speedup=speedup,
+        )
+
+    def layer_stats(self) -> list[dict]:
+        """Per-layer statistics summary."""
+        stats = []
+        for i in range(self.num_layers):
+            history = self._layer_sparsity_history[i]
+            stats.append(
+                {
+                    "layer": i,
+                    "avg_sparsity": float(np.mean(history)) if history else 0.0,
+                    "std_sparsity": float(np.std(history)) if history else 0.0,
+                    "samples": len(history),
+                }
+            )
+        return stats
+
+
+class SparsityCache:
+    """
+    Caches sparsity masks for similar inputs to avoid recomputation.
+
+    Uses a hash of the input token sequence to look up previously computed
+    masks. LRU eviction keeps memory bounded.
+    """
+
+    def __init__(self, max_entries: int = 256):
+        self.max_entries = max_entries
+        self._cache: OrderedDict[str, SparseExecutionPlan] = OrderedDict()
+        self.hits = 0
+        self.misses = 0
+
+    def _hash_input(self, token_ids: list[int]) -> str:
+        key = ",".join(str(t) for t in token_ids[-32:])
+        return hashlib.md5(key.encode()).hexdigest()
+
+    def get(self, token_ids: list[int]) -> SparseExecutionPlan | None:
+        key = self._hash_input(token_ids)
+        if key in self._cache:
+            self.hits += 1
+            self._cache.move_to_end(key)
+            return self._cache[key]
+        self.misses += 1
+        return None
+
+    def put(self, token_ids: list[int], plan: SparseExecutionPlan) -> None:
+        key = self._hash_input(token_ids)
+        self._cache[key] = plan
+        self._cache.move_to_end(key)
+        while len(self._cache) > self.max_entries:
+            self._cache.popitem(last=False)
+
+    @property
+    def hit_rate(self) -> float:
+        total = self.hits + self.misses
+        return self.hits / total if total > 0 else 0.0
+
+    def clear(self) -> None:
+        self._cache.clear()
+        self.hits = 0
+        self.misses = 0