eigencore 0.4.0__py3-none-any.whl

eigencore/context/__init__.py

@@ -0,0 +1,3 @@
+from eigencore.context.manager import ContextManager, ContextWindow
+
+__all__ = ["ContextManager", "ContextWindow"]

eigencore/context/manager.py

@@ -0,0 +1,318 @@
+"""
+Attention-aware context manager — compresses conversation context while
+preserving the tokens that matter most for generation quality.
+
+Key insight (Gap 2): The first 3-4 tokens in any sequence act as "attention
+sinks" — they absorb disproportionate attention weight regardless of content.
+Deleting them destabilizes generation. Every other tool just truncates from
+the beginning. This manager preserves sink tokens, compresses middle context,
+and keeps recent context intact.
+
+Result: stable generation quality at ~50% the context length, letting a 16GB
+RAM machine maintain conversations that would normally need 32GB of context.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import sqlite3
+import time
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Optional
+
+
+@dataclass
+class Message:
+    role: str  # "system", "user", "assistant"
+    content: str
+    timestamp: float = field(default_factory=time.time)
+    token_estimate: int = 0
+    compressed: bool = False
+    message_id: str = ""
+
+    def __post_init__(self):
+        if not self.token_estimate:
+            self.token_estimate = self._estimate_tokens(self.content)
+        if not self.message_id:
+            h = hashlib.md5(f"{self.timestamp}{self.content[:50]}".encode()).hexdigest()[:12]
+            self.message_id = h
+
+    @staticmethod
+    def _estimate_tokens(text: str) -> int:
+        """Rough token estimate: ~4 chars per token for English."""
+        return max(len(text) // 4, 1)
+
+
+@dataclass
+class ContextWindow:
+    """Represents the current context state with compression metadata."""
+
+    messages: list[Message]
+    total_tokens: int
+    max_tokens: int
+    compression_ratio: float
+    sink_tokens_preserved: int
+    messages_compressed: int
+
+
+class ContextManager:
+    """
+    Manages conversation context with attention-aware compression.
+
+    Compression strategy (from most to least aggressive):
+    1. PRESERVE: Always keep the first system message + first user message
+       (these occupy the "attention sink" positions 0-4 in the token stream)
+    2. PRESERVE: Always keep the most recent N messages (active working memory)
+    3. COMPRESS: Middle messages get summarized into shorter versions
+    4. DROP: If still over budget after compression, drop oldest middle messages
+
+    Also provides persistent session storage via SQLite.
+    """
+
+    SINK_MESSAGES = 2  # first system + first user message to always preserve
+    RECENT_MESSAGES = 6  # recent messages to always keep uncompressed
+    COMPRESSION_TARGET = 0.4  # compress middle messages to 40% of original size
+
+    def __init__(
+        self,
+        max_context_tokens: int = 2048,
+        db_path: Optional[Path] = None,
+    ):
+        self.max_context_tokens = max_context_tokens
+        self.messages: list[Message] = []
+        self._db: Optional[sqlite3.Connection] = None
+        if db_path:
+            self._init_db(db_path)
+
+    def _init_db(self, db_path: Path) -> None:
+        db_path.parent.mkdir(parents=True, exist_ok=True)
+        self._db = sqlite3.connect(str(db_path))
+        self._db.execute("""
+            CREATE TABLE IF NOT EXISTS messages (
+                id TEXT PRIMARY KEY,
+                session_id TEXT NOT NULL,
+                role TEXT NOT NULL,
+                content TEXT NOT NULL,
+                timestamp REAL NOT NULL,
+                token_estimate INTEGER NOT NULL,
+                compressed INTEGER DEFAULT 0
+            )
+        """)
+        self._db.execute("""
+            CREATE INDEX IF NOT EXISTS idx_session
+            ON messages(session_id, timestamp)
+        """)
+        self._db.commit()
+
+    def add(self, role: str, content: str) -> Message:
+        msg = Message(role=role, content=content)
+        self.messages.append(msg)
+        return msg
+
+    def get_context(self) -> ContextWindow:
+        """
+        Build the context window with attention-aware compression.
+        Returns messages ready to send to the model, respecting token budget.
+        """
+        if not self.messages:
+            return ContextWindow(
+                messages=[],
+                total_tokens=0,
+                max_tokens=self.max_context_tokens,
+                compression_ratio=1.0,
+                sink_tokens_preserved=0,
+                messages_compressed=0,
+            )
+
+        total = sum(m.token_estimate for m in self.messages)
+
+        # if everything fits, no compression needed
+        if total <= self.max_context_tokens:
+            return ContextWindow(
+                messages=list(self.messages),
+                total_tokens=total,
+                max_tokens=self.max_context_tokens,
+                compression_ratio=1.0,
+                sink_tokens_preserved=0,
+                messages_compressed=0,
+            )
+
+        # split into three zones
+        sink = self.messages[: self.SINK_MESSAGES]
+        n_recent = min(self.RECENT_MESSAGES, len(self.messages) - self.SINK_MESSAGES)
+        recent = self.messages[-n_recent:] if n_recent > 0 else []
+        middle_end = len(self.messages) - n_recent if n_recent > 0 else len(self.messages)
+        middle = self.messages[self.SINK_MESSAGES : middle_end]
+
+        sink_tokens = sum(m.token_estimate for m in sink)
+        recent_tokens = sum(m.token_estimate for m in recent)
+
+        # if sink + recent alone exceed budget, trim recent count
+        while sink_tokens + recent_tokens > self.max_context_tokens and len(recent) > 1:
+            dropped = recent.pop(0)
+            recent_tokens -= dropped.token_estimate
+
+        budget_for_middle = max(self.max_context_tokens - sink_tokens - recent_tokens, 0)
+
+        compressed_middle, middle_compressed_count = self._compress_middle(
+            middle, budget_for_middle
+        )
+
+        result = sink + compressed_middle + recent
+        result_tokens = sum(m.token_estimate for m in result)
+
+        # final safety: if still over, drop middle entirely
+        if result_tokens > self.max_context_tokens and compressed_middle:
+            result = sink + recent
+            result_tokens = sum(m.token_estimate for m in result)
+            middle_compressed_count = len(middle)
+
+        return ContextWindow(
+            messages=result,
+            total_tokens=result_tokens,
+            max_tokens=self.max_context_tokens,
+            compression_ratio=result_tokens / total if total > 0 else 1.0,
+            sink_tokens_preserved=sink_tokens,
+            messages_compressed=middle_compressed_count,
+        )
+
+    def _compress_middle(
+        self, messages: list[Message], token_budget: int
+    ) -> tuple[list[Message], int]:
+        """
+        Compress middle messages to fit within token budget.
+        Strategy: truncate each message proportionally, keeping the first
+        and last sentences (they tend to carry the most information).
+        """
+        if not messages:
+            return [], 0
+
+        if token_budget <= 0:
+            # no room for middle context — create a single summary placeholder
+            combined = " | ".join(f"[{m.role}]: {m.content[:30]}..." for m in messages[:5])
+            summary = Message(
+                role="system",
+                content=f"[Earlier context compressed: {len(messages)} messages] {combined}",
+                compressed=True,
+            )
+            return [summary], len(messages)
+
+        total_middle_tokens = sum(m.token_estimate for m in messages)
+
+        if total_middle_tokens <= token_budget:
+            return list(messages), 0
+
+        # compress each message proportionally
+        ratio = token_budget / total_middle_tokens
+        compressed = []
+        count = 0
+
+        for msg in messages:
+            target_chars = int(len(msg.content) * ratio)
+            if target_chars < 50:
+                # too small to be useful — skip but note it
+                continue
+
+            truncated = self._smart_truncate(msg.content, target_chars)
+            compressed.append(
+                Message(
+                    role=msg.role,
+                    content=truncated,
+                    timestamp=msg.timestamp,
+                    compressed=True,
+                )
+            )
+            count += 1
+
+        # if we still exceed budget after proportional compression, drop oldest
+        while sum(m.token_estimate for m in compressed) > token_budget and len(compressed) > 1:
+            compressed.pop(0)
+            count += 1
+
+        return compressed, count
+
+    @staticmethod
+    def _smart_truncate(text: str, target_chars: int) -> str:
+        """
+        Truncate text keeping the first and last sentences.
+        These carry the most semantic weight (topic introduction + conclusion).
+        """
+        if len(text) <= target_chars:
+            return text
+
+        sentences = text.replace("! ", ".\n").replace("? ", ".\n").split(". ")
+        if len(sentences) <= 2:
+            return text[:target_chars] + "..."
+
+        half = target_chars // 2
+        first_part = text[:half]
+        last_part = text[-half:]
+
+        # find clean sentence boundary in first part
+        last_period = first_part.rfind(". ")
+        if last_period > half // 2:
+            first_part = first_part[: last_period + 1]
+
+        # find clean sentence boundary in last part
+        first_period = last_part.find(". ")
+        if first_period > 0 and first_period < len(last_part) // 2:
+            last_part = last_part[first_period + 2 :]
+
+        return f"{first_part} [...] {last_part}"
+
+    def save_session(self, session_id: str) -> None:
+        """Persist current messages to SQLite."""
+        if not self._db:
+            return
+
+        for msg in self.messages:
+            self._db.execute(
+                "INSERT OR REPLACE INTO messages VALUES (?, ?, ?, ?, ?, ?, ?)",
+                (
+                    msg.message_id,
+                    session_id,
+                    msg.role,
+                    msg.content,
+                    msg.timestamp,
+                    msg.token_estimate,
+                    int(msg.compressed),
+                ),
+            )
+        self._db.commit()
+
+    def load_session(self, session_id: str) -> int:
+        """Load messages from a previous session. Returns count loaded."""
+        if not self._db:
+            return 0
+
+        cursor = self._db.execute(
+            "SELECT role, content, timestamp, token_estimate, compressed, id "
+            "FROM messages WHERE session_id = ? ORDER BY timestamp",
+            (session_id,),
+        )
+        self.messages.clear()
+        count = 0
+        for row in cursor:
+            msg = Message(
+                role=row[0],
+                content=row[1],
+                timestamp=row[2],
+                token_estimate=row[3],
+                compressed=bool(row[4]),
+                message_id=row[5],
+            )
+            self.messages.append(msg)
+            count += 1
+        return count
+
+    def clear(self) -> None:
+        self.messages.clear()
+
+    @property
+    def token_count(self) -> int:
+        return sum(m.token_estimate for m in self.messages)
+
+    @property
+    def message_count(self) -> int:
+        return len(self.messages)

eigencore/engine/__init__.py

@@ -0,0 +1,3 @@
+from eigencore.engine.inference import InferenceEngine
+
+__all__ = ["InferenceEngine"]

eigencore/engine/inference.py

@@ -0,0 +1,378 @@
+"""
+Inference engine — wraps llama.cpp with hardware-aware configuration.
+Handles model loading, generation, and resource management.
+
+Phase 2 integration: layer skipping, sparsity analysis, and speculative
+decoding hooks are initialized on model load and report stats via
+the `optimization_stats` property.
+"""
+
+from __future__ import annotations
+
+import time
+from collections.abc import Iterator
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from eigencore.engine.layer_skip import LayerSkipScheduler, SkipStrategy
+from eigencore.engine.prompt_lookup import longest_prompt_continuation
+from eigencore.engine.sparse_inference import SparsityCache, SparsityPredictor
+from eigencore.engine.speculative import SpeculativeDecoder
+from eigencore.hal.profiler import HardwareProfile, InstructionSet
+from eigencore.models.registry import ModelSpec
+
+# llama-cpp-python expects an int KV type; map our string names.
+_KV_TYPE_MAP = {"fp16": 0, "q8_0": 1}
+
+
+@dataclass
+class GenerationConfig:
+    max_tokens: int = 512
+    temperature: float = 0.7
+    top_p: float = 0.9
+    top_k: int = 40
+    repeat_penalty: float = 1.1
+    stop: list[str] = field(default_factory=list)
+    stream: bool = True
+    enable_layer_skip: bool = True
+    enable_sparse_inference: bool = True
+    complexity: float = 0.5
+    # Prompt-lookup decoding: copy prompt-verbatim spans instead of decoding
+    # them. Default OFF; enable only after owner benchmarks (FINAL_REPORT §9 item 9).
+    prompt_lookup: bool = False
+
+
+@dataclass
+class GenerationResult:
+    text: str
+    tokens_generated: int
+    time_seconds: float
+    tokens_per_second: float
+    prompt_tokens: int
+    optimization_stats: dict | None = None
+    # Optimizations actually applied to the backend call. Empty today:
+    # layer-skip/sparsity plans and the speculative decoder are computed and
+    # reported but NOT wired into create_chat_completion yet.
+    applied_optimizations: list[str] = field(default_factory=list)
+
+
+class InferenceEngine:
+    def __init__(
+        self,
+        model_path: Path,
+        profile: HardwareProfile,
+        model_spec: ModelSpec | None = None,
+        kv_type: str = "fp16",
+        flash_attn: bool = True,
+        n_gpu_layers: int = 0,
+    ):
+        self.model_path = model_path
+        self.profile = profile
+        self.model_spec = model_spec
+        # Load-time backend options: KV cache type ("fp16" default; "q8_0"
+        # halves KV memory at a small quality cost) and flash attention.
+        self.kv_type = kv_type
+        self.flash_attn = flash_attn
+        # 0 = CPU-only (default, safe). Set from the fit estimator's
+        # gpu_layers_for_vram when a VRAM budget exists.
+        self.n_gpu_layers = n_gpu_layers
+        self._llm = None
+
+        self._layer_skipper: LayerSkipScheduler | None = None
+        self._sparsity_predictor: SparsityPredictor | None = None
+        self._sparsity_cache = SparsityCache()
+        self._speculative_decoder = SpeculativeDecoder(adaptive=True)
+        self.last_pld_copied_chars = 0
+
+    def load(self) -> None:
+        """Load the model with hardware-optimized settings."""
+        try:
+            from llama_cpp import Llama
+        except ImportError:
+            raise ImportError(
+                "llama-cpp-python is required for inference. "
+                "Install with: pip install llama-cpp-python"
+            )
+
+        ctx_length = self.profile.recommended_context_length
+        if self.model_spec and self.model_spec.context_length < ctx_length:
+            ctx_length = self.model_spec.context_length
+
+        n_threads = self.profile.recommended_threads
+        n_batch = self._compute_batch_size()
+
+        self._llm = Llama(
+            model_path=str(self.model_path),
+            n_ctx=ctx_length,
+            n_threads=n_threads,
+            n_threads_batch=n_threads,
+            n_batch=n_batch,
+            n_gpu_layers=self.n_gpu_layers,
+            type_kv=_KV_TYPE_MAP[self.kv_type],
+            flash_attn=self.flash_attn,
+            verbose=False,
+        )
+
+        self._init_optimizers()
+
+    def _init_optimizers(self) -> None:
+        """Initialize Phase 2 optimization modules based on loaded model.
+
+        Metadata keys are architecture-prefixed (`{arch}.block_count`, ...),
+        not always `llama.*` — resolve via the shared GGUF helper, falling
+        back to raw metadata with an architecture prefix.
+        """
+        from eigencore.fit.gguf_meta import read_gguf_model_info
+
+        info = read_gguf_model_info(self.model_path)
+        if info is not None:
+            num_layers = info.geometry.num_layers or 0
+            hidden_size = info.geometry.hidden_size or 0
+        else:
+            metadata = dict(self._llm.metadata) if self._llm else {}
+            arch = metadata.get("general.architecture", "llama")
+            num_layers = int(
+                metadata.get(f"{arch}.block_count", metadata.get("llama.block_count", 0))
+            )
+            hidden_size = int(
+                metadata.get(f"{arch}.embedding_length", metadata.get("llama.embedding_length", 0))
+            )
+
+        if num_layers > 0:
+            self._layer_skipper = LayerSkipScheduler(
+                num_layers=num_layers,
+                strategy=SkipStrategy.STATIC,
+                max_skip_rate=0.3,
+            )
+
+        if hidden_size > 0 and num_layers > 0:
+            self._sparsity_predictor = SparsityPredictor(
+                num_layers=num_layers,
+                neurons_per_layer=hidden_size,
+                warmup_samples=16,
+            )
+
+    def _compute_batch_size(self) -> int:
+        """Determine optimal batch size based on available RAM and ISA."""
+        if InstructionSet.AVX512F in self.profile.instruction_sets:
+            return 1024
+        elif InstructionSet.AVX2 in self.profile.instruction_sets:
+            return 512
+        else:
+            return 256
+
+    def generate(
+        self,
+        prompt: str,
+        config: GenerationConfig | None = None,
+    ) -> GenerationResult:
+        """Generate a complete response (non-streaming)."""
+        if self._llm is None:
+            raise RuntimeError("Model not loaded. Call .load() first.")
+
+        if config is None:
+            config = GenerationConfig(stream=False)
+
+        messages = [{"role": "user", "content": prompt}]
+        return self.chat(messages, config)
+
+    def stream(
+        self,
+        prompt: str,
+        config: GenerationConfig | None = None,
+    ) -> Iterator[str]:
+        """Stream tokens as they're generated."""
+        if self._llm is None:
+            raise RuntimeError("Model not loaded. Call .load() first.")
+
+        if config is None:
+            config = GenerationConfig(stream=True)
+
+        messages = [{"role": "user", "content": prompt}]
+
+        for chunk in self._llm.create_chat_completion(
+            messages=messages,
+            max_tokens=config.max_tokens,
+            temperature=config.temperature,
+            top_p=config.top_p,
+            top_k=config.top_k,
+            repeat_penalty=config.repeat_penalty,
+            stop=config.stop or None,
+            stream=True,
+        ):
+            delta = chunk["choices"][0].get("delta", {})
+            token = delta.get("content", "")
+            if token:
+                yield token
+
+    def stream_messages(
+        self,
+        messages: list[dict[str, str]],
+        config: GenerationConfig | None = None,
+    ) -> Iterator[str]:
+        """Stream a chat completion from a full message list (multi-turn).
+
+        When config.prompt_lookup is set, prompt-verbatim continuations are
+        emitted directly (byte-identical copy, see engine.prompt_lookup);
+        copied characters are reported via `last_pld_copied_chars`.
+        """
+        if self._llm is None:
+            raise RuntimeError("Model not loaded. Call .load() first.")
+        if config is None:
+            config = GenerationConfig(stream=True)
+        prompt_text = " ".join(m.get("content", "") for m in messages)
+        generated = ""
+        self.last_pld_copied_chars = 0
+        pending_skip = 0  # backend chars to swallow (already copied from prompt)
+        for chunk in self._llm.create_chat_completion(
+            messages=messages,
+            max_tokens=config.max_tokens,
+            temperature=config.temperature,
+            top_p=config.top_p,
+            top_k=config.top_k,
+            repeat_penalty=config.repeat_penalty,
+            stop=config.stop or None,
+            stream=True,
+        ):
+            delta = chunk["choices"][0].get("delta", {})
+            token = delta.get("content", "")
+            if token:
+                if pending_skip > 0:
+                    # this backend token is part of a span we already copied
+                    pending_skip -= len(token)
+                    continue
+                if config.prompt_lookup:
+                    continuation = longest_prompt_continuation(prompt_text, generated + token)
+                    if continuation:
+                        # The prompt continues the generation verbatim: emit the
+                        # copied span after this token and swallow its backend copy.
+                        pending_skip = len(continuation)
+                        self.last_pld_copied_chars += len(continuation)
+                        generated += token + continuation
+                        yield token
+                        yield continuation
+                        continue
+                generated += token
+                yield token
+
+    def chat(
+        self,
+        messages: list[dict[str, str]],
+        config: GenerationConfig | None = None,
+    ) -> GenerationResult:
+        """Chat-style completion with message list."""
+        if self._llm is None:
+            raise RuntimeError("Model not loaded. Call .load() first.")
+
+        if config is None:
+            config = GenerationConfig(stream=False)
+
+        start = time.perf_counter()
+
+        response = self._llm.create_chat_completion(
+            messages=messages,
+            max_tokens=config.max_tokens,
+            temperature=config.temperature,
+            top_p=config.top_p,
+            top_k=config.top_k,
+            repeat_penalty=config.repeat_penalty,
+            stop=config.stop or None,
+        )
+
+        elapsed = time.perf_counter() - start
+        text = response["choices"][0]["message"]["content"]
+        usage = response.get("usage", {})
+        prompt_tokens = usage.get("prompt_tokens", 0)
+        completion_tokens = usage.get("completion_tokens", 0)
+        tps = completion_tokens / elapsed if elapsed > 0 else 0.0
+
+        return GenerationResult(
+            text=text,
+            tokens_generated=completion_tokens,
+            time_seconds=elapsed,
+            tokens_per_second=tps,
+            prompt_tokens=prompt_tokens,
+            optimization_stats=self._collect_stats(config),
+            applied_optimizations=self._applied_optimizations(),
+        )
+
+    def _applied_optimizations(self) -> list[str]:
+        """Optimizations actually wired into the backend call.
+
+        Currently none: layer-skip/sparsity only produce reported plans and
+        the speculative decoder is never invoked during generation. Kept
+        explicit so callers can distinguish measured from planned work.
+        """
+        return []
+
+    def _collect_stats(self, config: GenerationConfig) -> dict:
+        """Collect optimization statistics from Phase 2 modules."""
+        stats: dict = {}
+        stats["backend_wired"] = self._applied_optimizations()
+        stats["note"] = (
+            "stats are planned/reported only; no optimization is applied to the backend call yet"
+        )
+
+        if self._layer_skipper and config.enable_layer_skip:
+            plan = self._layer_skipper.plan(complexity=config.complexity)
+            stats["layer_skip"] = {
+                "applied": False,
+                "strategy": plan.strategy.name,
+                "layers_skipped": len(plan.layers_to_skip),
+                "total_layers": plan.total_layers,
+                "skip_rate": plan.skip_rate,
+                "expected_speedup": plan.expected_speedup,
+            }
+
+        if self._sparsity_predictor and config.enable_sparse_inference:
+            if self._sparsity_predictor.is_warmed_up:
+                exec_plan = self._sparsity_predictor.create_execution_plan()
+                stats["sparse_inference"] = {
+                    "applied": False,
+                    "overall_sparsity": exec_plan.overall_sparsity,
+                    "estimated_speedup": exec_plan.estimated_speedup,
+                    "flop_savings": exec_plan.estimated_flop_savings,
+                }
+
+        stats["sparsity_cache"] = {
+            "hit_rate": self._sparsity_cache.hit_rate,
+            "hits": self._sparsity_cache.hits,
+            "misses": self._sparsity_cache.misses,
+        }
+
+        stats["speculative"] = {
+            "applied": False,
+            "draft_length": self._speculative_decoder.draft_length,
+            "overall_accept_rate": self._speculative_decoder.stats.overall_accept_rate,
+            "total_rounds": self._speculative_decoder.stats.total_rounds,
+        }
+
+        return stats
+
+    def unload(self) -> None:
+        """Release model from memory."""
+        if self._llm is not None:
+            del self._llm
+            self._llm = None
+
+    @property
+    def is_loaded(self) -> bool:
+        return self._llm is not None
+
+    @property
+    def optimization_stats(self) -> dict:
+        """Summary of all Phase 2 optimization module states."""
+        config = GenerationConfig()
+        return self._collect_stats(config) if self._llm else {}
+
+    @property
+    def layer_skipper(self) -> LayerSkipScheduler | None:
+        return self._layer_skipper
+
+    @property
+    def sparsity_predictor(self) -> SparsityPredictor | None:
+        return self._sparsity_predictor
+
+    @property
+    def speculative_decoder(self) -> SpeculativeDecoder:
+        return self._speculative_decoder