sharp-context 0.1.0__py3-none-any.whl

sharp_context/server.py

@@ -0,0 +1,624 @@
+"""
+SharpContext MCP Server
+========================
+
+The main MCP server that exposes all five mathematical engines
+as callable tools for any MCP-compatible AI coding tool.
+
+Architecture mirrors the Ebbiforge Rust core's ContextScorer pipeline:
+  1. Fragments ingested → SimHash fingerprinted → dedup checked
+  2. Shannon entropy scored → information density computed
+  3. Ebbinghaus decay applied per turn → staleness measured
+  4. Knapsack DP selects optimal subset within token budget
+  5. Predictive pre-fetcher pre-loads likely-needed context
+  6. Auto-checkpoint every N tool calls for crash recovery
+
+Supported clients:
+  - Cursor (add to .cursor/mcp.json)
+  - Claude Code (claude mcp add)
+  - Cline (add to mcp settings)
+  - Any MCP-compatible client
+
+Run:
+    sharp-context        # Start as STDIO server
+    python -m sharp_context.server   # Alternative
+
+References:
+  - MCP Protocol: https://modelcontextprotocol.io
+  - Ebbiforge ContextScorer (lsh.rs) — multi-dimensional weighted scoring
+  - Ebbiforge HippocampusEngine — Ebbinghaus decay + spaced repetition
+"""
+
+from __future__ import annotations
+
+import hashlib
+import sys
+import json
+import logging
+from collections import Counter
+from typing import Any, Dict, List, Optional
+
+from .config import SharpContextConfig
+from .knapsack import (
+    ContextFragment,
+    apply_ebbinghaus_decay,
+    compute_relevance,
+    knapsack_optimize,
+)
+from .entropy import compute_information_score
+from .dedup import DedupIndex, simhash
+from .prefetch import PrefetchEngine
+from .checkpoint import CheckpointManager
+
+# Configure logging to stderr (MCP requires stdout for JSON-RPC)
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s [sharp-context] %(message)s",
+    stream=sys.stderr,
+)
+logger = logging.getLogger("sharp-context")
+
+
+class SharpContextEngine:
+    """
+    The core engine that orchestrates all five subsystems.
+
+    This is the brains of SharpContext — it manages the fragment store,
+    applies Ebbinghaus decay each turn, scores fragments using Shannon
+    entropy, deduplicates via SimHash, and selects the optimal context
+    subset using knapsack DP.
+
+    Modeled after Ebbiforge's HippocampusEngine which uses the same
+    pipeline: SimHash → LSH index → ContextScorer → ranked results.
+    """
+
+    def __init__(self, config: Optional[SharpContextConfig] = None):
+        self.config = config or SharpContextConfig()
+
+        # Fragment store
+        self._fragments: Dict[str, ContextFragment] = {}
+        self._current_turn: int = 0
+
+        # Global token distribution (for surprisal computation)
+        self._global_token_counts: Counter = Counter()
+        self._total_token_count: int = 0
+
+        # Subsystems
+        self._dedup = DedupIndex(hamming_threshold=3)
+        self._prefetch = PrefetchEngine(co_access_window=5)
+        self._checkpoint_mgr = CheckpointManager(
+            checkpoint_dir=self.config.checkpoint_dir,
+            auto_interval=self.config.auto_checkpoint_interval,
+        )
+
+        # Stats
+        self._total_tokens_saved: int = 0
+        self._total_optimizations: int = 0
+        self._total_fragments_ingested: int = 0
+        self._total_duplicates_caught: int = 0
+
+    def advance_turn(self) -> None:
+        """Advance the turn counter and apply Ebbinghaus decay."""
+        self._current_turn += 1
+        fragments = list(self._fragments.values())
+        apply_ebbinghaus_decay(
+            fragments,
+            self._current_turn,
+            self.config.decay_half_life_turns,
+        )
+
+        # Evict fragments below minimum relevance
+        to_evict = [
+            fid for fid, f in self._fragments.items()
+            if f.recency_score < self.config.min_relevance_threshold
+            and not f.is_pinned
+        ]
+        for fid in to_evict:
+            self._dedup.remove(fid)
+            del self._fragments[fid]
+
+    def ingest_fragment(
+        self,
+        content: str,
+        source: str = "",
+        token_count: int = 0,
+        is_pinned: bool = False,
+    ) -> Dict[str, Any]:
+        """
+        Ingest a new context fragment.
+
+        Pipeline:
+          1. Estimate token count if not provided
+          2. Compute SimHash fingerprint
+          3. Check for near-duplicates
+          4. Compute Shannon entropy score
+          5. Update global token distribution
+          6. Store fragment
+
+        Returns a dict with status and fragment metadata.
+        """
+        self._total_fragments_ingested += 1
+
+        # Estimate tokens (~4 chars per token for English/code)
+        if token_count <= 0:
+            token_count = max(1, len(content) // 4)
+
+        # Generate fragment ID
+        frag_id = hashlib.sha256(
+            f"{source}:{content[:200]}:{self._total_fragments_ingested}".encode()
+        ).hexdigest()[:16]
+
+        # Check for duplicates
+        dup_id = self._dedup.insert(frag_id, content)
+        if dup_id is not None:
+            # Duplicate found — boost existing fragment
+            self._total_duplicates_caught += 1
+            existing = self._fragments.get(dup_id)
+            if existing:
+                existing.access_count += 1
+                existing.turn_last_accessed = self._current_turn
+                # Boost frequency score (spaced repetition from hippocampus engine)
+                max_freq = max(
+                    f.access_count for f in self._fragments.values()
+                ) or 1
+                existing.frequency_score = min(
+                    existing.access_count / max_freq, 1.0
+                )
+            return {
+                "status": "duplicate",
+                "duplicate_of": dup_id,
+                "fragment_id": frag_id,
+                "tokens_saved": token_count,
+            }
+
+        # Compute entropy score
+        other_contents = [
+            f.content for f in self._fragments.values()
+        ]
+        entropy_score = compute_information_score(
+            content,
+            global_token_counts=dict(self._global_token_counts),
+            total_tokens=self._total_token_count,
+            other_fragments=other_contents[:50],  # Limit for performance
+        )
+
+        # Update global token distribution
+        tokens = content.lower().split()
+        self._global_token_counts.update(tokens)
+        self._total_token_count += len(tokens)
+
+        # Create fragment
+        frag = ContextFragment(
+            fragment_id=frag_id,
+            content=content,
+            token_count=token_count,
+            source=source,
+            recency_score=1.0,
+            frequency_score=0.0,
+            semantic_score=0.0,
+            entropy_score=entropy_score,
+            turn_created=self._current_turn,
+            turn_last_accessed=self._current_turn,
+            access_count=1,
+            is_pinned=is_pinned,
+            simhash=simhash(content),
+        )
+
+        self._fragments[frag_id] = frag
+
+        # Record access for pre-fetcher
+        if source:
+            self._prefetch.record_access(source, self._current_turn)
+
+        # Auto-checkpoint
+        if self._checkpoint_mgr.should_auto_checkpoint():
+            self._auto_checkpoint()
+
+        return {
+            "status": "ingested",
+            "fragment_id": frag_id,
+            "token_count": token_count,
+            "entropy_score": round(entropy_score, 4),
+            "total_fragments": len(self._fragments),
+        }
+
+    def optimize_context(
+        self,
+        token_budget: int = 0,
+        query: str = "",
+    ) -> Dict[str, Any]:
+        """
+        Select the mathematically optimal subset of context fragments
+        that fits within the token budget.
+
+        Uses the knapsack DP algorithm with four-dimensional scoring:
+          - Recency (Ebbinghaus decay)
+          - Frequency (spaced repetition)
+          - Semantic similarity (SimHash distance to query)
+          - Information density (Shannon entropy)
+
+        This is the core innovation — the context equivalent of
+        Ebbiforge's ContextScorer.score() method, but applied to
+        the knapsack selection problem.
+        """
+        if token_budget <= 0:
+            token_budget = self.config.default_token_budget
+
+        self._total_optimizations += 1
+
+        # Update semantic scores if a query is provided
+        if query:
+            query_hash = simhash(query)
+            for frag in self._fragments.values():
+                # Use SimHash Hamming distance for semantic similarity
+                # Same approach as Ebbiforge's ContextScorer:
+                #   similarity = 1.0 - (hamming_dist / max_bits)
+                from .dedup import hamming_distance
+                dist = hamming_distance(query_hash, frag.simhash)
+                frag.semantic_score = max(0.0, 1.0 - (dist / 32.0))
+
+        # Run knapsack optimizer
+        fragments = list(self._fragments.values())
+        selected, stats = knapsack_optimize(
+            fragments,
+            token_budget,
+            w_recency=self.config.weight_recency,
+            w_frequency=self.config.weight_frequency,
+            w_semantic=self.config.weight_semantic_sim,
+            w_entropy=self.config.weight_entropy,
+        )
+
+        # Track savings
+        total_available_tokens = sum(f.token_count for f in fragments)
+        tokens_saved = total_available_tokens - stats["total_tokens"]
+        self._total_tokens_saved += max(0, tokens_saved)
+
+        # Mark selected fragments as accessed
+        for frag in selected:
+            frag.turn_last_accessed = self._current_turn
+            frag.access_count += 1
+
+        return {
+            "selected_fragments": [
+                {
+                    "id": f.fragment_id,
+                    "source": f.source,
+                    "token_count": f.token_count,
+                    "relevance": round(
+                        compute_relevance(
+                            f,
+                            self.config.weight_recency,
+                            self.config.weight_frequency,
+                            self.config.weight_semantic_sim,
+                            self.config.weight_entropy,
+                        ), 4
+                    ),
+                    "content_preview": f.content[:100] + "..." if len(f.content) > 100 else f.content,
+                }
+                for f in selected
+            ],
+            "optimization_stats": stats,
+            "tokens_saved_this_call": max(0, tokens_saved),
+            "total_tokens_saved_session": self._total_tokens_saved,
+        }
+
+    def recall_relevant(
+        self,
+        query: str,
+        top_k: int = 5,
+    ) -> List[Dict[str, Any]]:
+        """
+        Semantic recall of the most relevant fragments for a query.
+
+        Uses the same SimHash + weighted scoring pipeline as the
+        Ebbiforge HippocampusEngine.recall() method.
+        """
+        if not self._fragments:
+            return []
+
+        query_hash = simhash(query)
+
+        scored = []
+        for frag in self._fragments.values():
+            from .dedup import hamming_distance
+            dist = hamming_distance(query_hash, frag.simhash)
+            frag.semantic_score = max(0.0, 1.0 - (dist / 32.0))
+
+            relevance = compute_relevance(
+                frag,
+                self.config.weight_recency,
+                self.config.weight_frequency,
+                self.config.weight_semantic_sim,
+                self.config.weight_entropy,
+            )
+            scored.append((frag, relevance))
+
+        scored.sort(key=lambda x: x[1], reverse=True)
+
+        return [
+            {
+                "fragment_id": f.fragment_id,
+                "source": f.source,
+                "relevance": round(rel, 4),
+                "entropy": round(f.entropy_score, 4),
+                "content": f.content,
+            }
+            for f, rel in scored[:top_k]
+        ]
+
+    def prefetch_related(
+        self,
+        file_path: str,
+        source_content: str = "",
+        language: str = "python",
+    ) -> List[Dict[str, Any]]:
+        """
+        Predict what context will be needed next and pre-fetch it.
+        """
+        predictions = self._prefetch.predict(
+            file_path, source_content, language
+        )
+        return [
+            {
+                "path": p.path,
+                "reason": p.reason,
+                "confidence": p.confidence,
+            }
+            for p in predictions
+        ]
+
+    def checkpoint(self, metadata: Optional[Dict[str, Any]] = None) -> str:
+        """Manually create a checkpoint."""
+        return self._auto_checkpoint(metadata)
+
+    def resume(self) -> Dict[str, Any]:
+        """Resume from the latest checkpoint."""
+        ckpt = self._checkpoint_mgr.load_latest()
+        if ckpt is None:
+            return {"status": "no_checkpoint_found"}
+
+        # Restore fragments
+        self._fragments.clear()
+        for frag in self._checkpoint_mgr.restore_fragments(ckpt):
+            self._fragments[frag.fragment_id] = frag
+
+        # Restore dedup index
+        self._dedup = DedupIndex(hamming_threshold=3)
+        for fid, fp in ckpt.dedup_fingerprints.items():
+            self._dedup._fingerprints[fid] = fp
+
+        # Restore co-access
+        for src, targets in ckpt.co_access_data.items():
+            self._prefetch._co_access[src] = Counter(targets)
+
+        self._current_turn = ckpt.current_turn
+
+        return {
+            "status": "resumed",
+            "checkpoint_id": ckpt.checkpoint_id,
+            "restored_fragments": len(self._fragments),
+            "restored_turn": self._current_turn,
+            "metadata": ckpt.metadata,
+        }
+
+    def get_stats(self) -> Dict[str, Any]:
+        """Get comprehensive session statistics."""
+        fragments = list(self._fragments.values())
+        total_tokens = sum(f.token_count for f in fragments)
+        avg_entropy = (
+            sum(f.entropy_score for f in fragments) / len(fragments)
+            if fragments else 0.0
+        )
+
+        return {
+            "session": {
+                "current_turn": self._current_turn,
+                "total_fragments": len(fragments),
+                "total_tokens_tracked": total_tokens,
+                "avg_entropy_score": round(avg_entropy, 4),
+                "pinned_fragments": sum(1 for f in fragments if f.is_pinned),
+            },
+            "savings": {
+                "total_tokens_saved": self._total_tokens_saved,
+                "total_duplicates_caught": self._total_duplicates_caught,
+                "total_optimizations": self._total_optimizations,
+                "total_fragments_ingested": self._total_fragments_ingested,
+                "estimated_cost_saved_usd": round(
+                    self._total_tokens_saved * 0.000003, 4  # ~$3/1M tokens
+                ),
+            },
+            "dedup": self._dedup.stats(),
+            "prefetch": self._prefetch.stats(),
+            "checkpoint": self._checkpoint_mgr.stats(),
+        }
+
+    def _auto_checkpoint(
+        self,
+        metadata: Optional[Dict[str, Any]] = None,
+    ) -> str:
+        """Create an auto-checkpoint."""
+        co_access = {
+            k: dict(v)
+            for k, v in self._prefetch._co_access.items()
+        }
+        return self._checkpoint_mgr.save(
+            fragments=list(self._fragments.values()),
+            dedup_fingerprints=dict(self._dedup._fingerprints),
+            co_access_data=co_access,
+            current_turn=self._current_turn,
+            metadata=metadata,
+            stats=self.get_stats(),
+        )
+
+
+# ══════════════════════════════════════════════════════════════════════
+# MCP Server Definition
+# ══════════════════════════════════════════════════════════════════════
+
+def create_mcp_server():
+    """
+    Create the MCP server with all tools registered.
+
+    Uses the FastMCP SDK for automatic tool schema generation
+    from Python type hints and docstrings.
+    """
+    try:
+        from mcp.server.fastmcp import FastMCP
+    except ImportError:
+        logger.error(
+            "MCP SDK not installed. Install with: pip install mcp"
+        )
+        raise
+
+    mcp = FastMCP(
+        "sharp-context",
+        version="0.1.0",
+        description=(
+            "Information-theoretic context optimization for AI coding agents. "
+            "Knapsack-optimal token budgeting, Shannon entropy scoring, "
+            "SimHash deduplication, predictive pre-fetch, and checkpoint/resume."
+        ),
+    )
+
+    # Shared engine instance
+    engine = SharpContextEngine()
+
+    @mcp.tool()
+    def remember_fragment(
+        content: str,
+        source: str = "",
+        token_count: int = 0,
+        is_pinned: bool = False,
+    ) -> str:
+        """Store a context fragment with automatic dedup and entropy scoring.
+
+        Fragments are fingerprinted via SimHash for O(1) duplicate detection.
+        Each fragment's information density is scored using Shannon entropy.
+        Duplicates are automatically merged with salience boosting.
+
+        Args:
+            content: The text content to store (code, tool output, etc.)
+            source: Origin label (e.g., 'file:utils.py', 'tool:grep')
+            token_count: Token count (auto-estimated if 0)
+            is_pinned: If True, always include in optimized context
+        """
+        engine.advance_turn()
+        result = engine.ingest_fragment(content, source, token_count, is_pinned)
+        return json.dumps(result, indent=2)
+
+    @mcp.tool()
+    def optimize_context(
+        token_budget: int = 128000,
+        query: str = "",
+    ) -> str:
+        """Select the mathematically optimal context subset for a token budget.
+
+        Uses 0/1 Knapsack dynamic programming to maximize relevance within
+        the budget. Scores fragments on four dimensions: recency (Ebbinghaus
+        decay), access frequency (spaced repetition), semantic similarity
+        (SimHash), and information density (Shannon entropy).
+
+        This is the core tool — call it before sending context to the LLM.
+
+        Args:
+            token_budget: Maximum tokens allowed (default: 128K)
+            query: Current query/task for semantic relevance scoring
+        """
+        result = engine.optimize_context(token_budget, query)
+        return json.dumps(result, indent=2)
+
+    @mcp.tool()
+    def recall_relevant(
+        query: str,
+        top_k: int = 5,
+    ) -> str:
+        """Semantic recall of the most relevant stored fragments.
+
+        Uses SimHash fingerprint distance + multi-dimensional scoring
+        (same pipeline as Ebbiforge's HippocampusEngine.recall()).
+
+        Args:
+            query: The search query
+            top_k: Number of results to return
+        """
+        results = engine.recall_relevant(query, top_k)
+        return json.dumps(results, indent=2)
+
+    @mcp.tool()
+    def checkpoint_state(
+        task_description: str = "",
+        current_step: str = "",
+    ) -> str:
+        """Save current state to disk for crash recovery and session resume.
+
+        Checkpoints include all fragments, dedup index, co-access patterns,
+        and custom metadata. Stored as gzipped JSON (~50-200 KB).
+
+        Args:
+            task_description: What the agent is working on
+            current_step: Where in the task it currently is
+        """
+        metadata = {}
+        if task_description:
+            metadata["task"] = task_description
+        if current_step:
+            metadata["step"] = current_step
+
+        path = engine.checkpoint(metadata)
+        return json.dumps({
+            "status": "checkpoint_saved",
+            "path": path,
+            "fragments_saved": len(engine._fragments),
+        }, indent=2)
+
+    @mcp.tool()
+    def resume_state() -> str:
+        """Resume from the latest checkpoint.
+
+        Restores all context fragments, dedup index, co-access patterns,
+        and custom metadata from the most recent checkpoint.
+        """
+        result = engine.resume()
+        return json.dumps(result, indent=2)
+
+    @mcp.tool()
+    def prefetch_related(
+        file_path: str,
+        source_content: str = "",
+        language: str = "python",
+    ) -> str:
+        """Predict and pre-load context that will likely be needed next.
+
+        Combines static analysis (imports, callees, test files) with
+        learned co-access patterns to predict what the agent will need.
+
+        Args:
+            file_path: The file currently being accessed
+            source_content: The source code content (for static analysis)
+            language: Programming language (python, typescript, rust)
+        """
+        predictions = engine.prefetch_related(file_path, source_content, language)
+        return json.dumps(predictions, indent=2)
+
+    @mcp.tool()
+    def get_stats() -> str:
+        """Get comprehensive session statistics.
+
+        Shows token savings, duplicate detection counts, entropy
+        distribution, checkpoint status, and cost estimates.
+        """
+        stats = engine.get_stats()
+        return json.dumps(stats, indent=2)
+
+    return mcp
+
+
+def main():
+    """Entry point for the sharp-context MCP server."""
+    logger.info("Starting SharpContext MCP server v0.1.0")
+    mcp = create_mcp_server()
+    mcp.run()
+
+
+if __name__ == "__main__":
+    main()