code-context-control 2.28.0__py3-none-any.whl

services/session_preloader.py

@@ -0,0 +1,251 @@
+"""SessionPreloader — First-prompt auto-retrieval for Local RAG Pipeline.
+
+On the first c3_memory(action='recall') in a session, this module automatically
+retrieves relevant doc chunks, code context, and session history, then merges
+them into a pre-context block injected before the normal recall results.
+
+This eliminates repeated discovery work across sessions for the same topics.
+"""
+
+import logging
+import re
+from typing import Optional
+
+from core import count_tokens
+
+log = logging.getLogger(__name__)
+
+# Budget cap for pre-context injection
+_DEFAULT_MAX_PRECONTEXT_TOKENS = 400
+
+# Minimum score threshold for doc chunks to be included
+_MIN_DOC_SCORE = 0.05
+
+
+class SessionPreloader:
+    """Auto-retrieves relevant project context on first recall of a session."""
+
+    def __init__(self, doc_index, embedding_index=None, session_mgr=None,
+                 memory_store=None, config: Optional[dict] = None):
+        self.doc_index = doc_index
+        self.embedding_index = embedding_index
+        self.session_mgr = session_mgr
+        self.memory_store = memory_store
+        self._config = config or {}
+        self._preloaded_sessions: set = set()  # session IDs that already got preloaded
+
+    @property
+    def max_tokens(self) -> int:
+        return self._config.get("max_precontext_tokens", _DEFAULT_MAX_PRECONTEXT_TOKENS)
+
+    @property
+    def enabled(self) -> bool:
+        return self._config.get("enabled", True)
+
+    def should_preload(self, session_id: str) -> bool:
+        """Check if this session hasn't been preloaded yet."""
+        if not self.enabled:
+            return False
+        if not self.doc_index:
+            return False
+        if not self.doc_index.chunks:
+            return False
+        return session_id not in self._preloaded_sessions
+
+    def preload(self, query: str, session_id: str, top_k: int = 5) -> str:
+        """Generate pre-context for the first recall in a session.
+
+        Returns a formatted string to prepend to the recall results,
+        or empty string if nothing relevant found.
+        """
+        if not self.should_preload(session_id):
+            return ""
+
+        self._preloaded_sessions.add(session_id)
+
+        # Extract expanded signals from the query
+        signals = self._extract_signals(query)
+        # Skip preloading for simple/short queries — not worth 700+ token injection
+        if len(signals) < 3:
+            return ""
+
+        signal_query = " ".join(signals)
+
+        # Retrieve from doc index
+        doc_results = self.doc_index.search(signal_query, top_k=top_k * 2)
+
+        # Also try embedding-based search if available
+        if self.embedding_index and self.embedding_index.ready:
+            try:
+                embed_results = self.embedding_index.search(query, top_k=3)
+                # Convert to comparable format (embed results have different shape)
+                for er in embed_results:
+                    if er.get("content") and er.get("score", 0) > 0.3:
+                        doc_results.append({
+                            "id": er.get("chunk_id", er.get("id", "")),
+                            "doc_id": er.get("doc_id", ""),
+                            "content": er["content"],
+                            "tokens": count_tokens(er["content"]),
+                            "kind": "code",
+                            "source_type": "code_semantic",
+                            "priority": 1.0,
+                            "score": er["score"] * 0.8,  # slightly discount code vs docs
+                            "heading_path": [er.get("doc_id", "")],
+                        })
+            except Exception:
+                pass
+
+        # Get recent session context as weak signal
+        session_context = self._get_session_signals()
+
+        # Rank, deduplicate, and budget-cap
+        precontext = self._build_precontext(doc_results, session_context)
+
+        if not precontext:
+            return ""
+
+        return precontext
+
+    def _extract_signals(self, query: str) -> list[str]:
+        """Extract retrieval signals from the user's query."""
+        signals = []
+
+        # Direct tokens from query
+        tokens = re.findall(r"\w+", query.lower())
+        # Filter out very short/common words
+        stopwords = {"the", "a", "an", "is", "are", "was", "were", "be", "been",
+                      "for", "to", "of", "in", "on", "at", "by", "with", "from",
+                      "and", "or", "not", "this", "that", "it", "as", "do", "does",
+                      "has", "have", "had", "can", "could", "will", "would", "should",
+                      "may", "might", "about", "what", "how", "when", "where", "which",
+                      "who", "all", "any", "some", "no", "my", "your", "our", "their"}
+        signals.extend(t for t in tokens if t not in stopwords and len(t) > 1)
+
+        # Extract file paths mentioned in query
+        file_patterns = re.findall(r"[\w/\\]+\.[\w]+", query)
+        for fp in file_patterns:
+            # Add stem words from file path
+            parts = re.split(r"[/\\_.\-]", fp)
+            signals.extend(p.lower() for p in parts if len(p) > 1)
+
+        return list(dict.fromkeys(signals))  # deduplicate preserving order
+
+    def _get_session_signals(self) -> str:
+        """Get compressed context from recent sessions as weak signals."""
+        if not self.session_mgr:
+            return ""
+
+        try:
+            return self.session_mgr.get_session_context(n_sessions=2)
+        except Exception:
+            return ""
+
+    def _build_precontext(self, doc_results: list, session_context: str) -> str:
+        """Build the final pre-context string within token budget."""
+        if not doc_results and not session_context:
+            return ""
+
+        budget = self.max_tokens
+        parts = []
+        used_tokens = 0
+        seen_docs = set()
+
+        # Header
+        header = "[session:pre-context] Auto-retrieved project context"
+        used_tokens += count_tokens(header) + 2  # +2 for newlines
+
+        # Sort by score descending
+        doc_results.sort(key=lambda x: x.get("score", 0), reverse=True)
+
+        # Filter low-score results
+        doc_results = [r for r in doc_results if r.get("score", 0) >= _MIN_DOC_SCORE]
+
+        # Group by source file for cleaner output
+        for result in doc_results:
+            doc_id = result.get("doc_id", "")
+            content = result.get("content", "").strip()
+            if not content:
+                continue
+
+            # Deduplicate by doc_id + heading
+            dedup_key = f"{doc_id}::{result.get('id', '')}"
+            if dedup_key in seen_docs:
+                continue
+            seen_docs.add(dedup_key)
+
+            # Format chunk
+            source_label = self._source_label(result)
+            chunk_text = f"\n## {source_label}\n{content}"
+            chunk_tokens = count_tokens(chunk_text)
+
+            if used_tokens + chunk_tokens > budget:
+                # Try to fit a truncated version
+                remaining = budget - used_tokens - 20  # margin
+                if remaining > 50:
+                    lines = content.split("\n")
+                    truncated = []
+                    t = 0
+                    for line in lines:
+                        lt = count_tokens(line)
+                        if t + lt > remaining:
+                            break
+                        truncated.append(line)
+                        t += lt
+                    if truncated:
+                        chunk_text = f"\n## {source_label}\n" + "\n".join(truncated) + "\n..."
+                        chunk_tokens = count_tokens(chunk_text)
+                        parts.append(chunk_text)
+                        used_tokens += chunk_tokens
+                break  # budget exhausted
+            else:
+                parts.append(chunk_text)
+                used_tokens += chunk_tokens
+
+        # Add session context if budget remains
+        if session_context and used_tokens < budget - 100:
+            remaining = budget - used_tokens - 10
+            sc_tokens = count_tokens(session_context)
+            if sc_tokens > remaining:
+                # Truncate session context
+                lines = session_context.split("\n")
+                truncated = []
+                t = 0
+                for line in lines:
+                    lt = count_tokens(line)
+                    if t + lt > remaining:
+                        break
+                    truncated.append(line)
+                    t += lt
+                session_context = "\n".join(truncated)
+
+            if session_context.strip():
+                parts.append(f"\n## Recent Session Context\n{session_context.strip()}")
+
+        if not parts:
+            return ""
+
+        chunk_count = len([p for p in parts if p.startswith("\n##")])
+        total_tokens = sum(count_tokens(p) for p in parts)
+        header = f"[session:pre-context] Auto-retrieved project context ({chunk_count} chunks, {total_tokens} tokens)"
+
+        return header + "\n" + "\n".join(parts) + "\n\n---\n"
+
+    def _source_label(self, result: dict) -> str:
+        """Generate a human-readable source label for a chunk."""
+        doc_id = result.get("doc_id", "unknown")
+        source_type = result.get("source_type", "")
+        heading_path = result.get("heading_path", [])
+
+        if source_type == "markdown":
+            if len(heading_path) > 1:
+                return f"{heading_path[-1]} (from {doc_id})"
+            return f"From {doc_id}"
+        elif source_type == "docstring":
+            name = result.get("id", "").split("::")[-1] if "::" in result.get("id", "") else doc_id
+            return f"Docstring: {name} (from {doc_id})"
+        elif source_type == "config":
+            return f"Config: {doc_id}"
+        elif source_type == "code_semantic":
+            return f"Related code: {doc_id}"
+        else:
+            return f"From {doc_id}"

services/text_index.py

@@ -0,0 +1,90 @@
+"""Lightweight incremental text index used by local memory stores."""
+
+from __future__ import annotations
+
+import math
+import re
+from collections import Counter
+
+
+class TextIndex:
+    """Incremental TF-IDF index over small local document collections."""
+
+    def __init__(self):
+        self._docs: dict[str, str] = {}
+        self._tf: dict[str, Counter] = {}
+        self._df: Counter = Counter()
+
+    @staticmethod
+    def tokenize(text: str) -> list[str]:
+        text = re.sub(r"([a-z])([A-Z])", r"\1 \2", text or "")
+        text = text.replace("_", " ").replace("-", " ")
+        return re.findall(r"[a-zA-Z0-9]{2,}", text.lower())
+
+    def __len__(self) -> int:
+        return len(self._docs)
+
+    def clear(self):
+        self._docs.clear()
+        self._tf.clear()
+        self._df.clear()
+
+    def ids(self) -> list[str]:
+        return list(self._docs.keys())
+
+    def get_text(self, doc_id: str) -> str:
+        return self._docs.get(doc_id, "")
+
+    def rebuild(self, docs: dict[str, str]):
+        self.clear()
+        for doc_id, text in docs.items():
+            self.add_or_update(doc_id, text)
+
+    def add_or_update(self, doc_id: str, text: str):
+        text = text or ""
+        old_tf = self._tf.get(doc_id)
+        if old_tf:
+            for token in old_tf:
+                self._df[token] -= 1
+                if self._df[token] <= 0:
+                    del self._df[token]
+
+        tokens = self.tokenize(text)
+        tf = Counter(tokens)
+        self._docs[doc_id] = text
+        self._tf[doc_id] = tf
+        for token in tf:
+            self._df[token] += 1
+
+    def remove(self, doc_id: str):
+        old_tf = self._tf.pop(doc_id, None)
+        self._docs.pop(doc_id, None)
+        if not old_tf:
+            return
+        for token in old_tf:
+            self._df[token] -= 1
+            if self._df[token] <= 0:
+                del self._df[token]
+
+    def search(self, query: str, top_k: int = 5) -> list[tuple[str, float]]:
+        query_tokens = self.tokenize(query)
+        if not query_tokens or not self._docs:
+            return []
+
+        q_terms = Counter(query_tokens)
+        total_docs = len(self._docs)
+        scores: dict[str, float] = {}
+
+        for doc_id, term_counts in self._tf.items():
+            total_terms = sum(term_counts.values()) or 1
+            score = 0.0
+            for term, q_count in q_terms.items():
+                if term not in term_counts:
+                    continue
+                tf = term_counts[term] / total_terms
+                idf = math.log((total_docs + 1) / (self._df.get(term, 0) + 1)) + 1.0
+                score += tf * idf * q_count
+            if score > 0:
+                scores[doc_id] = score
+
+        return sorted(scores.items(), key=lambda item: item[1], reverse=True)[:top_k]

services/tool_classifier.py

@@ -0,0 +1,176 @@
+"""Tool classifier for the MCP proxy — dynamically filters visible tools.
+
+Categorizes the ~26 C3 tools into 6 groups and selects which groups are
+relevant based on recent tool usage, keyword patterns, and optional SLM input.
+"""
+import re
+from typing import Optional
+
+from services.ollama_client import OllamaClient
+
+# ── Tool Categories ────────────────────────────────────────
+
+CATEGORIES = {
+    "core": {
+        "tools": [
+            "c3_search", "c3_compress", "c3_validate", "c3_filter",
+            "c3_session", "c3_memory", "c3_read", "c3_impact", "c3_shell",
+        ],
+        "keywords": None,  # Always included
+        "priority": 0,
+    },
+    "analysis": {
+        "tools": [
+            "c3_delegate",
+        ],
+        "keywords": re.compile(
+            r"hybrid|ollama|filter|route|summarize|llm|tier|raw\s*output|delegate",
+            re.IGNORECASE,
+        ),
+        "priority": 1,
+    },
+    "meta": {
+        "tools": [
+            "c3_status",
+        ],
+        "keywords": re.compile(
+            r"token|stats|optimi[sz]e|index|rebuild|notif|budget|context\s*status",
+            re.IGNORECASE,
+        ),
+        "priority": 2,
+    },
+}
+
+# Reverse lookup: tool name -> category
+_TOOL_TO_CATEGORY = {}
+for _cat, _info in CATEGORIES.items():
+    for _tool in _info["tools"]:
+        _TOOL_TO_CATEGORY[_tool] = _cat
+
+
+class ToolClassifier:
+    """Selects which tool categories are active based on context."""
+
+    def __init__(self, always_visible: list[str] = None,
+                 max_tools: int = 12,
+                 use_slm: bool = True,
+                 slm_model: str = "gemma3n:latest",
+                 ollama: Optional[OllamaClient] = None):
+        self.always_visible = always_visible or ["core"]
+        self.max_tools = max_tools
+        self.use_slm = use_slm
+        self.slm_model = slm_model
+        self.ollama = ollama
+        self.classification_reasons: dict[str, str] = {}
+
+    def classify(self, recent_tool_names: list[str],
+                 recent_text: str) -> list[str]:
+        """Return list of active category names."""
+        # "all" shortcut — every category is always visible
+        if "all" in self.always_visible:
+            all_cats = sorted(CATEGORIES, key=lambda c: CATEGORIES[c].get("priority", 99))
+            self.classification_reasons = {c: "always" for c in all_cats}
+            return all_cats
+
+        active = set(self.always_visible)
+        reasons: dict[str, str] = {}
+
+        # Always-visible categories
+        for cat in self.always_visible:
+            reasons[cat] = "always"
+
+        # Include categories of recently-used tools
+        for name in recent_tool_names[-5:]:
+            cat = _TOOL_TO_CATEGORY.get(name)
+            if cat and cat not in active:
+                reasons[cat] = "recent"
+                active.add(cat)
+            elif cat and cat not in reasons:
+                reasons[cat] = "recent"
+
+        # Keyword scan
+        for cat_name, cat_info in CATEGORIES.items():
+            if cat_name in active:
+                continue
+            pattern = cat_info["keywords"]
+            if pattern and pattern.search(recent_text):
+                active.add(cat_name)
+                reasons[cat_name] = "keyword"
+
+        # SLM refinement if heuristic is narrow
+        if (len(active) <= 2 and self.use_slm
+                and self.ollama and recent_text.strip()):
+            slm_cats = self._slm_classify(recent_text, active)
+            if slm_cats:
+                for cat in slm_cats:
+                    reasons[cat] = "slm"
+                active.update(slm_cats)
+
+        self.classification_reasons = reasons
+        return sorted(active, key=lambda c: CATEGORIES.get(c, {}).get("priority", 99))
+
+    def filter_tools(self, all_tools: list[dict],
+                     active_categories: list[str]) -> list[dict]:
+        """Filter a tools/list response to only include active categories."""
+        # Build set of allowed tool names
+        allowed = set()
+        for cat in active_categories:
+            cat_info = CATEGORIES.get(cat)
+            if cat_info:
+                allowed.update(cat_info["tools"])
+
+        filtered = [t for t in all_tools if t.get("name") in allowed]
+
+        # Cap at max_tools by priority
+        if len(filtered) > self.max_tools:
+            # Sort by category priority, keep first max_tools
+            def tool_priority(t):
+                cat = _TOOL_TO_CATEGORY.get(t.get("name"), "")
+                return CATEGORIES.get(cat, {}).get("priority", 99)
+            filtered.sort(key=tool_priority)
+            filtered = filtered[:self.max_tools]
+
+        return filtered
+
+    def get_active_tool_count(self, active_categories: list[str]) -> int:
+        """Count how many tools would be visible for given categories."""
+        count = 0
+        for cat in active_categories:
+            cat_info = CATEGORIES.get(cat)
+            if cat_info:
+                count += len(cat_info["tools"])
+        return min(count, self.max_tools)
+
+    # ── SLM Refinement ─────────────────────────────────────
+
+    def _slm_classify(self, text: str, current: set[str]) -> list[str]:
+        """Ask SLM which additional categories might be relevant."""
+        available = [c for c in CATEGORIES if c not in current]
+        if not available:
+            return []
+
+        prompt = (
+            f"Given this context: {text[:200]}\n\n"
+            f"Which of these tool categories are relevant? "
+            f"Categories: {', '.join(available)}\n"
+            f"- core: search, compress, read, filter, validate, session, memory\n"
+            f"- analysis: delegate tasks to local LLM\n"
+            f"- meta: status, budget, notifications, health\n\n"
+            f"Reply with ONLY the category names, comma-separated. "
+            f"If none are relevant, reply NONE."
+        )
+
+        try:
+            result = self.ollama.generate(
+                prompt=prompt,
+                model=self.slm_model,
+                temperature=0.0,
+                max_tokens=50,
+            )
+            if not result or "NONE" in result.upper():
+                return []
+            # Parse comma-separated category names
+            cats = [c.strip().lower() for c in result.split(",")]
+            return [c for c in cats if c in CATEGORIES]
+        except Exception:
+            return []