code-context-engine 0.4.0__py3-none-any.whl

context_engine/cli_style.py

@@ -0,0 +1,66 @@
+"""Shared CLI styling — consistent colorful output across all cce commands."""
+import sys
+import time
+
+import click
+
+
+# ── Colors ─────────────────────────────────────────────────────────────
+# Cyan for headers/labels, green for success, yellow for warnings,
+# dim white for secondary info, bold for emphasis.
+
+def header(text: str) -> str:
+    return click.style(text, fg="cyan", bold=True)
+
+def label(text: str) -> str:
+    return click.style(text, fg="cyan")
+
+def success(text: str) -> str:
+    return click.style(text, fg="green")
+
+def warn(text: str) -> str:
+    return click.style(text, fg="yellow")
+
+def error(text: str) -> str:
+    return click.style(text, fg="red")
+
+def dim(text: str) -> str:
+    return click.style(text, dim=True)
+
+def bold(text: str) -> str:
+    return click.style(text, bold=True)
+
+def value(text: str) -> str:
+    return click.style(text, fg="white", bold=True)
+
+def magenta(text: str) -> str:
+    return click.style(text, fg="magenta")
+
+
+# ── Prefixes ───────────────────────────────────────────────────────────
+
+CHECK = click.style("✓", fg="green")
+CROSS = click.style("✗", fg="red")
+DOT = click.style("·", fg="yellow")
+ARROW = click.style("→", fg="cyan")
+BULLET = click.style("●", fg="green")
+BULLET_OFF = click.style("○", fg="yellow")
+
+
+# ── Section headers ────────────────────────────────────────────────────
+
+def section(title: str, width: int = 50) -> str:
+    """Render a styled section divider: ── Title ──────────"""
+    bar = "─" * max(1, width - len(title) - 4)
+    return f"  {dim('──')} {header(title)} {dim(bar)}"
+
+
+# ── Animation ──────────────────────────────────────────────────────────
+
+def animate(lines: list[str], delay: float = 0.025) -> None:
+    """Print lines with a reveal animation on TTY, instant otherwise."""
+    is_tty = sys.stdout.isatty()
+    for i, line in enumerate(lines):
+        click.echo(line)
+        if is_tty and i < 12 and delay > 0:
+            time.sleep(delay)

context_engine/compression/compressor.py

@@ -0,0 +1,144 @@
+"""Compression pipeline — groups chunks, summarizes via LLM, falls back to truncation."""
+import asyncio
+import logging
+import time
+from context_engine.models import Chunk, ChunkType
+from context_engine.compression.ollama_client import OllamaClient
+from context_engine.compression.prompts import CODE_PROMPT, DECISION_PROMPT, ARCHITECTURE_PROMPT, DOC_PROMPT
+from context_engine.compression.quality import QualityChecker
+
+log = logging.getLogger(__name__)
+
+# Ollama liveness probe is ~5ms over loopback but fires once per compress() call,
+# adding up across a session. Refresh at most every _OLLAMA_PROBE_TTL seconds.
+_OLLAMA_PROBE_TTL = 30.0
+
+_PROMPT_MAP = {
+    ChunkType.FUNCTION: CODE_PROMPT, ChunkType.CLASS: CODE_PROMPT,
+    ChunkType.MODULE: ARCHITECTURE_PROMPT, ChunkType.DOC: DOC_PROMPT,
+    ChunkType.DECISION: DECISION_PROMPT, ChunkType.SESSION: DOC_PROMPT,
+    ChunkType.COMMIT: DOC_PROMPT, ChunkType.COMMENT: DOC_PROMPT,
+}
+_TRUNCATION_LIMITS: dict[str, int] = {"minimal": 100, "standard": 300, "full": 800}
+# In "full" mode we pass a chunk through uncompressed only if we're confident it
+# came from the retrieval pipeline (which sets confidence_score) — chunks
+# ingested by other paths default to 0.0 and would otherwise be falsely
+# classified as low-confidence. A chunk with no embedding wasn't retrieved.
+_FULL_PASSTHROUGH_THRESHOLD = 0.8
+
+
+class Compressor:
+    def __init__(
+        self,
+        ollama_url: str = "http://localhost:11434",
+        model: str = "phi3:mini",
+        cache=None,
+    ) -> None:
+        """`cache` is any object exposing get_cached_compression(chunk_id, level)
+        and put_cached_compression(chunk_id, level, text). LocalBackend satisfies
+        this; RemoteBackend currently doesn't, so we duck-type check and disable
+        caching gracefully if either method is missing."""
+        self._client = OllamaClient(base_url=ollama_url, model=model)
+        self._quality = QualityChecker()
+        if cache is not None and hasattr(cache, "get_cached_compression") and hasattr(cache, "put_cached_compression"):
+            self._cache = cache
+        else:
+            self._cache = None
+        self._ollama_available: bool | None = None
+        self._ollama_probed_at: float = 0.0
+        # Single-flight the probe so concurrent compress() calls don't all
+        # fire it at once when the cached value is stale.
+        self._probe_lock = asyncio.Lock()
+
+    async def _is_ollama_available(self) -> bool:
+        now = time.monotonic()
+        if self._ollama_available is not None and now - self._ollama_probed_at < _OLLAMA_PROBE_TTL:
+            return self._ollama_available
+        async with self._probe_lock:
+            # Re-check inside the lock — another coroutine may have refreshed
+            # the cache while we were waiting.
+            now = time.monotonic()
+            if self._ollama_available is not None and now - self._ollama_probed_at < _OLLAMA_PROBE_TTL:
+                return self._ollama_available
+            self._ollama_available = await self._client.is_available()
+            self._ollama_probed_at = now
+            return self._ollama_available
+
+    async def compress(self, chunks: list[Chunk], level: str = "standard") -> list[Chunk]:
+        ollama_available = await self._is_ollama_available()
+        for chunk in chunks:
+            if level == "full" and self._is_full_passthrough(chunk):
+                chunk.compressed_content = chunk.content
+                continue
+            # Cache hit: skip the LLM round-trip and the truncation work.
+            if self._cache is not None:
+                cached = self._cache.get_cached_compression(chunk.id, level)
+                if cached is not None:
+                    chunk.compressed_content = cached
+                    continue
+            if ollama_available and level != "minimal":
+                chunk.compressed_content = await self._llm_compress(chunk, level)
+            else:
+                chunk.compressed_content = self._fallback_compress(chunk, level)
+            # Persist for next time. Truncation is cheap, but caching it still
+            # saves the recompute and keeps behaviour symmetric with LLM mode.
+            if self._cache is not None and chunk.compressed_content:
+                self._cache.put_cached_compression(
+                    chunk.id, level, chunk.compressed_content
+                )
+        return chunks
+
+    def _is_full_passthrough(self, chunk: Chunk) -> bool:
+        """Accept either a high retrieval-confidence chunk *or* a chunk that
+        was never routed through retrieval (no embedding) — otherwise a direct
+        lookup via `expand_chunk` would get silently truncated in full mode.
+        """
+        if chunk.confidence_score > _FULL_PASSTHROUGH_THRESHOLD:
+            return True
+        return chunk.embedding is None
+
+    async def _llm_compress(self, chunk: Chunk, level: str) -> str:
+        prompt = _PROMPT_MAP.get(chunk.chunk_type, CODE_PROMPT)
+        try:
+            summary = await self._client.summarize(chunk.content, prompt)
+        except Exception as exc:
+            log.info(
+                "Ollama summarize failed for chunk %s; falling back to truncation (%s)",
+                chunk.id,
+                exc,
+            )
+            return self._fallback_compress(chunk, level)
+        if self._quality.check(chunk.content, summary):
+            return summary
+        log.info(
+            "Quality check failed for chunk %s (identifier retention < 40%%); "
+            "falling back to truncation.",
+            chunk.id,
+        )
+        return self._fallback_compress(chunk, level)
+
+    def _fallback_compress(self, chunk: Chunk, level: str) -> str:
+        limit = _TRUNCATION_LIMITS.get(level, 300)
+        if chunk.chunk_type in (ChunkType.FUNCTION, ChunkType.CLASS):
+            return self._extract_signature(chunk.content, limit)
+        if len(chunk.content) <= limit:
+            return chunk.content
+        return chunk.content[:limit] + "..."
+
+    def _extract_signature(self, content: str, limit: int) -> str:
+        lines = content.split("\n")
+        result_lines: list[str] = []
+        in_docstring = False
+        char_count = 0
+        for line in lines:
+            if char_count + len(line) > limit and result_lines:
+                break
+            result_lines.append(line)
+            char_count += len(line) + 1
+            if '"""' in line or "'''" in line:
+                if in_docstring:
+                    break
+                in_docstring = True
+            if not in_docstring and line.strip().endswith(":") and len(result_lines) > 1:
+                break
+        return "\n".join(result_lines)

context_engine/compression/ollama_client.py

@@ -0,0 +1,33 @@
+"""Ollama API client for local LLM summarization."""
+
+import httpx
+
+
+class OllamaClient:
+    def __init__(self, base_url="http://localhost:11434", model="phi3:mini", timeout=30.0):
+        self.base_url = base_url
+        self.model = model
+        self._timeout = timeout
+
+    async def is_available(self) -> bool:
+        try:
+            async with httpx.AsyncClient(timeout=5.0) as client:
+                resp = await client.get(f"{self.base_url}/api/tags")
+                return resp.status_code == 200
+        except (httpx.ConnectError, httpx.TimeoutException):
+            return False
+
+    async def summarize(self, content: str, prompt: str) -> str:
+        full_prompt = prompt.format(content=content) if "{content}" in prompt else f"{prompt}\n\n{content}"
+        async with httpx.AsyncClient(timeout=self._timeout) as client:
+            resp = await client.post(
+                f"{self.base_url}/api/generate",
+                json={
+                    "model": self.model,
+                    "prompt": full_prompt,
+                    "stream": False,
+                    "options": {"temperature": 0.1, "num_predict": 256},
+                },
+            )
+            resp.raise_for_status()
+            return resp.json()["response"].strip()

context_engine/compression/output_rules.py

@@ -0,0 +1,77 @@
+"""Output compression rules — reduces Claude's output token usage via style directives."""
+
+LEVELS = ("off", "lite", "standard", "max")
+
+# Estimated baseline reply size (tokens) when no compression is active.
+# Used to estimate output_compression savings: the MCP server can't see
+# Claude's actual reply length, so we assume an average per affected
+# response and apply the advertised reduction. Tunable — the renderer
+# footnotes the value so users can interpret the estimate.
+ESTIMATED_AVG_REPLY_TOKENS = 500
+
+# Advertised output-token reduction per level. Sourced from the level
+# descriptions ("~65% savings", "~75% savings"). `lite` has no advertised
+# number; we use a conservative 20% based on how much filler/hedging
+# typically lives in default-mode replies.
+ADVERTISED_PCT = {
+    "off": 0.0,
+    "lite": 0.20,
+    "standard": 0.65,
+    "max": 0.75,
+}
+
+_RULES = {
+    "lite": (
+        "## Output Compression: Lite\n"
+        "Respond concisely. Rules:\n"
+        "- Remove filler words (just, really, basically, actually, simply)\n"
+        "- Remove hedging (I think, it seems, perhaps, might want to)\n"
+        "- No pleasantries (Sure!, Happy to help, Great question)\n"
+        "- No trailing summaries — the diff/output speaks for itself\n"
+        "- Keep full grammar and articles\n"
+        "- Code blocks, paths, commands, URLs: NEVER compress"
+    ),
+    "standard": (
+        "## Output Compression: Standard\n"
+        "Respond in compressed style. Rules:\n"
+        "- Drop articles (a, an, the) in prose\n"
+        "- Use sentence fragments over full sentences\n"
+        "- Use short synonyms (fix > resolve, check > investigate, big > large)\n"
+        "- Pattern: [thing] [action] [reason]. [next step].\n"
+        "- No filler, hedging, pleasantries, or trailing summaries\n"
+        "- No restating what the user said\n"
+        "- One-line explanations unless detail is asked for\n"
+        "- Code blocks, paths, commands, URLs, errors: NEVER compress\n"
+        "- Security warnings and destructive action confirmations: use full clarity"
+    ),
+    "max": (
+        "## Output Compression: Max\n"
+        "Respond in telegraphic style. Rules:\n"
+        "- Drop articles, pronouns, conjunctions where meaning survives\n"
+        "- Abbreviate common terms: DB, auth, config, fn, dep, impl, req, resp, init\n"
+        "- Use arrows for causality: → (leads to), ← (caused by)\n"
+        "- Use symbols: + (add), - (remove), ~ (change), ! (warning), ? (unclear)\n"
+        "- Max 1-2 sentences per explanation\n"
+        "- Pattern: [thing] → [action]. [reason].\n"
+        "- Code blocks, paths, commands, URLs, errors: NEVER compress\n"
+        "- Security warnings and destructive action confirmations: use full clarity"
+    ),
+}
+
+
+def get_output_rules(level: str) -> str | None:
+    """Return the output compression rules for the given level, or None if off."""
+    if level == "off":
+        return None
+    return _RULES.get(level)
+
+
+def get_level_description(level: str) -> str:
+    """Return a human-readable description of the compression level."""
+    descriptions = {
+        "off": "No output compression — Claude responds normally",
+        "lite": "Removes filler, hedging, and pleasantries. Keeps full grammar.",
+        "standard": "Drops articles, uses fragments, short synonyms. ~65% output token savings.",
+        "max": "Telegraphic style with abbreviations and symbols. ~75% output token savings.",
+    }
+    return descriptions.get(level, "Unknown level")

context_engine/compression/prompts.py

@@ -0,0 +1,9 @@
+"""Summarization prompt templates for different chunk types."""
+
+CODE_PROMPT: str = "Summarize this code. Include: function/class name, purpose, inputs/outputs, key side effects. Be concise (2-3 sentences max).\n\nCode:\n{content}"
+
+DECISION_PROMPT: str = "Summarize this decision. Include: what was decided, why, and what the outcome/action was. One paragraph max.\n\nDecision:\n{content}"
+
+ARCHITECTURE_PROMPT: str = "Summarize this component. Include: what it does, its role in the system, and its key dependencies. Be concise (2-3 sentences).\n\nComponent:\n{content}"
+
+DOC_PROMPT: str = "Summarize this documentation. Keep the key information, remove boilerplate. Be concise.\n\nDocumentation:\n{content}"

context_engine/compression/quality.py

@@ -0,0 +1,37 @@
+"""Lossy detection — verify compressed summaries preserve key identifiers."""
+import re
+
+_MIN_IDENTIFIER_RATIO = 0.4
+_MIN_IDENTIFIER_LEN = 3
+
+class QualityChecker:
+    def check(self, original: str, summary: str) -> bool:
+        identifiers = self.extract_identifiers(original)
+        if not identifiers:
+            return True
+        summary_lower = summary.lower()
+        preserved = sum(1 for ident in identifiers if ident.lower() in summary_lower)
+        ratio = preserved / len(identifiers)
+        return ratio >= _MIN_IDENTIFIER_RATIO
+
+    def extract_identifiers(self, code: str) -> list[str]:
+        patterns = [
+            r"(?:def|class|function)\s+([a-zA-Z_][a-zA-Z0-9_]*)",
+            r"([a-zA-Z_][a-zA-Z0-9_]*)\s*[=:(]",
+            r"self\.([a-zA-Z_][a-zA-Z0-9_]*)",
+        ]
+        identifiers = set()
+        for pattern in patterns:
+            for match in re.finditer(pattern, code):
+                name = match.group(1)
+                if len(name) >= _MIN_IDENTIFIER_LEN and name not in {"self", "None", "True", "False"}:
+                    identifiers.add(name)
+        # Extract function/method parameters
+        for param_match in re.finditer(r"(?:def|function)\s+\w+\s*\(([^)]*)\)", code):
+            for param in re.split(r"[,\s]+", param_match.group(1)):
+                param = param.strip().split(":")[0].split("=")[0].strip()
+                if len(param) >= _MIN_IDENTIFIER_LEN and param not in {"self", "cls", "None", "True", "False"}:
+                    identifiers.add(param)
+        for match in re.finditer(r"\b([A-Z][a-zA-Z0-9]+)\b", code):
+            identifiers.add(match.group(1))
+        return sorted(identifiers)

context_engine/config.py

@@ -0,0 +1,198 @@
+"""Configuration loading — global + per-project with defaults."""
+from dataclasses import dataclass, field
+from pathlib import Path
+
+import yaml
+
+
+_CCE_HOME = Path.home() / ".cce"
+
+DEFAULT_GLOBAL_PATH = _CCE_HOME / "config.yaml"
+PROJECT_CONFIG_NAME = ".context-engine.yaml"
+
+DEFAULT_IGNORE = [
+    # Version control
+    ".git", ".svn", ".hg",
+    # Dependencies (JS, PHP, Python, Ruby, Go, Rust, Java, .NET)
+    "node_modules", "vendor", "bower_components",
+    ".pnpm-store", ".pnpm", ".yarn",
+    ".venv", "venv", "env", ".env",
+    ".tox", ".nox", ".mypy_cache", ".pytest_cache",
+    ".ruff_cache", ".cache",
+    "Pods",  # iOS CocoaPods
+    # Build output
+    "dist", "build", "_build", "out", "target",
+    "bin", "obj",  # .NET
+    ".next", ".nuxt", ".output", ".vercel",
+    ".turbo", ".parcel-cache",
+    # IDE / editor
+    ".idea", ".vscode", ".vs",
+    # Coverage / test artifacts
+    "coverage", ".coverage", "htmlcov", ".nyc_output",
+    # OS files
+    ".DS_Store",
+    # Compiled / generated
+    "__pycache__", ".sass-cache", ".gradle",
+    # Infra
+    ".terraform", ".vagrant",
+    # Package locks (huge, not useful)
+    "package-lock.json", "yarn.lock", "pnpm-lock.yaml",
+    "composer.lock", "poetry.lock",
+    # Storage / logs
+    "storage", "logs", "tmp", "temp",
+]
+
+
+@dataclass
+class Config:
+    # Compression
+    compression_level: str = "standard"
+    compression_model: str = "phi3:mini"
+
+    # Output compression
+    output_compression: str = "standard"  # off | lite | standard | max
+
+    # Embedding
+    embedding_model: str = "BAAI/bge-small-en-v1.5"
+
+    # Retrieval
+    retrieval_confidence_threshold: float = 0.2
+    retrieval_top_k: int = 20
+    bootstrap_max_tokens: int = 10000
+
+    # Indexer
+    indexer_watch: bool = True
+    indexer_debounce_ms: int = 500
+    indexer_ignore: list[str] = field(default_factory=lambda: list(DEFAULT_IGNORE))
+    # When True, the indexer skips well-known credential filenames
+    # (.env*, *.pem, secrets.yml, credentials.json, …) and redacts
+    # AWS/GitHub/JWT/etc. patterns from the content of files it does
+    # index. See indexer/secrets.py for the full pattern list. Default
+    # True; users on non-sensitive corpora can opt out.
+    indexer_redact_secrets: bool = True
+    # When True, memory.db writes (decisions, code_areas, turn_summaries,
+    # session rollups) get PII scrubbed before storage: emails, IPs,
+    # credit cards (Luhn-validated), SSNs, phone numbers. Free-form
+    # session text often captures user data — for regulated industries
+    # this is the difference between "tool" and "compliance blocker".
+    memory_redact_pii: bool = True
+    # When True, every context_search call appends one JSON line to
+    # {storage_base}/audit.log: timestamp, query length, top_k, served
+    # chunks (file:start-end), score range, output compression level.
+    # The query text is hashed (sha256, 12-char prefix) — the log is
+    # for "what did Claude see when?" not "what did the user ask?".
+    audit_log_enabled: bool = False
+
+    # Pricing (for savings estimates)
+    pricing_model: str = "opus"
+
+    # Storage
+    storage_path: str = str(_CCE_HOME / "projects")
+
+    def detect_resource_profile(self) -> str:
+        try:
+            import psutil
+            ram_gb = psutil.virtual_memory().total / (1024 ** 3)
+        except ImportError:
+            ram_gb = 16
+        if ram_gb >= 32:
+            return "full"
+        if ram_gb >= 12:
+            return "standard"
+        return "light"
+
+
+def _deep_merge(base: dict, override: dict) -> dict:
+    result = base.copy()
+    for key, value in override.items():
+        if key in result and isinstance(result[key], dict) and isinstance(value, dict):
+            result[key] = _deep_merge(result[key], value)
+        else:
+            result[key] = value
+    return result
+
+
+_EXPECTED_TYPES: dict[str, type | tuple[type, ...]] = {
+    "compression_level": str,
+    "compression_model": str,
+    "output_compression": str,
+    "embedding_model": str,
+    "retrieval_confidence_threshold": (int, float),
+    "retrieval_top_k": int,
+    "bootstrap_max_tokens": int,
+    "indexer_watch": bool,
+    "indexer_debounce_ms": int,
+    "indexer_ignore": list,
+    "indexer_redact_secrets": bool,
+    "memory_redact_pii": bool,
+    "audit_log_enabled": bool,
+    "storage_path": str,
+    "pricing_model": str,
+}
+
+
+def _apply_dict_to_config(config: Config, data: dict) -> None:
+    mapping = {
+        ("compression", "level"): "compression_level",
+        ("compression", "model"): "compression_model",
+        ("compression", "output"): "output_compression",
+        ("embedding", "model"): "embedding_model",
+        ("retrieval", "confidence_threshold"): "retrieval_confidence_threshold",
+        ("retrieval", "top_k"): "retrieval_top_k",
+        ("retrieval", "bootstrap_max_tokens"): "bootstrap_max_tokens",
+        ("indexer", "watch"): "indexer_watch",
+        ("indexer", "debounce_ms"): "indexer_debounce_ms",
+        ("indexer", "ignore"): "indexer_ignore",
+        ("indexer", "redact_secrets"): "indexer_redact_secrets",
+        ("memory", "redact_pii"): "memory_redact_pii",
+        ("audit", "enabled"): "audit_log_enabled",
+        ("storage", "path"): "storage_path",
+        ("pricing", "model"): "pricing_model",
+    }
+    for (section, key), attr in mapping.items():
+        if section in data and isinstance(data[section], dict) and key in data[section]:
+            value = data[section][key]
+            expected = _EXPECTED_TYPES.get(attr)
+            if expected is not None and not isinstance(value, expected):
+                # `bool` is a subclass of `int`, so guard against that edge case.
+                if expected is int and isinstance(value, bool):
+                    raise ValueError(
+                        f"Config {section}.{key} must be int, got bool ({value!r})"
+                    )
+                raise ValueError(
+                    f"Config {section}.{key} must be "
+                    f"{getattr(expected, '__name__', expected)}, "
+                    f"got {type(value).__name__} ({value!r})"
+                )
+            # For ignore lists, merge with defaults instead of replacing.
+            # This way user config adds to the defaults, not overrides them.
+            if attr == "indexer_ignore" and isinstance(value, list):
+                merged = list(DEFAULT_IGNORE)
+                for item in value:
+                    if item not in merged:
+                        merged.append(item)
+                setattr(config, attr, merged)
+            else:
+                setattr(config, attr, value)
+
+
+def load_config(
+    global_path: Path | None = None,
+    project_path: Path | None = None,
+) -> Config:
+    global_path = global_path or DEFAULT_GLOBAL_PATH
+    config = Config()
+
+    global_data: dict = {}
+    if global_path.exists():
+        with open(global_path) as f:
+            global_data = yaml.safe_load(f) or {}
+
+    project_data: dict = {}
+    if project_path and project_path.exists():
+        with open(project_path) as f:
+            project_data = yaml.safe_load(f) or {}
+
+    merged = _deep_merge(global_data, project_data)
+    _apply_dict_to_config(config, merged)
+    return config