codexa 0.4.0__py3-none-any.whl

semantic_code_intelligence/embeddings/enhanced.py

@@ -0,0 +1,131 @@
+"""Enhanced embedding pipeline — semantic preprocessing for code vectors.
+
+Wraps the base generator with code-aware preprocessing:
+- Prepends semantic labels to improve embedding quality
+- Normalizes code formatting for more consistent representations
+- Supports batch processing with progress tracking
+"""
+
+from __future__ import annotations
+
+import re
+from typing import TYPE_CHECKING, Any
+
+import numpy as np
+
+from semantic_code_intelligence.embeddings.generator import (
+    generate_embeddings,
+    get_embedding_dimension,
+    get_model,
+)
+from semantic_code_intelligence.indexing.semantic_chunker import SemanticChunk
+from semantic_code_intelligence.utils.logging import get_logger
+
+if TYPE_CHECKING:
+    pass
+
+logger = get_logger("embeddings.enhanced")
+
+
+def preprocess_code_for_embedding(content: str, semantic_label: str = "") -> str:
+    """Preprocess a code string to improve embedding quality.
+
+    Transformations:
+    1. Prepend semantic label (e.g. "[python] function authenticate(user, password)")
+    2. Collapse excessive blank lines
+    3. Strip trailing whitespace per line
+    4. Normalize indentation depth (reduce deep nesting visual noise)
+
+    Args:
+        content: Raw code string.
+        semantic_label: Optional semantic prefix.
+
+    Returns:
+        Preprocessed text ready for embedding.
+    """
+    lines = content.splitlines()
+    processed: list[str] = []
+
+    blank_count = 0
+    for line in lines:
+        stripped = line.rstrip()
+        if not stripped:
+            blank_count += 1
+            if blank_count <= 1:
+                processed.append("")
+            continue
+        blank_count = 0
+        processed.append(stripped)
+
+    text = "\n".join(processed).strip()
+
+    if semantic_label:
+        text = f"{semantic_label}\n{text}"
+
+    return text
+
+
+def prepare_semantic_texts(chunks: list[SemanticChunk]) -> list[str]:
+    """Convert semantic chunks into preprocessed text strings for embedding.
+
+    Each chunk's content is enhanced with its semantic label to give
+    the embedding model structural context about what it's encoding.
+
+    Args:
+        chunks: List of SemanticChunk objects.
+
+    Returns:
+        List of preprocessed text strings, one per chunk.
+    """
+    return [
+        preprocess_code_for_embedding(c.content, c.semantic_label)
+        for c in chunks
+    ]
+
+
+def generate_semantic_embeddings(
+    chunks: list[SemanticChunk],
+    model_name: str = "all-MiniLM-L6-v2",
+    batch_size: int = 64,
+    show_progress: bool = False,
+) -> np.ndarray:
+    """Generate embeddings from semantic chunks with preprocessing.
+
+    This is the recommended entry point for the enhanced pipeline.
+    It preprocesses each chunk with its semantic label before encoding.
+
+    Args:
+        chunks: List of SemanticChunk objects.
+        model_name: Sentence-transformers model name.
+        batch_size: Encoding batch size.
+        show_progress: Show progress bar.
+
+    Returns:
+        NumPy array of shape (len(chunks), embedding_dim), L2-normalized.
+    """
+    if not chunks:
+        return np.array([], dtype=np.float32).reshape(0, 0)
+
+    texts = prepare_semantic_texts(chunks)
+    return generate_embeddings(texts, model_name, batch_size, show_progress)
+
+
+def generate_query_embedding(
+    query: str,
+    model_name: str = "all-MiniLM-L6-v2",
+) -> np.ndarray:
+    """Generate embedding for a search query with light preprocessing.
+
+    Queries are treated differently from code: they are natural language,
+    so we do minimal transformation.
+
+    Args:
+        query: Natural language search query.
+        model_name: Sentence-transformers model name.
+
+    Returns:
+        NumPy array of shape (1, embedding_dim), L2-normalized.
+    """
+    # Light cleanup only
+    clean_query = query.strip()
+    return generate_embeddings([clean_query], model_name)

semantic_code_intelligence/embeddings/generator.py

@@ -0,0 +1,149 @@
+"""Embedding generator — converts code chunks into vector embeddings.
+
+Supports two backends:
+- **sentence-transformers** (default): PyTorch-based, full-featured.
+- **onnx**: Lightweight ONNX Runtime backend via ``optimum`` — lower
+  memory (~50% less) and often faster inference on CPU.
+"""
+
+from __future__ import annotations
+
+import os
+from typing import TYPE_CHECKING
+
+import numpy as np
+
+from semantic_code_intelligence.embeddings.model_registry import resolve_model_name
+from semantic_code_intelligence.utils.logging import get_logger
+
+if TYPE_CHECKING:
+    from sentence_transformers import SentenceTransformer
+
+logger = get_logger("embeddings")
+
+# Module-level cache for loaded model instances
+_model_cache: dict[str, "SentenceTransformer"] = {}
+
+
+def _configure_hf_token() -> None:
+    """Set HF_TOKEN from common env vars if not already set.
+
+    Checks ``HF_TOKEN``, ``HUGGING_FACE_HUB_TOKEN``, and
+    ``HUGGINGFACE_TOKEN`` so the user only needs to export one.
+    """
+    if os.environ.get("HF_TOKEN"):
+        return
+    for var in ("HUGGING_FACE_HUB_TOKEN", "HUGGINGFACE_TOKEN"):
+        value = os.environ.get(var)
+        if value:
+            os.environ["HF_TOKEN"] = value
+            return
+
+
+def _onnx_available() -> bool:
+    """Check if the ONNX Runtime backend is available."""
+    try:
+        import optimum  # noqa: F401
+        import onnxruntime  # noqa: F401
+        return True
+    except ImportError:
+        return False
+
+
+def get_model(
+    model_name: str = "all-MiniLM-L6-v2",
+    backend: str = "auto",
+) -> "SentenceTransformer":
+    """Load and cache a sentence-transformers model.
+
+    Args:
+        model_name: Name of the model to load (full HF name or alias).
+        backend: ``"auto"`` (ONNX if available, else PyTorch),
+                 ``"onnx"``, or ``"torch"``.
+
+    Returns:
+        A SentenceTransformer model instance.
+    """
+    model_name = resolve_model_name(model_name)
+    cache_key = f"{model_name}:{backend}"
+
+    if cache_key not in _model_cache:
+        _configure_hf_token()
+        from sentence_transformers import SentenceTransformer
+
+        use_onnx = False
+        if backend == "onnx":
+            if _onnx_available():
+                use_onnx = True
+            else:
+                logger.warning("ONNX requested but optimum/onnxruntime not installed; falling back to PyTorch.")
+        elif backend == "auto" and _onnx_available():
+            use_onnx = True
+
+        logger.info("Loading embedding model: %s (backend=%s)", model_name, "onnx" if use_onnx else "torch")
+
+        if use_onnx:
+            try:
+                _model_cache[cache_key] = SentenceTransformer(model_name, backend="onnx")
+                logger.info("Model loaded with ONNX backend.")
+                return _model_cache[cache_key]
+            except Exception:
+                logger.warning("ONNX load failed; falling back to PyTorch.")
+
+        _model_cache[cache_key] = SentenceTransformer(model_name)
+        logger.info("Model loaded successfully (PyTorch).")
+
+    return _model_cache[cache_key]
+
+
+def generate_embeddings(
+    texts: list[str],
+    model_name: str = "all-MiniLM-L6-v2",
+    batch_size: int = 64,
+    show_progress: bool = False,
+    backend: str = "auto",
+) -> np.ndarray:
+    """Generate vector embeddings for a list of text strings.
+
+    Args:
+        texts: List of code/text strings to embed.
+        model_name: Name of the sentence-transformers model (or alias).
+        batch_size: Batch size for encoding.
+        show_progress: Whether to show a progress bar.
+        backend: ``"auto"``, ``"onnx"``, or ``"torch"``.
+
+    Returns:
+        NumPy array of shape (len(texts), embedding_dim).
+    """
+    if not texts:
+        return np.array([], dtype=np.float32).reshape(0, 0)
+
+    model = get_model(model_name, backend=backend)
+    embeddings = model.encode(
+        texts,
+        batch_size=batch_size,
+        show_progress_bar=show_progress,
+        convert_to_numpy=True,
+        normalize_embeddings=True,
+    )
+    return embeddings.astype(np.float32)
+
+
+def get_embedding_dimension(
+    model_name: str = "all-MiniLM-L6-v2",
+    backend: str = "auto",
+) -> int:
+    """Return the dimensionality of embeddings produced by the given model.
+
+    Args:
+        model_name: Name of the sentence-transformers model (or alias).
+        backend: ``"auto"``, ``"onnx"``, or ``"torch"``.
+
+    Returns:
+        Integer dimension of the embedding vectors.
+    """
+    model = get_model(model_name, backend=backend)
+    dim = model.get_sentence_embedding_dimension()
+    if dim is None:
+        raise RuntimeError(f"Model {model_name!r} returned None for embedding dimension")
+    return dim

semantic_code_intelligence/embeddings/model_registry.py

@@ -0,0 +1,100 @@
+"""Embedding model registry — defines available models and their properties.
+
+Provides a catalogue of supported embedding models so users can switch
+between models optimised for different use-cases (code-heavy, doc-heavy,
+speed vs quality, etc.).
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class ModelInfo:
+    """Metadata about a supported embedding model."""
+
+    name: str
+    display_name: str
+    dimension: int
+    description: str
+    recommended_for: str
+    backend: str = "sentence-transformers"  # or "onnx"
+
+
+# Built-in model catalogue — mirrors ck's 4 models plus extras
+AVAILABLE_MODELS: dict[str, ModelInfo] = {
+    "all-MiniLM-L6-v2": ModelInfo(
+        name="all-MiniLM-L6-v2",
+        display_name="MiniLM L6 v2",
+        dimension=384,
+        description="Default balanced model — good quality, fast inference.",
+        recommended_for="General purpose, balanced speed/quality.",
+        backend="sentence-transformers",
+    ),
+    "BAAI/bge-small-en-v1.5": ModelInfo(
+        name="BAAI/bge-small-en-v1.5",
+        display_name="BGE Small EN v1.5",
+        dimension=384,
+        description="Compact BGE model — strong text retrieval performance.",
+        recommended_for="Retrieval-heavy workloads, lower memory.",
+        backend="sentence-transformers",
+    ),
+    "nomic-ai/nomic-embed-text-v1.5": ModelInfo(
+        name="nomic-ai/nomic-embed-text-v1.5",
+        display_name="Nomic Embed Text v1.5",
+        dimension=768,
+        description="High-quality long-context model (8192 tokens).",
+        recommended_for="Documentation-heavy repos, long files.",
+        backend="sentence-transformers",
+    ),
+    "jinaai/jina-embeddings-v2-base-code": ModelInfo(
+        name="jinaai/jina-embeddings-v2-base-code",
+        display_name="Jina Code v2",
+        dimension=768,
+        description="Code-specialised model trained on programming languages.",
+        recommended_for="Code-heavy repos, programming-specific search.",
+        backend="sentence-transformers",
+    ),
+    "mixedbread-ai/mxbai-embed-xsmall-v1": ModelInfo(
+        name="mixedbread-ai/mxbai-embed-xsmall-v1",
+        display_name="Mixedbread XSmall v1",
+        dimension=384,
+        description="Ultra-compact model — fastest inference, smallest footprint.",
+        recommended_for="Large repos where speed matters most.",
+        backend="sentence-transformers",
+    ),
+}
+
+# Shorthand aliases for CLI convenience
+MODEL_ALIASES: dict[str, str] = {
+    "minilm": "all-MiniLM-L6-v2",
+    "bge-small": "BAAI/bge-small-en-v1.5",
+    "nomic": "nomic-ai/nomic-embed-text-v1.5",
+    "jina-code": "jinaai/jina-embeddings-v2-base-code",
+    "mxbai-xsmall": "mixedbread-ai/mxbai-embed-xsmall-v1",
+}
+
+DEFAULT_MODEL = "all-MiniLM-L6-v2"
+
+
+def resolve_model_name(name_or_alias: str) -> str:
+    """Resolve a model name or alias to the full model identifier."""
+    if name_or_alias in AVAILABLE_MODELS:
+        return name_or_alias
+    resolved = MODEL_ALIASES.get(name_or_alias.lower())
+    if resolved:
+        return resolved
+    # Assume it's a custom HF model name
+    return name_or_alias
+
+
+def get_model_info(name_or_alias: str) -> ModelInfo | None:
+    """Look up model info by name or alias. Returns None for unknown models."""
+    resolved = resolve_model_name(name_or_alias)
+    return AVAILABLE_MODELS.get(resolved)
+
+
+def list_models() -> list[ModelInfo]:
+    """Return all available models."""
+    return list(AVAILABLE_MODELS.values())

semantic_code_intelligence/evolution/__init__.py

@@ -0,0 +1 @@
+"""Self-improving development loop — LLM-driven incremental code evolution."""

semantic_code_intelligence/evolution/budget_guard.py

@@ -0,0 +1,111 @@
+"""Budget guard — tracks token usage, iterations, and wall-clock time.
+
+Enforces hard limits so that the evolution loop cannot run away with
+unbounded LLM calls.  The guard is passed through every stage and
+checked before each LLM invocation.
+"""
+
+from __future__ import annotations
+
+import time
+from dataclasses import dataclass, field
+
+
+@dataclass
+class BudgetGuard:
+    """Resource budget tracker for the evolution loop.
+
+    Parameters
+    ----------
+    max_tokens : int
+        Maximum total tokens (prompt + completion) across all LLM calls.
+    max_iterations : int
+        Maximum evolution iterations to attempt.
+    max_seconds : float
+        Maximum wall-clock seconds before the loop is force-stopped.
+    """
+
+    max_tokens: int = 20_000
+    max_iterations: int = 5
+    max_seconds: float = 600.0  # 10 minutes default
+
+    # Counters
+    tokens_used: int = 0
+    iterations_done: int = 0
+    _start_time: float = field(default=0.0, repr=False)
+
+    # ------------------------------------------------------------------ #
+    # Lifecycle
+    # ------------------------------------------------------------------ #
+
+    def start(self) -> None:
+        """Mark the beginning of the evolution run."""
+        self._start_time = time.time()
+
+    @property
+    def elapsed_seconds(self) -> float:
+        """Wall-clock seconds since :meth:`start` was called."""
+        if self._start_time == 0.0:
+            return 0.0
+        return time.time() - self._start_time
+
+    # ------------------------------------------------------------------ #
+    # Checks
+    # ------------------------------------------------------------------ #
+
+    @property
+    def tokens_remaining(self) -> int:
+        """Tokens still available before the budget is exhausted."""
+        return max(0, self.max_tokens - self.tokens_used)
+
+    @property
+    def iterations_remaining(self) -> int:
+        """Iterations still available before the limit is hit."""
+        return max(0, self.max_iterations - self.iterations_done)
+
+    def can_continue(self) -> bool:
+        """Return ``True`` if budget allows another iteration."""
+        if self.iterations_done >= self.max_iterations:
+            return False
+        if self.tokens_used >= self.max_tokens:
+            return False
+        if self._start_time > 0.0 and self.elapsed_seconds >= self.max_seconds:
+            return False
+        return True
+
+    def stop_reason(self) -> str | None:
+        """Return a human-readable reason if budget is exhausted, else ``None``."""
+        if self.iterations_done >= self.max_iterations:
+            return f"iteration limit reached ({self.max_iterations})"
+        if self.tokens_used >= self.max_tokens:
+            return f"token budget exhausted ({self.tokens_used}/{self.max_tokens})"
+        if self._start_time > 0.0 and self.elapsed_seconds >= self.max_seconds:
+            return f"time limit reached ({self.elapsed_seconds:.0f}s/{self.max_seconds:.0f}s)"
+        return None
+
+    # ------------------------------------------------------------------ #
+    # Recording
+    # ------------------------------------------------------------------ #
+
+    def record_tokens(self, tokens: int) -> None:
+        """Record token usage from an LLM call."""
+        self.tokens_used += tokens
+
+    def record_iteration(self) -> None:
+        """Mark one iteration as completed."""
+        self.iterations_done += 1
+
+    # ------------------------------------------------------------------ #
+    # Summary
+    # ------------------------------------------------------------------ #
+
+    def summary(self) -> dict[str, object]:
+        """Return a dict snapshot of current budget usage."""
+        return {
+            "tokens_used": self.tokens_used,
+            "tokens_max": self.max_tokens,
+            "iterations_done": self.iterations_done,
+            "iterations_max": self.max_iterations,
+            "elapsed_seconds": round(self.elapsed_seconds, 2),
+            "max_seconds": self.max_seconds,
+        }

semantic_code_intelligence/evolution/commit_manager.py

@@ -0,0 +1,88 @@
+"""Commit manager — handles git add / commit / revert for evolution patches.
+
+All git operations run as subprocesses against the project root.
+The manager only commits files that the evolution loop explicitly touched.
+"""
+
+from __future__ import annotations
+
+import subprocess
+from pathlib import Path
+
+from semantic_code_intelligence.utils.logging import get_logger
+
+logger = get_logger("evolution.commit_manager")
+
+
+class CommitManager:
+    """Thin wrapper around git for safe commit/revert cycles."""
+
+    def __init__(self, project_root: Path) -> None:
+        self._root = project_root.resolve()
+
+    # ------------------------------------------------------------------ #
+    # Queries
+    # ------------------------------------------------------------------ #
+
+    def git_diff(self, staged: bool = False) -> str:
+        """Return the current ``git diff`` output (unstaged by default)."""
+        cmd = ["git", "diff"]
+        if staged:
+            cmd.append("--staged")
+        return self._run(cmd)
+
+    def git_diff_stat(self) -> str:
+        """Return the ``--stat`` summary of uncommitted changes."""
+        return self._run(["git", "diff", "--stat"])
+
+    def has_changes(self) -> bool:
+        """Return ``True`` if there are uncommitted changes."""
+        output = self._run(["git", "status", "--porcelain"])
+        return bool(output.strip())
+
+    # ------------------------------------------------------------------ #
+    # Mutations
+    # ------------------------------------------------------------------ #
+
+    def stage_files(self, paths: list[str]) -> None:
+        """``git add`` a list of relative file paths."""
+        if not paths:
+            return
+        self._run(["git", "add", "--"] + paths)
+
+    def commit(self, message: str) -> str:
+        """Create a commit with the given message.  Returns the short SHA."""
+        self._run(["git", "commit", "-m", message])
+        sha = self._run(["git", "rev-parse", "--short", "HEAD"]).strip()
+        logger.info("Committed %s: %s", sha, message)
+        return sha
+
+    def revert_files(self, paths: list[str]) -> None:
+        """Restore files to their last committed state."""
+        if not paths:
+            return
+        self._run(["git", "checkout", "--"] + paths)
+        logger.info("Reverted %d file(s).", len(paths))
+
+    def stash_push(self, message: str = "evolution-wip") -> None:
+        """Stash current changes."""
+        self._run(["git", "stash", "push", "-m", message])
+
+    def stash_pop(self) -> None:
+        """Pop the most recent stash."""
+        self._run(["git", "stash", "pop"])
+
+    # ------------------------------------------------------------------ #
+    # Internal
+    # ------------------------------------------------------------------ #
+
+    def _run(self, cmd: list[str]) -> str:
+        proc = subprocess.run(
+            cmd,
+            capture_output=True,
+            text=True,
+            cwd=str(self._root),
+        )
+        if proc.returncode != 0 and "nothing to commit" not in proc.stdout:
+            logger.debug("git stderr: %s", proc.stderr.strip())
+        return proc.stdout