code-finder 0.1.0__py3-none-any.whl

claude_context/embeddings.py

@@ -0,0 +1,307 @@
+"""
+Local embeddings for Claude Context - No API keys needed
+
+This module provides local text embeddings using sentence-transformers,
+following our fail-fast principles with clear error reporting.
+"""
+
+import logging
+from typing import List, Optional, Union
+import numpy as np
+
+# Import sentence-transformers - fail fast if not installed
+try:
+    from sentence_transformers import SentenceTransformer
+except ImportError as e:
+    raise ImportError(
+        "sentence-transformers is required for embeddings. "
+        "Install with: pip install sentence-transformers"
+    ) from e
+
+logger = logging.getLogger(__name__)
+
+
+class LocalEmbeddings:
+    """
+    Local embeddings using sentence-transformers.
+    
+    No API keys needed, runs entirely on your machine.
+    Fail-fast approach: errors are explicit, not hidden.
+    """
+    
+    # Recommended models with trade-offs clearly documented
+    MODELS = {
+        "all-MiniLM-L6-v2": {
+            "dimension": 384,
+            "description": "Fast and good quality, best for development",
+            "size_mb": 80,
+            "speed": "fast"
+        },
+        "all-mpnet-base-v2": {
+            "dimension": 768,
+            "description": "Higher quality, slower, good for production",
+            "size_mb": 420,
+            "speed": "medium"
+        },
+        "all-distilroberta-v1": {
+            "dimension": 768,
+            "description": "Good balance of speed and quality",
+            "size_mb": 290,
+            "speed": "medium"
+        }
+    }
+    
+    def __init__(self, model_name: str = "all-MiniLM-L6-v2"):
+        """
+        Initialize the embedding model.
+        
+        Args:
+            model_name: Name of the sentence-transformers model to use
+            
+        Raises:
+            ValueError: If model_name is empty
+            RuntimeError: If model fails to load
+        """
+        if not model_name:
+            raise ValueError("model_name cannot be empty")
+        
+        self.model_name = model_name
+        logger.info(f"Loading embedding model: {self.model_name}")
+        
+        # Get model info if available
+        model_info = self.MODELS.get(self.model_name, {})
+        if model_info:
+            logger.info(f"  Description: {model_info.get('description')}")
+            logger.info(f"  Dimension: {model_info.get('dimension')}")
+            logger.info(f"  Speed: {model_info.get('speed')}")
+        
+        # Load model - fail fast if it doesn't work
+        try:
+            self.model = SentenceTransformer(model_name)
+            self.dimension = self.model.get_sentence_embedding_dimension()
+            logger.info(f"✅ Model loaded successfully (dimension: {self.dimension})")
+        except Exception as e:
+            # Don't hide the error - make it clear what failed
+            raise RuntimeError(
+                f"Failed to load embedding model '{model_name}': {e}\n"
+                f"Try one of the tested models: {list(self.MODELS.keys())}"
+            ) from e
+    
+    def embed_texts(self, texts: List[str], batch_size: int = 32) -> np.ndarray:
+        """
+        Embed a list of texts into vectors.
+        
+        Args:
+            texts: List of texts to embed
+            batch_size: Batch size for processing (for memory efficiency)
+            
+        Returns:
+            numpy array of embeddings, shape (n_texts, dimension)
+            
+        Raises:
+            ValueError: If texts is empty or contains only empty strings
+            RuntimeError: If embedding fails
+        """
+        # Validate input - fail fast on bad data
+        if not texts:
+            raise ValueError("Cannot embed empty text list")
+        
+        if not any(texts):  # All texts are empty
+            raise ValueError("Cannot embed list of empty strings")
+        
+        # Log what we're doing
+        logger.info(f"Embedding {len(texts)} texts (batch_size={batch_size})")
+        
+        try:
+            # Process embeddings
+            embeddings = self.model.encode(
+                texts,
+                batch_size=batch_size,
+                show_progress_bar=len(texts) > 100,  # Show progress for large batches
+                convert_to_numpy=True
+            )
+            
+            # Validate output
+            if embeddings.shape[0] != len(texts):
+                raise RuntimeError(
+                    f"Embedding count mismatch: expected {len(texts)}, got {embeddings.shape[0]}"
+                )
+            
+            logger.debug(f"Generated embeddings shape: {embeddings.shape}")
+            return embeddings
+            
+        except Exception as e:
+            # Don't hide errors - be explicit about what failed
+            raise RuntimeError(f"Failed to embed texts: {e}") from e
+    
+    def embed_text(self, text: str) -> np.ndarray:
+        """
+        Embed a single text into a vector.
+        
+        Args:
+            text: Text to embed
+            
+        Returns:
+            numpy array of embedding, shape (dimension,)
+            
+        Raises:
+            ValueError: If text is empty
+            RuntimeError: If embedding fails
+        """
+        if not text or not text.strip():
+            raise ValueError("Cannot embed empty text")
+        
+        logger.debug(f"Embedding single text of length {len(text)}")
+        
+        # Use embed_texts for consistency
+        embeddings = self.embed_texts([text], batch_size=1)
+        return embeddings[0]
+    
+    def embed_query(self, query: str) -> np.ndarray:
+        """
+        Embed a search query.
+        
+        This is an alias for embed_text but semantically indicates
+        this is for a search query rather than document text.
+        
+        Args:
+            query: Search query to embed
+            
+        Returns:
+            numpy array of embedding, shape (dimension,)
+            
+        Raises:
+            ValueError: If query is empty
+            RuntimeError: If embedding fails
+        """
+        if not query or not query.strip():
+            raise ValueError("Cannot embed empty query")
+        
+        logger.debug(f"Embedding query: '{query[:50]}...'")
+        return self.embed_text(query)
+    
+    def get_dimension(self) -> int:
+        """
+        Get the dimension of the embeddings.
+        
+        Returns:
+            Embedding dimension
+        """
+        return self.dimension
+    
+    def get_model_info(self) -> dict:
+        """
+        Get information about the current model.
+        
+        Returns:
+            Dictionary with model information
+        """
+        info = {
+            "model_name": self.model_name,
+            "dimension": self.dimension,
+            "type": "local",
+            "requires_api_key": False
+        }
+        
+        # Add known model info if available
+        if self.model_name in self.MODELS:
+            info.update(self.MODELS[self.model_name])
+        
+        return info
+    
+    def __repr__(self) -> str:
+        """String representation"""
+        return f"LocalEmbeddings(model='{self.model_name}', dim={self.dimension})"
+
+
+# Convenience function for quick setup
+def create_embeddings(model_name: Optional[str] = None) -> LocalEmbeddings:
+    """
+    Create an embedding instance with default or specified model.
+    
+    Args:
+        model_name: Optional model name, defaults to fast model
+        
+    Returns:
+        LocalEmbeddings instance
+        
+    Raises:
+        RuntimeError: If embeddings cannot be created
+    """
+    model = model_name or "all-MiniLM-L6-v2"
+    logger.info(f"Creating embeddings with model: {model}")
+    return LocalEmbeddings(model)
+
+
+# Example usage and testing
+if __name__ == "__main__":
+    import time
+    
+    print("Testing Local Embeddings Module...")
+    print("-" * 50)
+    
+    # Test 1: Create embeddings
+    print("\n1. Creating embedding model...")
+    embeddings = LocalEmbeddings()
+    print(f"   ✅ {embeddings}")
+    
+    # Test 2: Embed single text
+    print("\n2. Testing single text embedding...")
+    text = "This is a test sentence for embedding."
+    embedding = embeddings.embed_text(text)
+    print(f"   ✅ Embedded text to shape: {embedding.shape}")
+    print(f"   Sample values: {embedding[:5]}")
+    
+    # Test 3: Embed multiple texts
+    print("\n3. Testing batch embedding...")
+    texts = [
+        "Python is a programming language",
+        "JavaScript runs in the browser",
+        "Rust is memory safe",
+        "Go has great concurrency"
+    ]
+    start = time.time()
+    batch_embeddings = embeddings.embed_texts(texts)
+    elapsed = time.time() - start
+    print(f"   ✅ Embedded {len(texts)} texts in {elapsed:.3f}s")
+    print(f"   Output shape: {batch_embeddings.shape}")
+    
+    # Test 4: Embed query (semantic similarity)
+    print("\n4. Testing query embedding...")
+    query = "What programming language is best for web development?"
+    query_embedding = embeddings.embed_query(query)
+    
+    # Calculate similarities
+    from numpy.linalg import norm
+    similarities = []
+    for i, text in enumerate(texts):
+        # Cosine similarity
+        similarity = np.dot(query_embedding, batch_embeddings[i]) / (
+            norm(query_embedding) * norm(batch_embeddings[i])
+        )
+        similarities.append((text, similarity))
+    
+    # Sort by similarity
+    similarities.sort(key=lambda x: x[1], reverse=True)
+    print(f"   Query: '{query}'")
+    print("   Most similar texts:")
+    for text, sim in similarities[:2]:
+        print(f"     - {text}: {sim:.3f}")
+    
+    # Test 5: Error handling
+    print("\n5. Testing error handling...")
+    try:
+        embeddings.embed_text("")
+        print("   ❌ Should have raised error for empty text")
+    except ValueError as e:
+        print(f"   ✅ Correctly raised error: {e}")
+    
+    # Test 6: Model info
+    print("\n6. Model information:")
+    info = embeddings.get_model_info()
+    for key, value in info.items():
+        print(f"   - {key}: {value}")
+    
+    print("\n" + "=" * 50)
+    print("🎉 All tests passed! Embeddings module ready.")
+    print("=" * 50)

claude_context/embeddings_interface.py

@@ -0,0 +1,226 @@
+"""
+Embeddings Interface - Easy to swap between different providers
+
+This shows how we can easily switch between local and API-based embeddings
+in the future without changing the rest of our code.
+"""
+
+from abc import ABC, abstractmethod
+from typing import List, Optional
+import numpy as np
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class EmbeddingsInterface(ABC):
+    """Abstract base class for all embedding providers"""
+    
+    @abstractmethod
+    def embed_texts(self, texts: List[str]) -> np.ndarray:
+        """Embed multiple texts"""
+        pass
+    
+    @abstractmethod
+    def embed_text(self, text: str) -> np.ndarray:
+        """Embed single text"""
+        pass
+    
+    @abstractmethod
+    def get_dimension(self) -> int:
+        """Get embedding dimension"""
+        pass
+
+
+class LocalEmbeddings(EmbeddingsInterface):
+    """Local embeddings (what we have now)"""
+    
+    def __init__(self, model_name: str = "all-MiniLM-L6-v2"):
+        from sentence_transformers import SentenceTransformer
+        self.model = SentenceTransformer(model_name)
+        self.dimension = self.model.get_sentence_embedding_dimension()
+    
+    def embed_texts(self, texts: List[str]) -> np.ndarray:
+        return self.model.encode(texts, convert_to_numpy=True)
+    
+    def embed_text(self, text: str) -> np.ndarray:
+        return self.embed_texts([text])[0]
+    
+    def get_dimension(self) -> int:
+        return self.dimension
+
+
+class OpenAIEmbeddings(EmbeddingsInterface):
+    """OpenAI embeddings (future option)"""
+    
+    def __init__(self, api_key: str, model: str = "text-embedding-3-small"):
+        import openai
+        self.client = openai.OpenAI(api_key=api_key)
+        self.model = model
+        self.dimension = {
+            "text-embedding-3-small": 1536,
+            "text-embedding-3-large": 3072,
+            "text-embedding-ada-002": 1536,
+        }.get(model, 1536)
+    
+    def embed_texts(self, texts: List[str]) -> np.ndarray:
+        response = self.client.embeddings.create(
+            input=texts,
+            model=self.model
+        )
+        return np.array([e.embedding for e in response.data])
+    
+    def embed_text(self, text: str) -> np.ndarray:
+        return self.embed_texts([text])[0]
+    
+    def get_dimension(self) -> int:
+        return self.dimension
+
+
+class VoyageAIEmbeddings(EmbeddingsInterface):
+    """Voyage AI embeddings - optimized for code (future option)"""
+    
+    def __init__(self, api_key: str, model: str = "voyage-code-2"):
+        import voyageai
+        self.client = voyageai.Client(api_key=api_key)
+        self.model = model
+        self.dimension = 1536  # voyage-code-2 dimension
+    
+    def embed_texts(self, texts: List[str]) -> np.ndarray:
+        result = self.client.embed(texts, model=self.model)
+        return np.array(result.embeddings)
+    
+    def embed_text(self, text: str) -> np.ndarray:
+        return self.embed_texts([text])[0]
+    
+    def get_dimension(self) -> int:
+        return self.dimension
+
+
+def create_embeddings(provider: str = "local", **kwargs) -> EmbeddingsInterface:
+    """
+    Factory function to create embeddings based on provider.
+    
+    Args:
+        provider: One of "local", "openai", "voyage"
+        **kwargs: Provider-specific arguments
+        
+    Returns:
+        EmbeddingsInterface implementation
+        
+    Examples:
+        # Development
+        embeddings = create_embeddings("local")
+        
+        # Production with OpenAI
+        embeddings = create_embeddings("openai", api_key="sk-...")
+        
+        # Production with code-optimized embeddings
+        embeddings = create_embeddings("voyage", api_key="...", model="voyage-code-2")
+    """
+    if provider == "local":
+        model = kwargs.get("model", "all-MiniLM-L6-v2")
+        logger.info(f"Creating local embeddings: {model}")
+        return LocalEmbeddings(model)
+    
+    elif provider == "openai":
+        api_key = kwargs.get("api_key")
+        if not api_key:
+            raise ValueError("OpenAI requires api_key")
+        model = kwargs.get("model", "text-embedding-3-small")
+        logger.info(f"Creating OpenAI embeddings: {model}")
+        return OpenAIEmbeddings(api_key, model)
+    
+    elif provider == "voyage":
+        api_key = kwargs.get("api_key")
+        if not api_key:
+            raise ValueError("Voyage requires api_key")
+        model = kwargs.get("model", "voyage-code-2")
+        logger.info(f"Creating Voyage AI embeddings: {model}")
+        return VoyageAIEmbeddings(api_key, model)
+    
+    else:
+        raise ValueError(f"Unknown provider: {provider}")
+
+
+# Configuration-based selection
+class EmbeddingsConfig:
+    """Configuration for embeddings selection"""
+    
+    @staticmethod
+    def from_environment() -> EmbeddingsInterface:
+        """
+        Create embeddings based on environment variables.
+        
+        This makes it easy to switch providers without code changes:
+        
+        Development:
+            EMBEDDINGS_PROVIDER=local
+            
+        Production:
+            EMBEDDINGS_PROVIDER=openai
+            OPENAI_API_KEY=sk-...
+            
+        Code-optimized:
+            EMBEDDINGS_PROVIDER=voyage
+            VOYAGE_API_KEY=...
+        """
+        import os
+        
+        provider = os.getenv("EMBEDDINGS_PROVIDER", "local")
+        
+        if provider == "local":
+            model = os.getenv("EMBEDDINGS_MODEL", "all-MiniLM-L6-v2")
+            return create_embeddings("local", model=model)
+        
+        elif provider == "openai":
+            api_key = os.getenv("OPENAI_API_KEY")
+            if not api_key:
+                raise ValueError("OPENAI_API_KEY environment variable required")
+            model = os.getenv("EMBEDDINGS_MODEL", "text-embedding-3-small")
+            return create_embeddings("openai", api_key=api_key, model=model)
+        
+        elif provider == "voyage":
+            api_key = os.getenv("VOYAGE_API_KEY")
+            if not api_key:
+                raise ValueError("VOYAGE_API_KEY environment variable required")
+            model = os.getenv("EMBEDDINGS_MODEL", "voyage-code-2")
+            return create_embeddings("voyage", api_key=api_key, model=model)
+        
+        else:
+            raise ValueError(f"Unknown EMBEDDINGS_PROVIDER: {provider}")
+
+
+if __name__ == "__main__":
+    print("Embeddings Interface Example")
+    print("-" * 50)
+    
+    # Show how easy it is to switch
+    print("\n1. Local embeddings (current):")
+    embeddings = create_embeddings("local")
+    print(f"   Provider: Local")
+    print(f"   Dimension: {embeddings.get_dimension()}")
+    print(f"   Cost: Free")
+    print(f"   Speed: Fast")
+    print(f"   Quality: Good")
+    
+    print("\n2. Future upgrade path:")
+    print("   OpenAI text-embedding-3-large:")
+    print("   - Dimension: 3072")
+    print("   - Cost: ~$0.13 per 1M tokens")
+    print("   - Speed: API latency")
+    print("   - Quality: Excellent")
+    
+    print("\n   Voyage AI voyage-code-2:")
+    print("   - Dimension: 1536")
+    print("   - Cost: ~$0.10 per 1M tokens")
+    print("   - Speed: API latency")
+    print("   - Quality: Excellent for code")
+    
+    print("\n3. Deployment flexibility:")
+    print("   - Development: Local model on Mac (MPS)")
+    print("   - Staging: Local model on Linux (CPU/GPU)")
+    print("   - Production: API-based for best quality")
+    print("   - Enterprise: Self-hosted models on GPU cluster")
+    
+    print("\n✅ Same interface, different providers!")