claude-mpm 5.4.85__py3-none-any.whl → 5.6.76__py3-none-any.whl

claude_mpm/commander/memory/embeddings.py

@@ -0,0 +1,230 @@
+"""Embedding service for semantic search.
+
+Generates vector embeddings using sentence-transformers (local) or
+OpenAI API (cloud). Defaults to local model for zero-cost operation.
+"""
+
+import asyncio
+import logging
+from typing import List, Literal, Optional
+
+logger = logging.getLogger(__name__)
+
+EmbeddingProvider = Literal["sentence-transformers", "openai"]
+
+
+class EmbeddingService:
+    """Generate vector embeddings for semantic search.
+
+    Supports multiple providers:
+    - sentence-transformers: Local, free, good quality (default)
+    - openai: Cloud API, best quality, costs money
+
+    Attributes:
+        provider: Embedding provider to use
+        model: Model name for the provider
+        dimension: Embedding vector dimension
+
+    Example:
+        >>> embeddings = EmbeddingService(provider="sentence-transformers")
+        >>> vector = await embeddings.embed("Fix the login bug")
+        >>> len(vector)
+        384
+    """
+
+    def __init__(
+        self,
+        provider: EmbeddingProvider = "sentence-transformers",
+        model: Optional[str] = None,
+    ):
+        """Initialize embedding service.
+
+        Args:
+            provider: Embedding provider ('sentence-transformers' or 'openai')
+            model: Model name (provider-specific default if None)
+        """
+        self.provider = provider
+        self._encoder = None
+        self._client = None
+
+        if provider == "sentence-transformers":
+            self.model = model or "all-MiniLM-L6-v2"
+            self.dimension = 384
+            self._init_sentence_transformers()
+        elif provider == "openai":
+            self.model = model or "text-embedding-3-small"
+            self.dimension = 1536
+            self._init_openai()
+        else:
+            raise ValueError(f"Unknown provider: {provider}")
+
+        logger.info(
+            "EmbeddingService initialized (provider: %s, model: %s, dim: %d)",
+            provider,
+            self.model,
+            self.dimension,
+        )
+
+    def _init_sentence_transformers(self) -> None:
+        """Initialize sentence-transformers encoder.
+
+        Lazy loads on first use to avoid startup delay.
+        """
+        # Lazy import to avoid dependency if not used
+        try:
+            from sentence_transformers import SentenceTransformer
+
+            self._encoder = SentenceTransformer(self.model)
+            logger.info("Loaded sentence-transformers model: %s", self.model)
+        except ImportError:
+            logger.error(
+                "sentence-transformers not installed. "
+                "Install with: pip install sentence-transformers"
+            )
+            raise
+
+    def _init_openai(self) -> None:
+        """Initialize OpenAI client.
+
+        Requires OPENAI_API_KEY environment variable.
+        """
+        try:
+            from openai import AsyncOpenAI
+
+            self._client = AsyncOpenAI()
+            logger.info("Initialized OpenAI client")
+        except ImportError:
+            logger.error("openai not installed. Install with: pip install openai")
+            raise
+
+    async def embed(self, text: str) -> List[float]:
+        """Generate embedding for text.
+
+        Args:
+            text: Text to embed
+
+        Returns:
+            Embedding vector as list of floats
+
+        Example:
+            >>> vector = await embeddings.embed("Fix the login bug")
+            >>> len(vector)
+            384
+        """
+        if self.provider == "sentence-transformers":
+            return await self._embed_sentence_transformers(text)
+        if self.provider == "openai":
+            return await self._embed_openai(text)
+        raise ValueError(f"Unknown provider: {self.provider}")
+
+    async def embed_batch(self, texts: List[str]) -> List[List[float]]:
+        """Generate embeddings for multiple texts.
+
+        More efficient than calling embed() in a loop.
+
+        Args:
+            texts: List of texts to embed
+
+        Returns:
+            List of embedding vectors
+
+        Example:
+            >>> vectors = await embeddings.embed_batch([
+            ...     "Fix the login bug",
+            ...     "Update the README"
+            ... ])
+            >>> len(vectors)
+            2
+        """
+        if self.provider == "sentence-transformers":
+            return await self._embed_batch_sentence_transformers(texts)
+        if self.provider == "openai":
+            return await self._embed_batch_openai(texts)
+        raise ValueError(f"Unknown provider: {self.provider}")
+
+    async def _embed_sentence_transformers(self, text: str) -> List[float]:
+        """Generate embedding using sentence-transformers.
+
+        Runs in executor to avoid blocking event loop.
+        """
+        if self._encoder is None:
+            raise RuntimeError("Encoder not initialized")
+
+        # Run encoding in executor (CPU-bound operation)
+        loop = asyncio.get_event_loop()
+        embedding = await loop.run_in_executor(
+            None, lambda: self._encoder.encode(text, convert_to_numpy=True)
+        )
+
+        return embedding.tolist()
+
+    async def _embed_batch_sentence_transformers(
+        self, texts: List[str]
+    ) -> List[List[float]]:
+        """Generate batch embeddings using sentence-transformers."""
+        if self._encoder is None:
+            raise RuntimeError("Encoder not initialized")
+
+        # Run batch encoding in executor
+        loop = asyncio.get_event_loop()
+        embeddings = await loop.run_in_executor(
+            None,
+            lambda: self._encoder.encode(
+                texts, convert_to_numpy=True, show_progress_bar=False
+            ),
+        )
+
+        return [emb.tolist() for emb in embeddings]
+
+    async def _embed_openai(self, text: str) -> List[float]:
+        """Generate embedding using OpenAI API."""
+        if self._client is None:
+            raise RuntimeError("OpenAI client not initialized")
+
+        response = await self._client.embeddings.create(
+            model=self.model,
+            input=text,
+        )
+
+        return response.data[0].embedding
+
+    async def _embed_batch_openai(self, texts: List[str]) -> List[List[float]]:
+        """Generate batch embeddings using OpenAI API."""
+        if self._client is None:
+            raise RuntimeError("OpenAI client not initialized")
+
+        response = await self._client.embeddings.create(
+            model=self.model,
+            input=texts,
+        )
+
+        return [item.embedding for item in response.data]
+
+    def cosine_similarity(self, vec1: List[float], vec2: List[float]) -> float:
+        """Calculate cosine similarity between two vectors.
+
+        Args:
+            vec1: First vector
+            vec2: Second vector
+
+        Returns:
+            Similarity score in range [-1, 1] (1 = identical, -1 = opposite)
+
+        Example:
+            >>> sim = embeddings.cosine_similarity(vec1, vec2)
+            >>> print(f"Similarity: {sim:.3f}")
+        """
+        import math
+
+        # Dot product
+        dot_product = sum(a * b for a, b in zip(vec1, vec2))
+
+        # Magnitudes
+        mag1 = math.sqrt(sum(a * a for a in vec1))
+        mag2 = math.sqrt(sum(b * b for b in vec2))
+
+        # Avoid division by zero
+        if mag1 == 0 or mag2 == 0:
+            return 0.0
+
+        return dot_product / (mag1 * mag2)

claude_mpm/commander/memory/entities.py

@@ -0,0 +1,310 @@
+"""Entity extraction from conversation messages.
+
+Extracts structured entities like files, functions, errors, and commands
+from conversation content for enhanced search and filtering.
+"""
+
+import logging
+import re
+from dataclasses import dataclass
+from enum import Enum
+from typing import Any, Dict, List
+
+logger = logging.getLogger(__name__)
+
+
+class EntityType(Enum):
+    """Types of entities that can be extracted.
+
+    Attributes:
+        FILE: File path (e.g., "src/auth.py")
+        FUNCTION: Function or method name (e.g., "login()")
+        CLASS: Class name (e.g., "UserService")
+        ERROR: Error type or message (e.g., "ValueError")
+        COMMAND: Shell command (e.g., "pytest tests/")
+        URL: Web URL (e.g., "https://example.com")
+        PACKAGE: Package or module name (e.g., "requests")
+    """
+
+    FILE = "file"
+    FUNCTION = "function"
+    CLASS = "class"
+    ERROR = "error"
+    COMMAND = "command"
+    URL = "url"
+    PACKAGE = "package"
+
+
+@dataclass
+class Entity:
+    """Extracted entity from conversation.
+
+    Attributes:
+        type: Entity type
+        value: Entity value (file path, function name, etc.)
+        context: Surrounding context (optional)
+        metadata: Additional metadata
+
+    Example:
+        >>> entity = Entity(
+        ...     type=EntityType.FILE,
+        ...     value="src/auth.py",
+        ...     context="Fix the login bug in auth.py"
+        ... )
+    """
+
+    type: EntityType
+    value: str
+    context: str = ""
+    metadata: Dict[str, Any] = None
+
+    def __post_init__(self) -> None:
+        """Initialize metadata if not provided."""
+        if self.metadata is None:
+            self.metadata = {}
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert to dictionary for JSON serialization."""
+        return {
+            "type": self.type.value,
+            "value": self.value,
+            "context": self.context,
+            "metadata": self.metadata,
+        }
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> "Entity":
+        """Create Entity from dictionary."""
+        return cls(
+            type=EntityType(data["type"]),
+            value=data["value"],
+            context=data.get("context", ""),
+            metadata=data.get("metadata", {}),
+        )
+
+
+class EntityExtractor:
+    """Extracts entities from conversation messages.
+
+    Uses regex patterns to identify files, functions, errors, commands, etc.
+
+    Example:
+        >>> extractor = EntityExtractor()
+        >>> entities = extractor.extract("Fix the login bug in src/auth.py")
+        >>> entities[0].type
+        <EntityType.FILE: 'file'>
+        >>> entities[0].value
+        'src/auth.py'
+    """
+
+    # File patterns (common extensions and paths)
+    FILE_PATTERNS = [
+        r"\b[\w/\-\.]+\.(?:py|js|ts|tsx|jsx|java|cpp|c|h|go|rs|rb|php|cs|swift|kt|md|txt|json|yaml|yml|toml|xml|html|css|scss|sh|bash)\b",
+        r"\b(?:src|lib|tests?|scripts?|docs?|config)/[\w/\-\.]+\b",
+    ]
+
+    # Function patterns (with parens or common prefixes)
+    FUNCTION_PATTERNS = [
+        r"\b[a-z_][a-z0-9_]*\(\)",
+        r"\bdef\s+([a-z_][a-z0-9_]*)",
+        r"\bfunction\s+([a-z_][a-z0-9_]*)",
+        r"\basync\s+(?:def|function)\s+([a-z_][a-z0-9_]*)",
+    ]
+
+    # Class patterns (PascalCase)
+    CLASS_PATTERNS = [
+        r"\bclass\s+([A-Z][a-zA-Z0-9_]*)",
+        r"\b([A-Z][a-zA-Z0-9_]+(?:Service|Controller|Manager|Handler|Repository|Model|View))\b",
+    ]
+
+    # Error patterns
+    ERROR_PATTERNS = [
+        r"\b([A-Z][a-zA-Z]*Error)\b",
+        r"\b([A-Z][a-zA-Z]*Exception)\b",
+        r"error:\s+(.+?)(?:\n|$)",
+    ]
+
+    # Command patterns (common CLI tools)
+    COMMAND_PATTERNS = [
+        r"\b(?:npm|yarn|pnpm|pip|poetry|cargo|go|make|docker|kubectl|git)\s+[\w\-]+",
+        r"\bpytest\s+[\w/\-\.]+",
+        r"\bpython\s+[\w/\-\.]+",
+    ]
+
+    # URL patterns
+    URL_PATTERN = r"https?://[\w\.\-/\?=&#%]+"
+
+    # Package patterns
+    PACKAGE_PATTERNS = [
+        r"\bimport\s+([\w\.]+)",
+        r"\bfrom\s+([\w\.]+)\s+import",
+        r"\brequire\(['\"]([\w\-@/]+)['\"]",
+        r"\buse\s+([\w:]+)",
+    ]
+
+    def extract(self, text: str) -> List[Entity]:
+        """Extract all entities from text.
+
+        Args:
+            text: Text to extract entities from
+
+        Returns:
+            List of extracted entities
+
+        Example:
+            >>> entities = extractor.extract("Fix the login bug in src/auth.py")
+        """
+        entities: List[Entity] = []
+
+        # Extract files
+        for pattern in self.FILE_PATTERNS:
+            for match in re.finditer(pattern, text):
+                entities.append(
+                    Entity(
+                        type=EntityType.FILE,
+                        value=match.group(0),
+                        context=self._get_context(text, match.start(), match.end()),
+                    )
+                )
+
+        # Extract functions
+        for pattern in self.FUNCTION_PATTERNS:
+            for match in re.finditer(pattern, text):
+                # Use group 1 if capturing group exists, else group 0
+                value = match.group(1) if match.lastindex else match.group(0)
+                entities.append(
+                    Entity(
+                        type=EntityType.FUNCTION,
+                        value=value.rstrip("()"),  # Remove parens
+                        context=self._get_context(text, match.start(), match.end()),
+                    )
+                )
+
+        # Extract classes
+        for pattern in self.CLASS_PATTERNS:
+            for match in re.finditer(pattern, text):
+                value = match.group(1) if match.lastindex else match.group(0)
+                entities.append(
+                    Entity(
+                        type=EntityType.CLASS,
+                        value=value,
+                        context=self._get_context(text, match.start(), match.end()),
+                    )
+                )
+
+        # Extract errors
+        for pattern in self.ERROR_PATTERNS:
+            for match in re.finditer(pattern, text):
+                value = match.group(1) if match.lastindex else match.group(0)
+                entities.append(
+                    Entity(
+                        type=EntityType.ERROR,
+                        value=value.strip(),
+                        context=self._get_context(text, match.start(), match.end()),
+                    )
+                )
+
+        # Extract commands
+        for pattern in self.COMMAND_PATTERNS:
+            for match in re.finditer(pattern, text):
+                entities.append(
+                    Entity(
+                        type=EntityType.COMMAND,
+                        value=match.group(0),
+                        context=self._get_context(text, match.start(), match.end()),
+                    )
+                )
+
+        # Extract URLs
+        for match in re.finditer(self.URL_PATTERN, text):
+            entities.append(
+                Entity(
+                    type=EntityType.URL,
+                    value=match.group(0),
+                    context=self._get_context(text, match.start(), match.end()),
+                )
+            )
+
+        # Extract packages
+        for pattern in self.PACKAGE_PATTERNS:
+            for match in re.finditer(pattern, text):
+                value = match.group(1) if match.lastindex else match.group(0)
+                entities.append(
+                    Entity(
+                        type=EntityType.PACKAGE,
+                        value=value,
+                        context=self._get_context(text, match.start(), match.end()),
+                    )
+                )
+
+        # Deduplicate entities (same type + value)
+        seen = set()
+        unique_entities = []
+        for entity in entities:
+            key = (entity.type, entity.value)
+            if key not in seen:
+                seen.add(key)
+                unique_entities.append(entity)
+
+        logger.debug("Extracted %d unique entities from text", len(unique_entities))
+        return unique_entities
+
+    def _get_context(self, text: str, start: int, end: int, window: int = 50) -> str:
+        """Get surrounding context for entity.
+
+        Args:
+            text: Full text
+            start: Entity start position
+            end: Entity end position
+            window: Characters to include before/after
+
+        Returns:
+            Context string with entity highlighted
+        """
+        context_start = max(0, start - window)
+        context_end = min(len(text), end + window)
+
+        context = text[context_start:context_end]
+
+        # Truncate at sentence boundaries if possible
+        if context_start > 0 and ". " in context[:window]:
+            context = context.split(". ", 1)[1]
+        if context_end < len(text) and ". " in context[-window:]:
+            context = context.rsplit(". ", 1)[0] + "."
+
+        return context.strip()
+
+    def filter_by_type(
+        self, entities: List[Entity], entity_type: EntityType
+    ) -> List[Entity]:
+        """Filter entities by type.
+
+        Args:
+            entities: List of entities to filter
+            entity_type: Type to filter for
+
+        Returns:
+            Filtered list of entities
+
+        Example:
+            >>> files = extractor.filter_by_type(entities, EntityType.FILE)
+        """
+        return [e for e in entities if e.type == entity_type]
+
+    def get_unique_values(
+        self, entities: List[Entity], entity_type: EntityType
+    ) -> List[str]:
+        """Get unique entity values for a type.
+
+        Args:
+            entities: List of entities
+            entity_type: Type to extract values for
+
+        Returns:
+            List of unique values
+
+        Example:
+            >>> files = extractor.get_unique_values(entities, EntityType.FILE)
+            ['src/auth.py', 'tests/test_auth.py']
+        """
+        return list({e.value for e in entities if e.type == entity_type})