claude-mpm 5.4.22__py3-none-any.whl → 5.6.34__py3-none-any.whl

claude_mpm/commander/memory/__init__.py

@@ -0,0 +1,45 @@
+"""Conversation memory system for Commander.
+
+This module provides semantic search, storage, and context compression
+for all Claude Code instance conversations.
+
+Key Components:
+    - ConversationStore: CRUD operations for conversations
+    - EmbeddingService: Generate vector embeddings
+    - SemanticSearch: Query conversations semantically
+    - ContextCompressor: Summarize conversations for context
+    - EntityExtractor: Extract files, functions, errors
+
+Example:
+    >>> from claude_mpm.commander.memory import (
+    ...     ConversationStore,
+    ...     EmbeddingService,
+    ...     SemanticSearch,
+    ...     ContextCompressor,
+    ... )
+    >>> store = ConversationStore()
+    >>> embeddings = EmbeddingService()
+    >>> search = SemanticSearch(store, embeddings)
+    >>> results = await search.search("how did we fix the login bug?")
+"""
+
+from .compression import ContextCompressor
+from .embeddings import EmbeddingService
+from .entities import Entity, EntityExtractor, EntityType
+from .integration import MemoryIntegration
+from .search import SearchResult, SemanticSearch
+from .store import Conversation, ConversationMessage, ConversationStore
+
+__all__ = [
+    "ContextCompressor",
+    "Conversation",
+    "ConversationMessage",
+    "ConversationStore",
+    "EmbeddingService",
+    "Entity",
+    "EntityExtractor",
+    "EntityType",
+    "MemoryIntegration",
+    "SearchResult",
+    "SemanticSearch",
+]

claude_mpm/commander/memory/compression.py

@@ -0,0 +1,347 @@
+"""Context compression and conversation summarization.
+
+Compresses long conversations into concise summaries for efficient context
+loading when resuming sessions or searching past work.
+"""
+
+import logging
+from typing import List, Optional
+
+from ..llm.openrouter_client import OpenRouterClient
+from .store import Conversation, ConversationMessage
+
+logger = logging.getLogger(__name__)
+
+
+class ContextCompressor:
+    """Compress conversations into summaries for context loading.
+
+    Uses cheap LLM (mistral-small) to generate summaries of conversations
+    and compress multiple conversations into context strings.
+
+    Attributes:
+        client: OpenRouterClient for LLM requests
+        summary_threshold: Minimum messages to trigger summarization
+        max_context_tokens: Maximum tokens for compressed context
+
+    Example:
+        >>> compressor = ContextCompressor(client)
+        >>> summary = await compressor.summarize(messages)
+        >>> context = await compressor.compress_for_context(
+        ...     conversations,
+        ...     max_tokens=4000
+        ... )
+    """
+
+    def __init__(
+        self,
+        client: OpenRouterClient,
+        summary_threshold: int = 10,
+        max_context_tokens: int = 4000,
+    ):
+        """Initialize context compressor.
+
+        Args:
+            client: OpenRouterClient for LLM requests
+            summary_threshold: Minimum messages to summarize
+            max_context_tokens: Maximum tokens for context string
+        """
+        self.client = client
+        self.summary_threshold = summary_threshold
+        self.max_context_tokens = max_context_tokens
+
+        logger.info(
+            "ContextCompressor initialized (threshold: %d msgs, max_tokens: %d)",
+            summary_threshold,
+            max_context_tokens,
+        )
+
+    async def summarize(
+        self,
+        messages: List[ConversationMessage],
+        focus: Optional[str] = None,
+    ) -> str:
+        """Generate summary of conversation messages.
+
+        Args:
+            messages: List of messages to summarize
+            focus: Optional focus area (e.g., "bug fixes", "API changes")
+
+        Returns:
+            Concise summary (2-4 sentences)
+
+        Example:
+            >>> summary = await compressor.summarize(messages)
+            >>> print(summary)
+            "Fixed login authentication bug in src/auth.py by updating token validation..."
+        """
+        if len(messages) < 2:
+            # Too short to summarize
+            return messages[0].content if messages else ""
+
+        # Build conversation text
+        conversation_text = self._format_messages(messages)
+
+        # Build summarization prompt
+        if focus:
+            prompt = f"""Summarize the following conversation, focusing on: {focus}
+
+Conversation:
+{conversation_text}
+
+Provide a concise summary (2-4 sentences) that captures:
+1. What was the main task or problem
+2. What actions were taken
+3. What was the outcome or current status
+4. Any important files, functions, or errors mentioned
+
+Summary:"""
+        else:
+            prompt = f"""Summarize the following conversation in 2-4 sentences.
+
+Conversation:
+{conversation_text}
+
+Focus on:
+1. What was the main task or problem
+2. What actions were taken
+3. What was the outcome or current status
+
+Summary:"""
+
+        messages_for_llm = [{"role": "user", "content": prompt}]
+
+        system = (
+            "You are a technical summarization assistant. "
+            "Provide clear, concise summaries of development conversations. "
+            "Focus on actionable information and key outcomes."
+        )
+
+        summary = await self.client.chat(messages_for_llm, system=system)
+        logger.debug(
+            "Generated summary (%d chars) from %d messages", len(summary), len(messages)
+        )
+
+        return summary.strip()
+
+    async def compress_for_context(
+        self,
+        conversations: List[Conversation],
+        max_tokens: Optional[int] = None,
+        prioritize_recent: bool = True,
+    ) -> str:
+        """Compress multiple conversations into context string.
+
+        Prioritizes recent conversations and uses summaries for older ones
+        to fit within token budget.
+
+        Args:
+            conversations: List of conversations to compress
+            max_tokens: Maximum tokens (default: self.max_context_tokens)
+            prioritize_recent: Whether to prioritize recent conversations
+
+        Returns:
+            Compressed context string ready for LLM input
+
+        Example:
+            >>> context = await compressor.compress_for_context(
+            ...     conversations,
+            ...     max_tokens=4000
+            ... )
+            >>> print(f"Context: {len(context)} chars")
+        """
+        if max_tokens is None:
+            max_tokens = self.max_context_tokens
+
+        # Sort by recency if prioritizing
+        if prioritize_recent:
+            conversations = sorted(
+                conversations, key=lambda c: c.updated_at, reverse=True
+            )
+
+        # Build context incrementally
+        context_parts = []
+        current_tokens = 0
+
+        for conv in conversations:
+            # Use summary if available, else generate one
+            if conv.summary:
+                summary_text = conv.summary
+            elif len(conv.messages) >= self.summary_threshold:
+                # Generate summary on-the-fly
+                summary_text = await self.summarize(conv.messages)
+            else:
+                # Use full conversation for short ones
+                summary_text = conv.get_full_text()
+
+            # Format conversation section
+            section = self._format_conversation_section(conv, summary_text)
+            section_tokens = len(section) // 4  # Rough approximation
+
+            # Check if adding this would exceed budget
+            if current_tokens + section_tokens > max_tokens:
+                # Try to fit summary only
+                short_summary = summary_text.split(". ")[0] + "."
+                short_section = self._format_conversation_section(conv, short_summary)
+                short_tokens = len(short_section) // 4
+
+                if current_tokens + short_tokens <= max_tokens:
+                    context_parts.append(short_section)
+                    current_tokens += short_tokens
+                else:
+                    # Can't fit any more, stop
+                    break
+            else:
+                context_parts.append(section)
+                current_tokens += section_tokens
+
+        context = "\n\n---\n\n".join(context_parts)
+
+        logger.info(
+            "Compressed %d conversations into context (%d chars, ~%d tokens)",
+            len(context_parts),
+            len(context),
+            current_tokens,
+        )
+
+        return context
+
+    def needs_summarization(self, messages: List[ConversationMessage]) -> bool:
+        """Check if conversation needs summarization.
+
+        Args:
+            messages: List of messages to check
+
+        Returns:
+            True if message count exceeds threshold
+
+        Example:
+            >>> if compressor.needs_summarization(messages):
+            ...     summary = await compressor.summarize(messages)
+        """
+        return len(messages) >= self.summary_threshold
+
+    def _format_messages(
+        self,
+        messages: List[ConversationMessage],
+        max_messages: Optional[int] = None,
+    ) -> str:
+        """Format messages as text for summarization.
+
+        Args:
+            messages: Messages to format
+            max_messages: Maximum messages to include
+
+        Returns:
+            Formatted conversation text
+        """
+        if max_messages:
+            messages = messages[:max_messages]
+
+        lines = []
+        for msg in messages:
+            # Format: ROLE: content
+            lines.append(f"{msg.role.upper()}: {msg.content}")
+
+        return "\n\n".join(lines)
+
+    def _format_conversation_section(
+        self, conversation: Conversation, summary: str
+    ) -> str:
+        """Format conversation section for context string.
+
+        Args:
+            conversation: Conversation to format
+            summary: Summary or full text
+
+        Returns:
+            Formatted section with metadata
+        """
+        # Format timestamp
+        timestamp = conversation.updated_at.strftime("%Y-%m-%d %H:%M")
+
+        # Build section
+        return f"""## Conversation: {conversation.id}
+**Project:** {conversation.project_id}
+**Instance:** {conversation.instance_name}
+**Updated:** {timestamp}
+**Messages:** {conversation.message_count}
+
+{summary}"""
+
+    async def auto_summarize_conversation(
+        self, conversation: Conversation
+    ) -> Optional[str]:
+        """Automatically summarize conversation if needed.
+
+        Checks if conversation needs summarization and generates one if so.
+        Updates the conversation's summary field but does NOT save to store.
+
+        Args:
+            conversation: Conversation to summarize
+
+        Returns:
+            Summary if generated, None if not needed
+
+        Example:
+            >>> summary = await compressor.auto_summarize_conversation(conv)
+            >>> if summary:
+            ...     conv.summary = summary
+            ...     await store.save(conv)
+        """
+        if not self.needs_summarization(conversation.messages):
+            logger.debug(
+                "Conversation %s too short to summarize (%d messages)",
+                conversation.id,
+                len(conversation.messages),
+            )
+            return None
+
+        if conversation.summary:
+            logger.debug("Conversation %s already has summary", conversation.id)
+            return conversation.summary
+
+        # Generate summary
+        summary = await self.summarize(conversation.messages)
+        logger.info("Auto-generated summary for conversation %s", conversation.id)
+
+        return summary
+
+    async def update_summary_if_stale(
+        self,
+        conversation: Conversation,
+        message_threshold: int = 5,
+    ) -> Optional[str]:
+        """Update summary if conversation has grown significantly.
+
+        Args:
+            conversation: Conversation to check
+            message_threshold: New messages required to trigger update
+
+        Returns:
+            Updated summary if regenerated, None otherwise
+
+        Example:
+            >>> updated = await compressor.update_summary_if_stale(conv)
+            >>> if updated:
+            ...     conv.summary = updated
+            ...     await store.save(conv)
+        """
+        if not conversation.summary:
+            # No existing summary, generate one
+            return await self.auto_summarize_conversation(conversation)
+
+        # Check if conversation has grown significantly
+        # (Simple heuristic: if more than threshold messages since last summarization)
+        # In practice, you'd track when summary was generated
+        if len(conversation.messages) < self.summary_threshold + message_threshold:
+            return None
+
+        # Regenerate summary
+        logger.info(
+            "Regenerating stale summary for conversation %s (%d messages)",
+            conversation.id,
+            len(conversation.messages),
+        )
+
+        return await self.summarize(conversation.messages)

claude_mpm/commander/memory/embeddings.py

@@ -0,0 +1,230 @@
+"""Embedding service for semantic search.
+
+Generates vector embeddings using sentence-transformers (local) or
+OpenAI API (cloud). Defaults to local model for zero-cost operation.
+"""
+
+import asyncio
+import logging
+from typing import List, Literal, Optional
+
+logger = logging.getLogger(__name__)
+
+EmbeddingProvider = Literal["sentence-transformers", "openai"]
+
+
+class EmbeddingService:
+    """Generate vector embeddings for semantic search.
+
+    Supports multiple providers:
+    - sentence-transformers: Local, free, good quality (default)
+    - openai: Cloud API, best quality, costs money
+
+    Attributes:
+        provider: Embedding provider to use
+        model: Model name for the provider
+        dimension: Embedding vector dimension
+
+    Example:
+        >>> embeddings = EmbeddingService(provider="sentence-transformers")
+        >>> vector = await embeddings.embed("Fix the login bug")
+        >>> len(vector)
+        384
+    """
+
+    def __init__(
+        self,
+        provider: EmbeddingProvider = "sentence-transformers",
+        model: Optional[str] = None,
+    ):
+        """Initialize embedding service.
+
+        Args:
+            provider: Embedding provider ('sentence-transformers' or 'openai')
+            model: Model name (provider-specific default if None)
+        """
+        self.provider = provider
+        self._encoder = None
+        self._client = None
+
+        if provider == "sentence-transformers":
+            self.model = model or "all-MiniLM-L6-v2"
+            self.dimension = 384
+            self._init_sentence_transformers()
+        elif provider == "openai":
+            self.model = model or "text-embedding-3-small"
+            self.dimension = 1536
+            self._init_openai()
+        else:
+            raise ValueError(f"Unknown provider: {provider}")
+
+        logger.info(
+            "EmbeddingService initialized (provider: %s, model: %s, dim: %d)",
+            provider,
+            self.model,
+            self.dimension,
+        )
+
+    def _init_sentence_transformers(self) -> None:
+        """Initialize sentence-transformers encoder.
+
+        Lazy loads on first use to avoid startup delay.
+        """
+        # Lazy import to avoid dependency if not used
+        try:
+            from sentence_transformers import SentenceTransformer
+
+            self._encoder = SentenceTransformer(self.model)
+            logger.info("Loaded sentence-transformers model: %s", self.model)
+        except ImportError:
+            logger.error(
+                "sentence-transformers not installed. "
+                "Install with: pip install sentence-transformers"
+            )
+            raise
+
+    def _init_openai(self) -> None:
+        """Initialize OpenAI client.
+
+        Requires OPENAI_API_KEY environment variable.
+        """
+        try:
+            from openai import AsyncOpenAI
+
+            self._client = AsyncOpenAI()
+            logger.info("Initialized OpenAI client")
+        except ImportError:
+            logger.error("openai not installed. Install with: pip install openai")
+            raise
+
+    async def embed(self, text: str) -> List[float]:
+        """Generate embedding for text.
+
+        Args:
+            text: Text to embed
+
+        Returns:
+            Embedding vector as list of floats
+
+        Example:
+            >>> vector = await embeddings.embed("Fix the login bug")
+            >>> len(vector)
+            384
+        """
+        if self.provider == "sentence-transformers":
+            return await self._embed_sentence_transformers(text)
+        if self.provider == "openai":
+            return await self._embed_openai(text)
+        raise ValueError(f"Unknown provider: {self.provider}")
+
+    async def embed_batch(self, texts: List[str]) -> List[List[float]]:
+        """Generate embeddings for multiple texts.
+
+        More efficient than calling embed() in a loop.
+
+        Args:
+            texts: List of texts to embed
+
+        Returns:
+            List of embedding vectors
+
+        Example:
+            >>> vectors = await embeddings.embed_batch([
+            ...     "Fix the login bug",
+            ...     "Update the README"
+            ... ])
+            >>> len(vectors)
+            2
+        """
+        if self.provider == "sentence-transformers":
+            return await self._embed_batch_sentence_transformers(texts)
+        if self.provider == "openai":
+            return await self._embed_batch_openai(texts)
+        raise ValueError(f"Unknown provider: {self.provider}")
+
+    async def _embed_sentence_transformers(self, text: str) -> List[float]:
+        """Generate embedding using sentence-transformers.
+
+        Runs in executor to avoid blocking event loop.
+        """
+        if self._encoder is None:
+            raise RuntimeError("Encoder not initialized")
+
+        # Run encoding in executor (CPU-bound operation)
+        loop = asyncio.get_event_loop()
+        embedding = await loop.run_in_executor(
+            None, lambda: self._encoder.encode(text, convert_to_numpy=True)
+        )
+
+        return embedding.tolist()
+
+    async def _embed_batch_sentence_transformers(
+        self, texts: List[str]
+    ) -> List[List[float]]:
+        """Generate batch embeddings using sentence-transformers."""
+        if self._encoder is None:
+            raise RuntimeError("Encoder not initialized")
+
+        # Run batch encoding in executor
+        loop = asyncio.get_event_loop()
+        embeddings = await loop.run_in_executor(
+            None,
+            lambda: self._encoder.encode(
+                texts, convert_to_numpy=True, show_progress_bar=False
+            ),
+        )
+
+        return [emb.tolist() for emb in embeddings]
+
+    async def _embed_openai(self, text: str) -> List[float]:
+        """Generate embedding using OpenAI API."""
+        if self._client is None:
+            raise RuntimeError("OpenAI client not initialized")
+
+        response = await self._client.embeddings.create(
+            model=self.model,
+            input=text,
+        )
+
+        return response.data[0].embedding
+
+    async def _embed_batch_openai(self, texts: List[str]) -> List[List[float]]:
+        """Generate batch embeddings using OpenAI API."""
+        if self._client is None:
+            raise RuntimeError("OpenAI client not initialized")
+
+        response = await self._client.embeddings.create(
+            model=self.model,
+            input=texts,
+        )
+
+        return [item.embedding for item in response.data]
+
+    def cosine_similarity(self, vec1: List[float], vec2: List[float]) -> float:
+        """Calculate cosine similarity between two vectors.
+
+        Args:
+            vec1: First vector
+            vec2: Second vector
+
+        Returns:
+            Similarity score in range [-1, 1] (1 = identical, -1 = opposite)
+
+        Example:
+            >>> sim = embeddings.cosine_similarity(vec1, vec2)
+            >>> print(f"Similarity: {sim:.3f}")
+        """
+        import math
+
+        # Dot product
+        dot_product = sum(a * b for a, b in zip(vec1, vec2))
+
+        # Magnitudes
+        mag1 = math.sqrt(sum(a * a for a in vec1))
+        mag2 = math.sqrt(sum(b * b for b in vec2))
+
+        # Avoid division by zero
+        if mag1 == 0 or mag2 == 0:
+            return 0.0
+
+        return dot_product / (mag1 * mag2)