claude-mpm 4.0.19__py3-none-any.whl → 4.0.20__py3-none-any.whl

claude_mpm/services/agents/memory/content_manager.py

@@ -15,7 +15,8 @@ This module provides:
 import logging
 import re
 from datetime import datetime
-from typing import Any, Dict, List, Optional
+from difflib import SequenceMatcher
+from typing import Any, Dict, List, Optional, Tuple
 
 
 class MemoryContentManager:
@@ -43,11 +44,13 @@ class MemoryContentManager:
         self.logger = logging.getLogger(f"{__name__}.{self.__class__.__name__}")
 
     def add_item_to_section(self, content: str, section: str, new_item: str) -> str:
-        """Add item to specified section, respecting limits.
+        """Add item to specified section with NLP-based deduplication.
 
         WHY: Each section has a maximum item limit to prevent information overload
-        and maintain readability. When limits are reached, oldest items are removed
-        to make room for new learnings (FIFO strategy).
+        and maintain readability. Additionally, we use NLP-based similarity detection
+        to prevent duplicate or highly similar items from cluttering the memory.
+        When similar items are found (>80% similarity), the newer item replaces the
+        older one to maintain recency while avoiding redundancy.
 
         Args:
             content: Current memory file content
@@ -55,7 +58,7 @@ class MemoryContentManager:
             new_item: Item to add
 
         Returns:
-            str: Updated content with new item added
+            str: Updated content with new item added and duplicates removed
         """
         lines = content.split("\n")
         section_start = None
@@ -76,7 +79,34 @@ class MemoryContentManager:
         if section_end is None:
             section_end = len(lines)
 
-        # Count existing items in section and find first item index
+        # Ensure line length limit (account for "- " prefix)
+        max_item_length = (
+            self.memory_limits["max_line_length"] - 2
+        )  # Subtract 2 for "- " prefix
+        if len(new_item) > max_item_length:
+            new_item = new_item[: max_item_length - 3] + "..."
+
+        # Check for duplicates or similar items using NLP similarity
+        items_to_remove = []
+        for i in range(section_start + 1, section_end):
+            if lines[i].strip().startswith("- "):
+                existing_item = lines[i].strip()[2:]  # Remove "- " prefix
+                similarity = self._calculate_similarity(existing_item, new_item)
+                
+                # If highly similar (>80%), mark for removal
+                if similarity > 0.8:
+                    items_to_remove.append(i)
+                    self.logger.debug(
+                        f"Found similar item (similarity={similarity:.2f}): "
+                        f"replacing '{existing_item[:50]}...' with '{new_item[:50]}...'"
+                    )
+
+        # Remove similar items (in reverse order to maintain indices)
+        for idx in reversed(items_to_remove):
+            lines.pop(idx)
+            section_end -= 1
+
+        # Count remaining items after deduplication
         item_count = 0
         first_item_index = None
         for i in range(section_start + 1, section_end):
@@ -85,7 +115,7 @@ class MemoryContentManager:
                     first_item_index = i
                 item_count += 1
 
-        # Check if we can add more items
+        # Check if we need to remove oldest item due to section limits
         if item_count >= self.memory_limits["max_items_per_section"]:
             # Remove oldest item (first one) to make room
             if first_item_index is not None:
@@ -100,13 +130,6 @@ class MemoryContentManager:
         ):
             insert_point += 1
 
-        # Ensure line length limit (account for "- " prefix)
-        max_item_length = (
-            self.memory_limits["max_line_length"] - 2
-        )  # Subtract 2 for "- " prefix
-        if len(new_item) > max_item_length:
-            new_item = new_item[: max_item_length - 3] + "..."
-
         lines.insert(insert_point, f"- {new_item}")
 
         # Update timestamp
@@ -335,6 +358,113 @@ class MemoryContentManager:
 
         return sections
 
+    def _calculate_similarity(self, str1: str, str2: str) -> float:
+        """Calculate similarity between two strings using fuzzy matching.
+
+        WHY: We use difflib's SequenceMatcher for lightweight NLP-based similarity
+        detection. This avoids heavy ML dependencies while still providing effective
+        duplicate detection. The algorithm finds the longest contiguous matching
+        subsequences and calculates a ratio between 0 and 1.
+
+        DESIGN DECISION: We normalize strings before comparison by:
+        - Converting to lowercase for case-insensitive matching
+        - Stripping whitespace to ignore formatting differences
+        - This balances accuracy with performance for real-time deduplication
+
+        Args:
+            str1: First string to compare
+            str2: Second string to compare
+
+        Returns:
+            float: Similarity score between 0 (completely different) and 1 (identical)
+        """
+        # Normalize strings for comparison
+        str1_normalized = str1.lower().strip()
+        str2_normalized = str2.lower().strip()
+        
+        # Handle exact matches quickly
+        if str1_normalized == str2_normalized:
+            return 1.0
+        
+        # Use SequenceMatcher for fuzzy matching
+        # None as first param tells it to use automatic junk heuristic
+        matcher = SequenceMatcher(None, str1_normalized, str2_normalized)
+        similarity = matcher.ratio()
+        
+        # Additional check: if one string contains the other (substring match)
+        # This catches cases where one item is a more detailed version of another
+        if len(str1_normalized) > 20 and len(str2_normalized) > 20:
+            if str1_normalized in str2_normalized or str2_normalized in str1_normalized:
+                # Boost similarity for substring matches
+                similarity = max(similarity, 0.85)
+        
+        return similarity
+
+    def deduplicate_section(self, content: str, section: str) -> Tuple[str, int]:
+        """Deduplicate items within a section using NLP similarity.
+
+        WHY: Over time, sections can accumulate similar or duplicate items from
+        different sessions. This method cleans up existing sections by removing
+        similar items while preserving the most recent/relevant ones.
+
+        Args:
+            content: Current memory file content
+            section: Section name to deduplicate
+
+        Returns:
+            Tuple of (updated content, number of items removed)
+        """
+        lines = content.split("\n")
+        section_start = None
+        section_end = None
+        
+        # Find section boundaries
+        for i, line in enumerate(lines):
+            if line.startswith(f"## {section}"):
+                section_start = i
+            elif section_start is not None and line.startswith("## "):
+                section_end = i
+                break
+        
+        if section_start is None:
+            return content, 0  # Section not found
+        
+        if section_end is None:
+            section_end = len(lines)
+        
+        # Collect all items in the section
+        items = []
+        item_indices = []
+        for i in range(section_start + 1, section_end):
+            if lines[i].strip().startswith("- "):
+                items.append(lines[i].strip()[2:])  # Remove "- " prefix
+                item_indices.append(i)
+        
+        # Find duplicates using pairwise comparison
+        duplicates_to_remove = set()
+        for i in range(len(items)):
+            if i in duplicates_to_remove:
+                continue
+            for j in range(i + 1, len(items)):
+                if j in duplicates_to_remove:
+                    continue
+                similarity = self._calculate_similarity(items[i], items[j])
+                if similarity > 0.8:
+                    # Remove the older item (lower index)
+                    duplicates_to_remove.add(i)
+                    self.logger.debug(
+                        f"Deduplicating: '{items[i][:50]}...' "
+                        f"(keeping newer: '{items[j][:50]}...')"
+                    )
+                    break  # Move to next item
+        
+        # Remove duplicates (in reverse order to maintain indices)
+        removed_count = len(duplicates_to_remove)
+        for idx in sorted(duplicates_to_remove, reverse=True):
+            lines.pop(item_indices[idx])
+        
+        return "\n".join(lines), removed_count
+
     def validate_memory_size(self, content: str) -> tuple[bool, Optional[str]]:
         """Validate memory content size and structure.
 

claude_mpm/services/agents/memory/template_generator.py

@@ -18,15 +18,14 @@ from pathlib import Path
 from typing import Any, Dict, List
 
 from claude_mpm.core.config import Config
-from claude_mpm.services.project.analyzer import ProjectAnalyzer
 
 
 class MemoryTemplateGenerator:
     """Generates project-specific memory templates for agents.
 
     WHY: Instead of generic templates, agents need project-specific knowledge
-    from the start. This class analyzes the current project and creates contextual
-    memories with actual project characteristics.
+    from the start. This class creates simple memory templates that agents
+    can populate as they learn about the project.
     """
 
     REQUIRED_SECTIONS = [
@@ -37,102 +36,34 @@ class MemoryTemplateGenerator:
     ]
 
     def __init__(
-        self, config: Config, working_directory: Path, project_analyzer: ProjectAnalyzer
+        self, config: Config, working_directory: Path
     ):
         """Initialize the template generator.
 
         Args:
             config: Configuration object
             working_directory: Working directory path
-            project_analyzer: Project analyzer instance
         """
         self.config = config
         self.working_directory = working_directory
-        self.project_analyzer = project_analyzer
         self.logger = logging.getLogger(f"{__name__}.{self.__class__.__name__}")
 
     def create_default_memory(self, agent_id: str, limits: Dict[str, Any]) -> str:
-        """Create project-specific default memory file for agent.
+        """Create basic memory template for agent.
 
         Args:
             agent_id: The agent identifier
             limits: Memory limits for this agent
 
         Returns:
-            str: The project-specific memory template content
+            str: The basic memory template content
         """
         # Convert agent_id to proper name, handling cases like "test_agent" -> "Test"
         agent_name = agent_id.replace("_agent", "").replace("_", " ").title()
         timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
 
-        # Analyze the project for context-specific content
-        try:
-            project_characteristics = self.project_analyzer.analyze_project()
-            project_context = self.project_analyzer.get_project_context_summary()
-
-            self.logger.info(
-                f"Creating project-specific memory for {agent_id} using analyzed project context"
-            )
-        except Exception as e:
-            self.logger.warning(
-                f"Error analyzing project for {agent_id}, falling back to basic template: {e}"
-            )
-            return self._create_basic_memory_template(agent_id, limits)
-
-        # Create project-specific sections
-        architecture_items = self._generate_architecture_section(
-            project_characteristics
-        )
-        coding_patterns = self._generate_coding_patterns_section(
-            project_characteristics
-        )
-        implementation_guidelines = self._generate_implementation_guidelines(
-            project_characteristics
-        )
-        tech_context = self._generate_technical_context(project_characteristics)
-        integration_points = self._generate_integration_points(project_characteristics)
-
-        template = f"""# {agent_name} Agent Memory - {project_characteristics.project_name}
-
-<!-- MEMORY LIMITS: {limits['max_file_size_kb']}KB max | {limits['max_sections']} sections max | {limits['max_items_per_section']} items per section -->
-<!-- Last Updated: {timestamp} | Auto-updated by: {agent_id} -->
-
-## Project Context
-{project_context}
-
-## Project Architecture
-{self._format_section_items(architecture_items)}
-
-## Coding Patterns Learned
-{self._format_section_items(coding_patterns)}
-
-## Implementation Guidelines
-{self._format_section_items(implementation_guidelines)}
-
-## Domain-Specific Knowledge
-<!-- Agent-specific knowledge for {project_characteristics.project_name} domain -->
-{self._generate_domain_knowledge_starters(project_characteristics, agent_id)}
-
-## Effective Strategies
-<!-- Successful approaches discovered through experience -->
-
-## Common Mistakes to Avoid
-{self._format_section_items(self._generate_common_mistakes(project_characteristics))}
-
-## Integration Points
-{self._format_section_items(integration_points)}
-
-## Performance Considerations
-{self._format_section_items(self._generate_performance_considerations(project_characteristics))}
-
-## Current Technical Context
-{self._format_section_items(tech_context)}
-
-## Recent Learnings
-<!-- Most recent discoveries and insights -->
-"""
-
-        return template
+        # Create a simple template that agents will populate through learning
+        return self._create_basic_memory_template(agent_id, limits)
 
     def _create_basic_memory_template(
         self, agent_id: str, limits: Dict[str, Any]
@@ -188,281 +119,3 @@ class MemoryTemplateGenerator:
 ## Recent Learnings
 <!-- Most recent discoveries and insights -->
 """
-
-    def _generate_architecture_section(self, characteristics) -> List[str]:
-        """Generate architecture section items based on project analysis."""
-        items = []
-
-        # Architecture type
-        items.append(
-            f"{characteristics.architecture_type} with {characteristics.primary_language or 'mixed'} implementation"
-        )
-
-        # Key directories structure
-        if characteristics.key_directories:
-            key_dirs = ", ".join(characteristics.key_directories[:5])
-            items.append(f"Main directories: {key_dirs}")
-
-        # Main modules
-        if characteristics.main_modules:
-            modules = ", ".join(characteristics.main_modules[:4])
-            items.append(f"Core modules: {modules}")
-
-        # Entry points
-        if characteristics.entry_points:
-            entries = ", ".join(characteristics.entry_points[:3])
-            items.append(f"Entry points: {entries}")
-
-        # Frameworks affecting architecture
-        if characteristics.web_frameworks:
-            frameworks = ", ".join(characteristics.web_frameworks[:3])
-            items.append(f"Web framework stack: {frameworks}")
-
-        return items[:8]  # Limit to prevent overwhelming
-
-    def _generate_coding_patterns_section(self, characteristics) -> List[str]:
-        """Generate coding patterns section based on project analysis."""
-        items = []
-
-        # Language-specific patterns
-        if characteristics.primary_language == "python":
-            items.append("Python project: use type hints, follow PEP 8 conventions")
-            if "django" in [fw.lower() for fw in characteristics.web_frameworks]:
-                items.append("Django patterns: models, views, templates separation")
-            elif "flask" in [fw.lower() for fw in characteristics.web_frameworks]:
-                items.append(
-                    "Flask patterns: blueprint organization, app factory pattern"
-                )
-        elif characteristics.primary_language == "node_js":
-            items.append("Node.js project: use async/await, ES6+ features")
-            if "express" in [fw.lower() for fw in characteristics.web_frameworks]:
-                items.append("Express patterns: middleware usage, route organization")
-
-        # Framework-specific patterns
-        for framework in characteristics.frameworks[:3]:
-            if "react" in framework.lower():
-                items.append("React patterns: component composition, hooks usage")
-            elif "vue" in framework.lower():
-                items.append("Vue patterns: single file components, composition API")
-
-        # Code conventions found
-        for convention in characteristics.code_conventions[:3]:
-            items.append(f"Project uses: {convention}")
-
-        return items[:8]
-
-    def _generate_implementation_guidelines(self, characteristics) -> List[str]:
-        """Generate implementation guidelines based on project analysis."""
-        items = []
-
-        # Package manager guidance
-        if characteristics.package_manager:
-            items.append(
-                f"Use {characteristics.package_manager} for dependency management"
-            )
-
-        # Testing guidelines
-        if characteristics.testing_framework:
-            items.append(f"Write tests using {characteristics.testing_framework}")
-
-        # Test patterns
-        for pattern in characteristics.test_patterns[:2]:
-            items.append(f"Follow {pattern.lower()}")
-
-        # Build tools
-        if characteristics.build_tools:
-            tools = ", ".join(characteristics.build_tools[:2])
-            items.append(f"Use build tools: {tools}")
-
-        # Configuration patterns
-        for config_pattern in characteristics.configuration_patterns[:2]:
-            items.append(f"Configuration: {config_pattern}")
-
-        # Important files to reference
-        important_configs = characteristics.important_configs[:3]
-        if important_configs:
-            configs = ", ".join(important_configs)
-            items.append(f"Key config files: {configs}")
-
-        return items[:8]
-
-    def _generate_technical_context(self, characteristics) -> List[str]:
-        """Generate current technical context based on project analysis."""
-        items = []
-
-        # Technology stack summary
-        tech_stack = []
-        if characteristics.primary_language:
-            tech_stack.append(characteristics.primary_language)
-        tech_stack.extend(characteristics.frameworks[:2])
-        if tech_stack:
-            items.append(f"Tech stack: {', '.join(tech_stack)}")
-
-        # Databases in use
-        if characteristics.databases:
-            dbs = ", ".join(characteristics.databases[:3])
-            items.append(f"Data storage: {dbs}")
-
-        # API patterns
-        if characteristics.api_patterns:
-            apis = ", ".join(characteristics.api_patterns[:2])
-            items.append(f"API patterns: {apis}")
-
-        # Key dependencies
-        if characteristics.key_dependencies:
-            deps = ", ".join(characteristics.key_dependencies[:4])
-            items.append(f"Key dependencies: {deps}")
-
-        # Documentation available
-        if characteristics.documentation_files:
-            docs = ", ".join(characteristics.documentation_files[:3])
-            items.append(f"Documentation: {docs}")
-
-        return items[:8]
-
-    def _generate_integration_points(self, characteristics) -> List[str]:
-        """Generate integration points based on project analysis."""
-        items = []
-
-        # Database integrations
-        for db in characteristics.databases[:3]:
-            items.append(f"{db.title()} database integration")
-
-        # Web framework integrations
-        for framework in characteristics.web_frameworks[:2]:
-            items.append(f"{framework} web framework integration")
-
-        # API integrations
-        for api_pattern in characteristics.api_patterns[:2]:
-            items.append(f"{api_pattern} integration pattern")
-
-        # Common integration patterns based on dependencies
-        integration_deps = [
-            dep
-            for dep in characteristics.key_dependencies
-            if any(
-                keyword in dep.lower()
-                for keyword in ["redis", "rabbit", "celery", "kafka", "docker"]
-            )
-        ]
-        for dep in integration_deps[:3]:
-            items.append(f"{dep} integration")
-
-        return items[:6]
-
-    def _generate_common_mistakes(self, characteristics) -> List[str]:
-        """Generate common mistakes based on project type and stack."""
-        items = []
-
-        # Language-specific mistakes
-        if characteristics.primary_language == "python":
-            items.append("Avoid circular imports - use late imports when needed")
-            items.append(
-                "Don't ignore virtual environment - always activate before work"
-            )
-        elif characteristics.primary_language == "node_js":
-            items.append("Avoid callback hell - use async/await consistently")
-            items.append("Don't commit node_modules - ensure .gitignore is correct")
-
-        # Framework-specific mistakes
-        if "django" in [fw.lower() for fw in characteristics.web_frameworks]:
-            items.append("Don't skip migrations - always create and apply them")
-        elif "flask" in [fw.lower() for fw in characteristics.web_frameworks]:
-            items.append("Avoid app context issues - use proper application factory")
-
-        # Database-specific mistakes
-        if characteristics.databases:
-            items.append("Don't ignore database transactions in multi-step operations")
-            items.append("Avoid N+1 queries - use proper joins or prefetching")
-
-        # Testing mistakes
-        if characteristics.testing_framework:
-            items.append(
-                "Don't skip test isolation - ensure tests can run independently"
-            )
-
-        return items[:8]
-
-    def _generate_performance_considerations(self, characteristics) -> List[str]:
-        """Generate performance considerations based on project stack."""
-        items = []
-
-        # Language-specific performance
-        if characteristics.primary_language == "python":
-            items.append("Use list comprehensions over loops where appropriate")
-            items.append("Consider caching for expensive operations")
-        elif characteristics.primary_language == "node_js":
-            items.append("Leverage event loop - avoid blocking operations")
-            items.append("Use streams for large data processing")
-
-        # Database performance
-        if characteristics.databases:
-            items.append("Index frequently queried columns")
-            items.append("Use connection pooling for database connections")
-
-        # Web framework performance
-        if characteristics.web_frameworks:
-            items.append("Implement appropriate caching strategies")
-            items.append("Optimize static asset delivery")
-
-        # Framework-specific performance
-        if "react" in [fw.lower() for fw in characteristics.frameworks]:
-            items.append("Use React.memo for expensive component renders")
-
-        return items[:6]
-
-    def _generate_domain_knowledge_starters(
-        self, characteristics, agent_id: str
-    ) -> str:
-        """Generate domain-specific knowledge starters based on project and agent type."""
-        items = []
-
-        # Project terminology
-        if characteristics.project_terminology:
-            terms = ", ".join(characteristics.project_terminology[:4])
-            items.append(f"- Key project terms: {terms}")
-
-        # Agent-specific starters
-        if "research" in agent_id.lower():
-            items.append(
-                "- Focus on code analysis, pattern discovery, and architectural insights"
-            )
-            if characteristics.documentation_files:
-                items.append(
-                    "- Prioritize documentation analysis for comprehensive understanding"
-                )
-        elif "engineer" in agent_id.lower():
-            items.append(
-                "- Focus on implementation patterns, coding standards, and best practices"
-            )
-            if characteristics.testing_framework:
-                items.append(
-                    f"- Ensure test coverage using {characteristics.testing_framework}"
-                )
-        elif "pm" in agent_id.lower() or "manager" in agent_id.lower():
-            items.append(
-                "- Focus on project coordination, task delegation, and progress tracking"
-            )
-            items.append(
-                "- Monitor integration points and cross-component dependencies"
-            )
-
-        return (
-            "\n".join(items)
-            if items
-            else "<!-- Domain knowledge will accumulate here -->"
-        )
-
-    def _format_section_items(self, items: List[str]) -> str:
-        """Format list of items as markdown bullet points."""
-        if not items:
-            return "<!-- Items will be added as knowledge accumulates -->"
-
-        formatted_items = []
-        for item in items:
-            # Ensure each item starts with a dash and is properly formatted
-            if not item.startswith("- "):
-                item = f"- {item}"
-            formatted_items.append(item)
-
-        return "\n".join(formatted_items)