claude-mpm 4.0.31__py3-none-any.whl → 4.0.34__py3-none-any.whl

claude_mpm/core/framework_loader.py

@@ -3,9 +3,10 @@
 import logging
 import os
 import sys
+import time
 from datetime import datetime
 from pathlib import Path
-from typing import Any, Dict, Optional
+from typing import Any, Dict, Optional, Set, Tuple
 
 # Import resource handling for packaged installations
 try:
@@ -34,9 +35,41 @@ class FrameworkLoader:
 
     This component handles:
     1. Finding the framework (claude-multiagent-pm)
-    2. Loading INSTRUCTIONS.md instructions
+    2. Loading custom instructions from .claude-mpm/ directories
     3. Preparing agent definitions
     4. Formatting for injection
+    
+    Custom Instructions Loading:
+    The framework loader supports custom instructions through .claude-mpm/ directories.
+    It NEVER reads from .claude/ directories to avoid conflicts with Claude Code.
+    
+    File Loading Precedence (highest to lowest):
+    
+    INSTRUCTIONS.md:
+      1. Project: ./.claude-mpm/INSTRUCTIONS.md
+      2. User: ~/.claude-mpm/INSTRUCTIONS.md
+      3. System: (built-in framework instructions)
+    
+    WORKFLOW.md:
+      1. Project: ./.claude-mpm/WORKFLOW.md
+      2. User: ~/.claude-mpm/WORKFLOW.md
+      3. System: src/claude_mpm/agents/WORKFLOW.md
+    
+    MEMORY.md:
+      1. Project: ./.claude-mpm/MEMORY.md
+      2. User: ~/.claude-mpm/MEMORY.md
+      3. System: src/claude_mpm/agents/MEMORY.md
+    
+    Actual Memories:
+      - User: ~/.claude-mpm/memories/PM_memories.md
+      - Project: ./.claude-mpm/memories/PM_memories.md (overrides user)
+      - Agent memories: *_memories.md files (only loaded if agent is deployed)
+    
+    Important Notes:
+    - Project-level files always override user-level files
+    - User-level files always override system defaults
+    - The framework NEVER reads from .claude/ directories
+    - Custom instructions are clearly labeled with their source level
     """
 
     def __init__(
@@ -54,6 +87,22 @@ class FrameworkLoader:
         self.agents_dir = agents_dir
         self.framework_version = None
         self.framework_last_modified = None
+        
+        # Performance optimization: Initialize caches
+        self._agent_capabilities_cache: Optional[str] = None
+        self._agent_capabilities_cache_time: float = 0
+        self._deployed_agents_cache: Optional[Set[str]] = None
+        self._deployed_agents_cache_time: float = 0
+        self._agent_metadata_cache: Dict[str, Tuple[Optional[Dict[str, Any]], float]] = {}
+        self._memories_cache: Optional[Dict[str, Any]] = None
+        self._memories_cache_time: float = 0
+        
+        # Cache TTL settings (in seconds)
+        self.CAPABILITIES_CACHE_TTL = 60  # 60 seconds for capabilities
+        self.DEPLOYED_AGENTS_CACHE_TTL = 30  # 30 seconds for deployed agents
+        self.METADATA_CACHE_TTL = 60  # 60 seconds for agent metadata
+        self.MEMORIES_CACHE_TTL = 60  # 60 seconds for memories
+        
         self.framework_content = self._load_framework_content()
 
         # Initialize agent registry
@@ -62,6 +111,32 @@ class FrameworkLoader:
         # Initialize output style manager (must be after content is loaded)
         self.output_style_manager = None
         # Defer initialization until first use to ensure content is loaded
+    
+    def clear_all_caches(self) -> None:
+        """Clear all caches to force reload on next access."""
+        self.logger.info("Clearing all framework loader caches")
+        self._agent_capabilities_cache = None
+        self._agent_capabilities_cache_time = 0
+        self._deployed_agents_cache = None
+        self._deployed_agents_cache_time = 0
+        self._agent_metadata_cache.clear()
+        self._memories_cache = None
+        self._memories_cache_time = 0
+    
+    def clear_agent_caches(self) -> None:
+        """Clear agent-related caches (capabilities, deployed agents, metadata)."""
+        self.logger.info("Clearing agent-related caches")
+        self._agent_capabilities_cache = None
+        self._agent_capabilities_cache_time = 0
+        self._deployed_agents_cache = None
+        self._deployed_agents_cache_time = 0
+        self._agent_metadata_cache.clear()
+    
+    def clear_memory_caches(self) -> None:
+        """Clear memory-related caches."""
+        self.logger.info("Clearing memory caches")
+        self._memories_cache = None
+        self._memories_cache_time = 0
 
     def _initialize_output_style(self) -> None:
         """Initialize output style management and deploy if applicable."""
@@ -349,28 +424,43 @@ class FrameworkLoader:
 
     def _load_workflow_instructions(self, content: Dict[str, Any]) -> None:
         """
-        Load WORKFLOW.md with project-specific override support.
+        Load WORKFLOW.md from .claude-mpm directories.
 
-        Precedence:
-        1. Project-specific: .claude-mpm/agents/WORKFLOW.md
-        2. System default: src/claude_mpm/agents/WORKFLOW.md or packaged
+        Precedence (highest to lowest):
+        1. Project-specific: ./.claude-mpm/WORKFLOW.md
+        2. User-specific: ~/.claude-mpm/WORKFLOW.md
+        3. System default: src/claude_mpm/agents/WORKFLOW.md or packaged
+        
+        NOTE: We do NOT load from .claude/ directories to avoid conflicts.
 
         Args:
             content: Dictionary to update with workflow instructions
         """
-        # Check for project-specific workflow first
-        project_workflow_path = Path.cwd() / ".claude-mpm" / "agents" / "WORKFLOW.md"
+        # Check for project-specific WORKFLOW.md first (highest priority)
+        project_workflow_path = Path.cwd() / ".claude-mpm" / "WORKFLOW.md"
         if project_workflow_path.exists():
             loaded_content = self._try_load_file(
                 project_workflow_path, "project-specific WORKFLOW.md"
             )
             if loaded_content:
                 content["workflow_instructions"] = loaded_content
-                content["project_workflow"] = "project"
-                self.logger.info("Using project-specific WORKFLOW.md")
+                content["workflow_instructions_level"] = "project"
+                self.logger.info("Using project-specific workflow instructions from .claude-mpm/WORKFLOW.md")
+                return
+        
+        # Check for user-specific WORKFLOW.md (medium priority)
+        user_workflow_path = Path.home() / ".claude-mpm" / "WORKFLOW.md"
+        if user_workflow_path.exists():
+            loaded_content = self._try_load_file(
+                user_workflow_path, "user-specific WORKFLOW.md"
+            )
+            if loaded_content:
+                content["workflow_instructions"] = loaded_content
+                content["workflow_instructions_level"] = "user"
+                self.logger.info("Using user-specific workflow instructions from ~/.claude-mpm/WORKFLOW.md")
                 return
 
-        # Fall back to system workflow
+        # Fall back to system workflow (lowest priority)
         if self.framework_path and self.framework_path != Path("__PACKAGED__"):
             system_workflow_path = (
                 self.framework_path / "src" / "claude_mpm" / "agents" / "WORKFLOW.md"
@@ -381,33 +471,48 @@ class FrameworkLoader:
                 )
                 if loaded_content:
                     content["workflow_instructions"] = loaded_content
-                    content["project_workflow"] = "system"
-                    self.logger.info("Using system WORKFLOW.md")
+                    content["workflow_instructions_level"] = "system"
+                    self.logger.info("Using system workflow instructions")
 
     def _load_memory_instructions(self, content: Dict[str, Any]) -> None:
         """
-        Load MEMORY.md with project-specific override support.
+        Load MEMORY.md from .claude-mpm directories.
 
-        Precedence:
-        1. Project-specific: .claude-mpm/agents/MEMORY.md
-        2. System default: src/claude_mpm/agents/MEMORY.md or packaged
+        Precedence (highest to lowest):
+        1. Project-specific: ./.claude-mpm/MEMORY.md
+        2. User-specific: ~/.claude-mpm/MEMORY.md
+        3. System default: src/claude_mpm/agents/MEMORY.md or packaged
+        
+        NOTE: We do NOT load from .claude/ directories to avoid conflicts.
 
         Args:
             content: Dictionary to update with memory instructions
         """
-        # Check for project-specific memory instructions first
-        project_memory_path = Path.cwd() / ".claude-mpm" / "agents" / "MEMORY.md"
+        # Check for project-specific MEMORY.md first (highest priority)
+        project_memory_path = Path.cwd() / ".claude-mpm" / "MEMORY.md"
         if project_memory_path.exists():
             loaded_content = self._try_load_file(
                 project_memory_path, "project-specific MEMORY.md"
             )
             if loaded_content:
                 content["memory_instructions"] = loaded_content
-                content["project_memory"] = "project"
-                self.logger.info("Using project-specific MEMORY.md")
+                content["memory_instructions_level"] = "project"
+                self.logger.info("Using project-specific memory instructions from .claude-mpm/MEMORY.md")
+                return
+        
+        # Check for user-specific MEMORY.md (medium priority)
+        user_memory_path = Path.home() / ".claude-mpm" / "MEMORY.md"
+        if user_memory_path.exists():
+            loaded_content = self._try_load_file(
+                user_memory_path, "user-specific MEMORY.md"
+            )
+            if loaded_content:
+                content["memory_instructions"] = loaded_content
+                content["memory_instructions_level"] = "user"
+                self.logger.info("Using user-specific memory instructions from ~/.claude-mpm/MEMORY.md")
                 return
 
-        # Fall back to system memory instructions
+        # Fall back to system memory instructions (lowest priority)
         if self.framework_path and self.framework_path != Path("__PACKAGED__"):
             system_memory_path = (
                 self.framework_path / "src" / "claude_mpm" / "agents" / "MEMORY.md"
@@ -418,16 +523,26 @@ class FrameworkLoader:
                 )
                 if loaded_content:
                     content["memory_instructions"] = loaded_content
-                    content["project_memory"] = "system"
-                    self.logger.info("Using system MEMORY.md")
+                    content["memory_instructions_level"] = "system"
+                    self.logger.info("Using system memory instructions")
     
     def _get_deployed_agents(self) -> set:
         """
         Get a set of deployed agent names from .claude/agents/ directories.
+        Uses caching to avoid repeated filesystem scans.
         
         Returns:
             Set of agent names (file stems) that are deployed
         """
+        # Check if cache is valid
+        current_time = time.time()
+        if (self._deployed_agents_cache is not None and 
+            current_time - self._deployed_agents_cache_time < self.DEPLOYED_AGENTS_CACHE_TTL):
+            self.logger.debug(f"Using cached deployed agents (age: {current_time - self._deployed_agents_cache_time:.1f}s)")
+            return self._deployed_agents_cache
+        
+        # Cache miss or expired - perform actual scan
+        self.logger.debug("Scanning for deployed agents (cache miss or expired)")
         deployed = set()
         
         # Check multiple locations for deployed agents
@@ -445,11 +560,17 @@ class FrameworkLoader:
                         self.logger.debug(f"Found deployed agent: {agent_file.stem} in {agents_dir}")
         
         self.logger.debug(f"Total deployed agents found: {len(deployed)}")
+        
+        # Update cache
+        self._deployed_agents_cache = deployed
+        self._deployed_agents_cache_time = current_time
+        
         return deployed
     
     def _load_actual_memories(self, content: Dict[str, Any]) -> None:
         """
         Load actual memories from both user and project directories.
+        Uses caching to avoid repeated file I/O operations.
         
         Loading order:
         1. User-level memories from ~/.claude-mpm/memories/ (global defaults)
@@ -462,6 +583,23 @@ class FrameworkLoader:
         Args:
             content: Dictionary to update with actual memories
         """
+        # Check if cache is valid
+        current_time = time.time()
+        if (self._memories_cache is not None and 
+            current_time - self._memories_cache_time < self.MEMORIES_CACHE_TTL):
+            cache_age = current_time - self._memories_cache_time
+            self.logger.debug(f"Using cached memories (age: {cache_age:.1f}s)")
+            
+            # Apply cached memories to content
+            if "actual_memories" in self._memories_cache:
+                content["actual_memories"] = self._memories_cache["actual_memories"]
+            if "agent_memories" in self._memories_cache:
+                content["agent_memories"] = self._memories_cache["agent_memories"]
+            return
+        
+        # Cache miss or expired - perform actual loading
+        self.logger.debug("Loading memories from disk (cache miss or expired)")
+        
         # Define memory directories in priority order (user first, then project)
         user_memories_dir = Path.home() / ".claude-mpm" / "memories"
         project_memories_dir = Path.cwd() / ".claude-mpm" / "memories"
@@ -513,6 +651,14 @@ class FrameworkLoader:
                 memory_size = len(memory_content.encode('utf-8'))
                 self.logger.debug(f"Aggregated {agent_name} memory: {memory_size:,} bytes")
         
+        # Update cache with loaded memories
+        self._memories_cache = {}
+        if "actual_memories" in content:
+            self._memories_cache["actual_memories"] = content["actual_memories"]
+        if "agent_memories" in content:
+            self._memories_cache["agent_memories"] = content["agent_memories"]
+        self._memories_cache_time = current_time
+        
         # Log detailed summary
         if loaded_count > 0 or skipped_count > 0:
             # Count unique agents with memories
@@ -664,11 +810,10 @@ class FrameworkLoader:
         Aggregate multiple memory entries into a single memory string.
         
         Strategy:
-        - Support both sectioned and non-sectioned memories
-        - Preserve all bullet-point items (lines starting with -)
-        - Merge sections when present, with project-level taking precedence
-        - Remove exact duplicates within sections and unsectioned items
-        - Preserve unique entries from both sources
+        - Simplified to support list-based memories only
+        - Preserve all unique bullet-point items (lines starting with -)
+        - Remove exact duplicates
+        - Project-level memories take precedence over user-level
         
         Args:
             memory_entries: List of memory entries with source, content, and path
@@ -683,97 +828,52 @@ class FrameworkLoader:
         if len(memory_entries) == 1:
             return memory_entries[0]["content"]
         
-        # Parse all memories into sections and unsectioned items
-        all_sections = {}
-        unsectioned_items = {}  # Items without a section header
+        # Parse all memories into a simple list
+        all_items = {}  # Dict to track items and their source
         metadata_lines = []
+        agent_id = None
         
         for entry in memory_entries:
             content = entry["content"]
             source = entry["source"]
             
-            # Parse content into sections and unsectioned items
-            current_section = None
-            current_items = []
-            
             for line in content.split('\n'):
+                # Check for header to extract agent_id
+                if line.startswith('# Agent Memory:'):
+                    agent_id = line.replace('# Agent Memory:', '').strip()
                 # Check for metadata lines
-                if line.startswith('<!-- ') and line.endswith(' -->'):
+                elif line.startswith('<!-- ') and line.endswith(' -->'):
                     # Only keep metadata from project source or if not already present
                     if source == "project" or line not in metadata_lines:
                         metadata_lines.append(line)
-                # Check for section headers (## Level 2 headers)
-                elif line.startswith('## '):
-                    # Save previous section if exists
-                    if current_section and current_items:
-                        if current_section not in all_sections:
-                            all_sections[current_section] = {}
-                        # Store items with their source
-                        for item in current_items:
-                            # Use content as key to detect duplicates
-                            all_sections[current_section][item] = source
+                # Check for list items
+                elif line.strip().startswith('-'):
+                    # Normalize the item for comparison
+                    item_text = line.strip()
+                    normalized = item_text.lstrip('- ').strip().lower()
                     
-                    # Start new section
-                    current_section = line
-                    current_items = []
-                # Check for content lines (including unsectioned bullet points)
-                elif line.strip():
-                    # If it's a bullet point or regular content
-                    if current_section:
-                        # Add to current section
-                        current_items.append(line)
-                    elif line.strip().startswith('-'):
-                        # It's an unsectioned bullet point - preserve it
-                        # Use content as key to detect duplicates
-                        # Project source overrides user source
-                        if line not in unsectioned_items or source == "project":
-                            unsectioned_items[line] = source
-                    # Skip other non-bullet unsectioned content (like headers)
-                    elif not line.strip().startswith('#'):
-                        # Include non-header orphaned content in unsectioned items
-                        if line not in unsectioned_items or source == "project":
-                            unsectioned_items[line] = source
-            
-            # Save last section if exists
-            if current_section and current_items:
-                if current_section not in all_sections:
-                    all_sections[current_section] = {}
-                for item in current_items:
-                    # Project source overrides user source
-                    if item not in all_sections[current_section] or source == "project":
-                        all_sections[current_section][item] = source
+                    # Add item if new or if project source overrides user source
+                    if normalized not in all_items or source == "project":
+                        all_items[normalized] = (item_text, source)
         
-        # Build aggregated content
+        # Build aggregated content as simple list
         lines = []
         
-        # Add metadata
-        if metadata_lines:
-            lines.extend(metadata_lines)
-            lines.append("")
-        
         # Add header
-        lines.append("# Aggregated Memory")
-        lines.append("")
-        lines.append("*This memory combines user-level and project-level memories.*")
-        lines.append("")
+        if agent_id:
+            lines.append(f"# Agent Memory: {agent_id}")
+        else:
+            lines.append("# Agent Memory")
         
-        # Add unsectioned items first (if any)
-        if unsectioned_items:
-            # Sort items to ensure consistent output
-            for item in sorted(unsectioned_items.keys()):
-                lines.append(item)
-            lines.append("")  # Empty line after unsectioned items
+        # Add latest timestamp from metadata
+        from datetime import datetime
+        lines.append(f"<!-- Last Updated: {datetime.now().isoformat()}Z -->")
+        lines.append("")
         
-        # Add sections
-        for section_header in sorted(all_sections.keys()):
-            lines.append(section_header)
-            section_items = all_sections[section_header]
-            
-            # Sort items to ensure consistent output
-            for item in sorted(section_items.keys()):
-                lines.append(item)
-            
-            lines.append("")  # Empty line after section
+        # Add all unique items (sorted for consistency)
+        for normalized_key in sorted(all_items.keys()):
+            item_text, source = all_items[normalized_key]
+            lines.append(item_text)
         
         return '\n'.join(lines)
 
@@ -859,9 +959,11 @@ class FrameworkLoader:
             "working_claude_md": "",
             "framework_instructions": "",
             "workflow_instructions": "",
-            "project_workflow": "",
+            "workflow_instructions_level": "",  # Track source level
             "memory_instructions": "",
-            "project_memory": "",
+            "memory_instructions_level": "",  # Track source level
+            "project_workflow": "",  # Deprecated, use workflow_instructions_level
+            "project_memory": "",  # Deprecated, use memory_instructions_level
             "actual_memories": "",  # Add field for actual memories from PM_memories.md
         }
 
@@ -1141,16 +1243,22 @@ class FrameworkLoader:
                 workflow_content = self._strip_metadata_comments(
                     self.framework_content["workflow_instructions"]
                 )
-                instructions += f"\n\n{workflow_content}\n"
-                # Note: project-specific workflow is being used (logged elsewhere)
+                level = self.framework_content.get("workflow_instructions_level", "system")
+                if level != "system":
+                    instructions += f"\n\n## Workflow Instructions ({level} level)\n\n"
+                    instructions += "**The following workflow instructions override system defaults:**\n\n"
+                instructions += f"{workflow_content}\n"
 
             # Add MEMORY.md after workflow instructions
             if self.framework_content.get("memory_instructions"):
                 memory_content = self._strip_metadata_comments(
                     self.framework_content["memory_instructions"]
                 )
-                instructions += f"\n\n{memory_content}\n"
-                # Note: project-specific memory instructions being used (logged elsewhere)
+                level = self.framework_content.get("memory_instructions_level", "system")
+                if level != "system":
+                    instructions += f"\n\n## Memory Instructions ({level} level)\n\n"
+                    instructions += "**The following memory instructions override system defaults:**\n\n"
+                instructions += f"{memory_content}\n"
             
             # Add actual PM memories after memory instructions
             if self.framework_content.get("actual_memories"):
@@ -1322,7 +1430,20 @@ Extract tickets from these patterns:
         return instructions
 
     def _generate_agent_capabilities_section(self) -> str:
-        """Generate dynamic agent capabilities section from deployed agents."""
+        """Generate dynamic agent capabilities section from deployed agents.
+        Uses caching to avoid repeated file I/O and parsing operations."""
+        
+        # Check if cache is valid
+        current_time = time.time()
+        if (self._agent_capabilities_cache is not None and 
+            current_time - self._agent_capabilities_cache_time < self.CAPABILITIES_CACHE_TTL):
+            cache_age = current_time - self._agent_capabilities_cache_time
+            self.logger.debug(f"Using cached agent capabilities (age: {cache_age:.1f}s)")
+            return self._agent_capabilities_cache
+        
+        # Cache miss or expired - generate capabilities
+        self.logger.debug("Generating agent capabilities (cache miss or expired)")
+        
         try:
             from pathlib import Path
 
@@ -1349,7 +1470,7 @@ Extract tickets from these patterns:
                         if agent_file.name.startswith("."):
                             continue
                         
-                        # Parse agent metadata
+                        # Parse agent metadata (with caching)
                         agent_data = self._parse_agent_metadata(agent_file)
                         if agent_data:
                             agent_id = agent_data["id"]
@@ -1361,7 +1482,11 @@ Extract tickets from these patterns:
 
             if not all_agents:
                 self.logger.warning(f"No agents found in any location: {agents_dirs}")
-                return self._get_fallback_capabilities()
+                result = self._get_fallback_capabilities()
+                # Cache the fallback result too
+                self._agent_capabilities_cache = result
+                self._agent_capabilities_cache_time = current_time
+                return result
             
             # Log agent collection summary
             project_agents = [aid for aid, (_, pri) in all_agents.items() if pri == 0]
@@ -1379,7 +1504,11 @@ Extract tickets from these patterns:
             deployed_agents = [agent_data for agent_data, _ in all_agents.values()]
 
             if not deployed_agents:
-                return self._get_fallback_capabilities()
+                result = self._get_fallback_capabilities()
+                # Cache the fallback result
+                self._agent_capabilities_cache = result
+                self._agent_capabilities_cache_time = current_time
+                return result
 
             # Sort agents alphabetically by ID
             deployed_agents.sort(key=lambda x: x["id"])
@@ -1426,19 +1555,46 @@ Extract tickets from these patterns:
             # Add summary
             section += f"\n**Total Available Agents**: {len(deployed_agents)}\n"
 
+            # Cache the generated capabilities
+            self._agent_capabilities_cache = section
+            self._agent_capabilities_cache_time = current_time
+            self.logger.debug(f"Cached agent capabilities section ({len(section)} chars)")
+            
             return section
 
         except Exception as e:
             self.logger.warning(f"Could not generate dynamic agent capabilities: {e}")
-            return self._get_fallback_capabilities()
+            result = self._get_fallback_capabilities()
+            # Cache even the fallback result
+            self._agent_capabilities_cache = result
+            self._agent_capabilities_cache_time = current_time
+            return result
 
     def _parse_agent_metadata(self, agent_file: Path) -> Optional[Dict[str, Any]]:
         """Parse agent metadata from deployed agent file.
+        Uses caching based on file path and modification time.
 
         Returns:
             Dictionary with agent metadata directly from YAML frontmatter.
         """
         try:
+            # Check cache based on file path and modification time
+            cache_key = str(agent_file)
+            file_mtime = agent_file.stat().st_mtime
+            current_time = time.time()
+            
+            # Check if we have cached data for this file
+            if cache_key in self._agent_metadata_cache:
+                cached_data, cached_mtime = self._agent_metadata_cache[cache_key]
+                # Use cache if file hasn't been modified and cache isn't too old
+                if (cached_mtime == file_mtime and 
+                    current_time - cached_mtime < self.METADATA_CACHE_TTL):
+                    self.logger.debug(f"Using cached metadata for {agent_file.name}")
+                    return cached_data
+            
+            # Cache miss or expired - parse the file
+            self.logger.debug(f"Parsing metadata for {agent_file.name} (cache miss or expired)")
+            
             import yaml
 
             with open(agent_file, "r") as f:
@@ -1476,6 +1632,9 @@ Extract tickets from these patterns:
                         # IMPORTANT: Do NOT add spaces to tools field - it breaks deployment!
                         # Tools must remain as comma-separated without spaces: "Read,Write,Edit"
 
+            # Cache the parsed metadata
+            self._agent_metadata_cache[cache_key] = (agent_data, file_mtime)
+            
             return agent_data
 
         except Exception as e: