claude-mpm 3.5.4__py3-none-any.whl → 3.6.0__py3-none-any.whl

claude_mpm/services/agents/deployment/agent_deployment.py

@@ -37,6 +37,7 @@ from typing import Optional, List, Dict, Any
 from claude_mpm.core.logger import get_logger
 from claude_mpm.constants import EnvironmentVars, Paths, AgentMetadata
 from claude_mpm.config.paths import paths
+from claude_mpm.core.config import Config
 
 
 class AgentDeploymentService:
@@ -70,13 +71,14 @@ class AgentDeploymentService:
     - YAML generation capabilities
     """
     
-    def __init__(self, templates_dir: Optional[Path] = None, base_agent_path: Optional[Path] = None):
+    def __init__(self, templates_dir: Optional[Path] = None, base_agent_path: Optional[Path] = None, working_directory: Optional[Path] = None):
         """
         Initialize agent deployment service.
         
         Args:
             templates_dir: Directory containing agent JSON files
             base_agent_path: Path to base_agent.md file
+            working_directory: User's working directory (for project agents)
             
         METRICS OPPORTUNITY: Track initialization performance:
         - Template directory scan time
@@ -100,6 +102,17 @@ class AgentDeploymentService:
             'deployment_errors': {}  # Track error types and frequencies
         }
         
+        # Determine the actual working directory
+        # Priority: working_directory param > CLAUDE_MPM_USER_PWD env var > current directory
+        if working_directory:
+            self.working_directory = Path(working_directory)
+        elif 'CLAUDE_MPM_USER_PWD' in os.environ:
+            self.working_directory = Path(os.environ['CLAUDE_MPM_USER_PWD'])
+        else:
+            self.working_directory = Path.cwd()
+        
+        self.logger.info(f"Working directory for deployment: {self.working_directory}")
+        
         # Find templates directory using centralized path management
         if templates_dir:
             self.templates_dir = Path(templates_dir)
@@ -119,7 +132,7 @@ class AgentDeploymentService:
         self.logger.info(f"Templates directory: {self.templates_dir}")
         self.logger.info(f"Base agent path: {self.base_agent_path}")
         
-    def deploy_agents(self, target_dir: Optional[Path] = None, force_rebuild: bool = False, deployment_mode: str = "update") -> Dict[str, Any]:
+    def deploy_agents(self, target_dir: Optional[Path] = None, force_rebuild: bool = False, deployment_mode: str = "update", config: Optional[Config] = None, use_async: bool = True) -> Dict[str, Any]:
         """
         Build and deploy agents by combining base_agent.md with templates.
         Also deploys system instructions for PM framework.
@@ -128,6 +141,12 @@ class AgentDeploymentService:
         - "update": Normal update mode - skip agents with matching versions (default)
         - "project": Project deployment mode - always deploy all agents regardless of version
         
+        CONFIGURATION:
+        The config parameter or default configuration is used to determine:
+        - Which agents to exclude from deployment
+        - Case sensitivity for agent name matching
+        - Whether to exclude agent dependencies
+        
         METRICS COLLECTED:
         - Deployment start/end timestamps
         - Individual agent deployment durations
@@ -137,6 +156,7 @@ class AgentDeploymentService:
         - Error type frequencies
         
         OPERATIONAL FLOW:
+        0. Validates and repairs broken frontmatter in existing agents (Step 0)
         1. Validates target directory (creates if needed)
         2. Loads base agent configuration
         3. Discovers all agent templates
@@ -168,6 +188,8 @@ class AgentDeploymentService:
             target_dir: Target directory for agents (default: .claude/agents/)
             force_rebuild: Force rebuild even if agents exist (useful for troubleshooting)
             deployment_mode: "update" for version-aware updates, "project" for always deploy
+            config: Optional configuration object (loads default if not provided)
+            use_async: Use async operations for 50-70% faster deployment (default: True)
             
         Returns:
             Dictionary with deployment results:
@@ -178,13 +200,73 @@ class AgentDeploymentService:
             - skipped: List of unchanged agents
             - errors: List of deployment errors
             - total: Total number of agents processed
+            - repaired: List of agents with repaired frontmatter
         """
         # METRICS: Record deployment start time for performance tracking
         deployment_start_time = time.time()
         
+        # Try async deployment for better performance if requested
+        if use_async:
+            try:
+                from .async_agent_deployment import deploy_agents_async_wrapper
+                self.logger.info("Using async deployment for improved performance")
+                
+                # Run async deployment
+                results = deploy_agents_async_wrapper(
+                    templates_dir=self.templates_dir,
+                    base_agent_path=self.base_agent_path,
+                    working_directory=self.working_directory,
+                    target_dir=target_dir,
+                    force_rebuild=force_rebuild,
+                    config=config
+                )
+                
+                # Add metrics about async vs sync
+                if 'metrics' in results:
+                    results['metrics']['deployment_method'] = 'async'
+                    duration_ms = results['metrics'].get('duration_ms', 0)
+                    self.logger.info(f"Async deployment completed in {duration_ms:.1f}ms")
+                    
+                    # Update internal metrics
+                    self._deployment_metrics['total_deployments'] += 1
+                    if not results.get('errors'):
+                        self._deployment_metrics['successful_deployments'] += 1
+                    else:
+                        self._deployment_metrics['failed_deployments'] += 1
+                
+                return results
+                
+            except ImportError:
+                self.logger.warning("Async deployment not available, falling back to sync")
+            except Exception as e:
+                self.logger.warning(f"Async deployment failed, falling back to sync: {e}")
+        
+        # Continue with synchronous deployment
+        self.logger.info("Using synchronous deployment")
+        
+        # Load configuration if not provided
+        if config is None:
+            config = Config()
+        
+        # Get agent exclusion configuration
+        excluded_agents = config.get('agent_deployment.excluded_agents', [])
+        case_sensitive = config.get('agent_deployment.case_sensitive', False)
+        exclude_dependencies = config.get('agent_deployment.exclude_dependencies', False)
+        
+        # Normalize excluded agents list for comparison
+        if not case_sensitive:
+            excluded_agents = [agent.lower() for agent in excluded_agents]
+        
+        # Log exclusion configuration if agents are being excluded
+        if excluded_agents:
+            self.logger.info(f"Excluding agents from deployment: {excluded_agents}")
+            self.logger.debug(f"Case sensitive matching: {case_sensitive}")
+            self.logger.debug(f"Exclude dependencies: {exclude_dependencies}")
+        
         if not target_dir:
-            # Default to user's home .claude/agents directory
-            agents_dir = Path(Paths.CLAUDE_AGENTS_DIR.value).expanduser()
+            # Default to working directory's .claude/agents directory (not home)
+            # This ensures we deploy to the user's project directory, not the framework directory
+            agents_dir = self.working_directory / ".claude" / "agents"
         else:
             # If target_dir provided, use it directly (caller decides structure)
             # This allows for both passing a project dir or the full agents path
@@ -211,6 +293,7 @@ class AgentDeploymentService:
             "updated": [],
             "migrated": [],  # Track agents migrated from old format
             "converted": [],  # Track YAML to MD conversions
+            "repaired": [],  # Track agents with repaired frontmatter
             "total": 0,
             # METRICS: Add detailed timing and performance data to results
             "metrics": {
@@ -226,6 +309,16 @@ class AgentDeploymentService:
         try:
             # Create agents directory if needed
             agents_dir.mkdir(parents=True, exist_ok=True)
+            
+            # STEP 0: Validate and repair broken frontmatter in existing agents
+            # This ensures all existing agents have valid YAML frontmatter before deploying new ones
+            repair_results = self._validate_and_repair_existing_agents(agents_dir)
+            if repair_results["repaired"]:
+                results["repaired"] = repair_results["repaired"]
+                self.logger.info(f"Repaired frontmatter in {len(repair_results['repaired'])} existing agents")
+                for agent_name in repair_results["repaired"]:
+                    self.logger.debug(f"  - Repaired: {agent_name}")
+            
             # Determine source tier for logging
             source_tier = "SYSTEM"
             if ".claude-mpm/agents" in str(self.templates_dir) and "/templates" not in str(self.templates_dir):
@@ -268,16 +361,54 @@ class AgentDeploymentService:
             
             # Get all template files
             template_files = list(self.templates_dir.glob("*.json"))
-            # Filter out non-agent files - exclude system files and uppercase special files
-            excluded_names = {"__init__", "MEMORIES", "TODOWRITE", "INSTRUCTIONS", "README"}
-            template_files = [
-                f for f in template_files 
-                if f.stem not in excluded_names 
-                and not f.stem.startswith(".")
-                and not f.stem.endswith(".backup")
-            ]
+            
+            # Build the combined exclusion set
+            # Start with hardcoded exclusions (these are ALWAYS excluded)
+            hardcoded_exclusions = {"__init__", "MEMORIES", "TODOWRITE", "INSTRUCTIONS", "README", "pm", "PM", "project_manager"}
+            
+            # Add user-configured exclusions
+            user_exclusions = set()
+            for agent in excluded_agents:
+                if case_sensitive:
+                    user_exclusions.add(agent)
+                else:
+                    # For case-insensitive, we'll check during filtering
+                    user_exclusions.add(agent.lower())
+            
+            # Filter out excluded agents
+            filtered_files = []
+            excluded_count = 0
+            for f in template_files:
+                agent_name = f.stem
+                
+                # Check hardcoded exclusions (always case-sensitive)
+                if agent_name in hardcoded_exclusions:
+                    self.logger.debug(f"Excluding {agent_name}: hardcoded system exclusion")
+                    excluded_count += 1
+                    continue
+                
+                # Check file patterns
+                if agent_name.startswith(".") or agent_name.endswith(".backup"):
+                    self.logger.debug(f"Excluding {agent_name}: file pattern exclusion")
+                    excluded_count += 1
+                    continue
+                
+                # Check user-configured exclusions
+                compare_name = agent_name.lower() if not case_sensitive else agent_name
+                if compare_name in user_exclusions:
+                    self.logger.info(f"Excluding {agent_name}: user-configured exclusion")
+                    excluded_count += 1
+                    continue
+                
+                # Agent is not excluded, add to filtered list
+                filtered_files.append(f)
+            
+            template_files = filtered_files
             results["total"] = len(template_files)
             
+            if excluded_count > 0:
+                self.logger.info(f"Excluded {excluded_count} agents from deployment")
+            
             for template_file in template_files:
                 try:
                     # METRICS: Track individual agent deployment time
@@ -373,6 +504,7 @@ class AgentDeploymentService:
                 f"updated {len(results['updated'])}, "
                 f"migrated {len(results['migrated'])}, "
                 f"converted {len(results['converted'])} YAML files, "
+                f"repaired {len(results['repaired'])} frontmatter, "
                 f"skipped {len(results['skipped'])}, "
                 f"errors: {len(results['errors'])}"
             )
@@ -864,7 +996,8 @@ temperature: {temperature}"""
             Dictionary of environment variables set for verification
         """
         if not config_dir:
-            config_dir = Path.cwd() / Paths.CLAUDE_CONFIG_DIR.value
+            # Use the working directory determined during initialization
+            config_dir = self.working_directory / Paths.CLAUDE_CONFIG_DIR.value
         
         env_vars = {}
         
@@ -930,7 +1063,8 @@ temperature: {temperature}"""
             - warnings: List of potential issues
         """
         if not config_dir:
-            config_dir = Path.cwd() / ".claude"
+            # Use the working directory determined during initialization
+            config_dir = self.working_directory / ".claude"
         
         results = {
             "config_dir": str(config_dir),
@@ -953,6 +1087,17 @@ temperature: {temperature}"""
         
         # List deployed agents
         agent_files = list(agents_dir.glob("*.md"))
+        
+        # Get exclusion configuration for logging purposes
+        try:
+            from claude_mpm.core.config import Config
+            config = Config()
+            excluded_agents = config.get('agent_deployment.excluded_agents', [])
+            if excluded_agents:
+                self.logger.debug(f"Note: The following agents are configured for exclusion: {excluded_agents}")
+        except Exception:
+            pass  # Ignore config loading errors in verification
+        
         for agent_file in agent_files:
             try:
                 # Read first few lines to get agent name from YAML
@@ -1093,14 +1238,46 @@ temperature: {temperature}"""
             return agents
         
         template_files = sorted(self.templates_dir.glob("*.json"))
-        # Filter out non-agent files - exclude system files and uppercase special files
-        excluded_names = {"__init__", "MEMORIES", "TODOWRITE", "INSTRUCTIONS", "README"}
-        template_files = [
-            f for f in template_files 
-            if f.stem not in excluded_names 
-            and not f.stem.startswith(".")
-            and not f.stem.endswith(".backup")
-        ]
+        
+        # Load configuration for exclusions
+        try:
+            from claude_mpm.core.config import Config
+            config = Config()
+            excluded_agents = config.get('agent_deployment.excluded_agents', [])
+            case_sensitive = config.get('agent_deployment.case_sensitive', False)
+            
+            # Normalize excluded agents for comparison
+            if not case_sensitive:
+                excluded_agents = [agent.lower() for agent in excluded_agents]
+        except Exception:
+            # If config loading fails, use empty exclusion list
+            excluded_agents = []
+            case_sensitive = False
+        
+        # Build combined exclusion set
+        hardcoded_exclusions = {"__init__", "MEMORIES", "TODOWRITE", "INSTRUCTIONS", "README", "pm", "PM", "project_manager"}
+        
+        # Filter out excluded agents
+        filtered_files = []
+        for f in template_files:
+            agent_name = f.stem
+            
+            # Check hardcoded exclusions
+            if agent_name in hardcoded_exclusions:
+                continue
+            
+            # Check file patterns
+            if agent_name.startswith(".") or agent_name.endswith(".backup"):
+                continue
+            
+            # Check user-configured exclusions
+            compare_name = agent_name.lower() if not case_sensitive else agent_name
+            if compare_name in excluded_agents:
+                continue
+            
+            filtered_files.append(f)
+        
+        template_files = filtered_files
         
         for template_file in template_files:
             try:
@@ -1283,7 +1460,8 @@ temperature: {temperature}"""
             Cleanup results
         """
         if not config_dir:
-            config_dir = Path.cwd() / ".claude"
+            # Use the working directory determined during initialization
+            config_dir = self.working_directory / ".claude"
         
         results = {
             "removed": [],
@@ -1934,4 +2112,128 @@ metadata:
         except Exception as e:
             self.logger.warning(f"Error extracting YAML field '{field_name}': {e}")
         
-        return None
+        return None
+    
+    def _validate_and_repair_existing_agents(self, agents_dir: Path) -> Dict[str, Any]:
+        """
+        Validate and repair broken frontmatter in existing agent files.
+        
+        This method scans existing .claude/agents/*.md files and validates their
+        frontmatter. If the frontmatter is broken or missing, it attempts to repair
+        it or marks the agent for replacement during deployment.
+        
+        WHY: Ensures all existing agents have valid YAML frontmatter before deployment,
+        preventing runtime errors in Claude Code when loading agents.
+        
+        Args:
+            agents_dir: Directory containing agent .md files
+            
+        Returns:
+            Dictionary with validation results:
+            - repaired: List of agent names that were repaired
+            - replaced: List of agent names marked for replacement
+            - errors: List of validation errors
+        """
+        results = {
+            "repaired": [],
+            "replaced": [],
+            "errors": []
+        }
+        
+        try:
+            # Import frontmatter validator
+            from claude_mpm.agents.frontmatter_validator import FrontmatterValidator
+            validator = FrontmatterValidator()
+            
+            # Find existing agent files
+            agent_files = list(agents_dir.glob("*.md"))
+            
+            if not agent_files:
+                self.logger.debug("No existing agent files to validate")
+                return results
+            
+            self.logger.debug(f"Validating frontmatter in {len(agent_files)} existing agents")
+            
+            for agent_file in agent_files:
+                try:
+                    agent_name = agent_file.stem
+                    
+                    # Read agent file content
+                    content = agent_file.read_text()
+                    
+                    # Check if this is a system agent (authored by claude-mpm)
+                    # Only repair system agents, leave user agents alone
+                    if "author: claude-mpm" not in content and "author: 'claude-mpm'" not in content:
+                        self.logger.debug(f"Skipping validation for user agent: {agent_name}")
+                        continue
+                    
+                    # Extract and validate frontmatter
+                    if not content.startswith("---"):
+                        # No frontmatter at all - mark for replacement
+                        self.logger.warning(f"Agent {agent_name} has no frontmatter, marking for replacement")
+                        results["replaced"].append(agent_name)
+                        # Delete the file so it will be recreated
+                        agent_file.unlink()
+                        continue
+                    
+                    # Try to extract frontmatter
+                    try:
+                        end_marker = content.find("\n---\n", 4)
+                        if end_marker == -1:
+                            end_marker = content.find("\n---\r\n", 4)
+                        
+                        if end_marker == -1:
+                            # Broken frontmatter - mark for replacement
+                            self.logger.warning(f"Agent {agent_name} has broken frontmatter, marking for replacement")
+                            results["replaced"].append(agent_name)
+                            # Delete the file so it will be recreated
+                            agent_file.unlink()
+                            continue
+                        
+                        # Validate frontmatter with the validator
+                        validation_result = validator.validate_file(agent_file)
+                        
+                        if not validation_result.is_valid:
+                            # Check if it can be corrected
+                            if validation_result.corrected_frontmatter:
+                                # Apply corrections
+                                correction_result = validator.correct_file(agent_file, dry_run=False)
+                                if correction_result.corrections:
+                                    results["repaired"].append(agent_name)
+                                    self.logger.info(f"Repaired frontmatter for agent {agent_name}")
+                                    for correction in correction_result.corrections:
+                                        self.logger.debug(f"  - {correction}")
+                            else:
+                                # Cannot be corrected - mark for replacement
+                                self.logger.warning(f"Agent {agent_name} has invalid frontmatter that cannot be repaired, marking for replacement")
+                                results["replaced"].append(agent_name)
+                                # Delete the file so it will be recreated
+                                agent_file.unlink()
+                        elif validation_result.warnings:
+                            # Has warnings but is valid
+                            for warning in validation_result.warnings:
+                                self.logger.debug(f"Agent {agent_name} warning: {warning}")
+                        
+                    except Exception as e:
+                        # Any error in parsing - mark for replacement
+                        self.logger.warning(f"Failed to parse frontmatter for {agent_name}: {e}, marking for replacement")
+                        results["replaced"].append(agent_name)
+                        # Delete the file so it will be recreated
+                        try:
+                            agent_file.unlink()
+                        except Exception:
+                            pass
+                    
+                except Exception as e:
+                    error_msg = f"Failed to validate agent {agent_file.name}: {e}"
+                    self.logger.error(error_msg)
+                    results["errors"].append(error_msg)
+            
+        except ImportError:
+            self.logger.warning("FrontmatterValidator not available, skipping validation")
+        except Exception as e:
+            error_msg = f"Agent validation failed: {e}"
+            self.logger.error(error_msg)
+            results["errors"].append(error_msg)
+        
+        return results