claude-mpm 4.24.0__py3-none-any.whl → 5.4.41__py3-none-any.whl

claude_mpm/services/agents/deployment/multi_source_deployment_service.py

@@ -16,24 +16,36 @@ import os
 from pathlib import Path
 from typing import Any, Dict, List, Optional, Tuple
 
+import yaml
+
 from claude_mpm.core.config import Config
 from claude_mpm.core.logging_config import get_logger
 
 from .agent_discovery_service import AgentDiscoveryService
 from .agent_version_manager import AgentVersionManager
+from .remote_agent_discovery_service import RemoteAgentDiscoveryService
 
 
 class MultiSourceAgentDeploymentService:
     """Service for deploying agents from multiple sources with version comparison.
 
     This service ensures that the highest version of each agent is deployed,
-    regardless of whether it comes from system templates, project agents, or
-    user agents.
+    regardless of whether it comes from system templates, project agents,
+    user agents, or remote agents.
+
+    4-Tier Agent Discovery:
+    1. System templates (lowest priority) - Built-in agents
+    2. User agents (DEPRECATED) - User-level customizations (~/.claude-mpm/agents/)
+    3. Remote agents - Agents cached from GitHub
+    4. Project agents (highest priority) - Project-specific customizations
 
     WHY: The current system processes agents from a single source at a time,
     which can result in lower version agents being deployed if they exist in
     a higher priority source. This service fixes that by comparing versions
     across all sources.
+
+    DEPRECATION: User-level agents (~/.claude-mpm/agents/) are deprecated and
+    will be removed in v5.0.0. Use project-level agents instead.
     """
 
     def __init__(self):
@@ -41,23 +53,156 @@ class MultiSourceAgentDeploymentService:
         self.logger = get_logger(__name__)
         self.version_manager = AgentVersionManager()
 
+    def _read_template_version(self, template_path: Path) -> Optional[str]:
+        """Read version from template file (supports both .md and .json formats).
+
+        For .md files: Extract version from YAML frontmatter
+        For .json files: Extract version from JSON structure
+
+        Args:
+            template_path: Path to template file
+
+        Returns:
+            Version string or None if version cannot be extracted
+        """
+        try:
+            if template_path.suffix == ".md":
+                # Parse markdown with YAML frontmatter
+                content = template_path.read_text()
+
+                # Extract YAML frontmatter (between --- markers)
+                if not content.strip().startswith("---"):
+                    return None
+
+                parts = content.split("---", 2)
+                if len(parts) < 3:
+                    return None
+
+                # Parse YAML frontmatter
+                frontmatter = yaml.safe_load(parts[1])
+                if not frontmatter:
+                    return None
+
+                # Extract version from frontmatter
+                version = frontmatter.get("version")
+                return version if version else None
+
+            if template_path.suffix == ".json":
+                # Parse JSON template
+                template_data = json.loads(template_path.read_text())
+                metadata = template_data.get("metadata", {})
+                version = (
+                    template_data.get("agent_version")
+                    or template_data.get("version")
+                    or metadata.get("version")
+                )
+                return version if version else None
+
+            self.logger.warning(
+                f"Unknown template format: {template_path.suffix} for {template_path.name}"
+            )
+            return None
+
+        except yaml.YAMLError as e:
+            self.logger.warning(
+                f"Invalid YAML frontmatter in {template_path.name}: {e}"
+            )
+            return None
+        except json.JSONDecodeError as e:
+            self.logger.warning(f"Invalid JSON in {template_path.name}: {e}")
+            return None
+        except Exception as e:
+            self.logger.warning(
+                f"Error reading template version from {template_path.name}: {e}"
+            )
+            return None
+
+    def _build_canonical_id_for_agent(self, agent_info: Dict[str, Any]) -> str:
+        """Build or retrieve canonical_id for an agent.
+
+        NEW: Supports enhanced agent matching via canonical_id.
+
+        Priority:
+        1. Use existing canonical_id from agent_info if present
+        2. Generate from collection_id + agent_id if available
+        3. Fallback to legacy:{filename} for backward compatibility
+
+        Args:
+            agent_info: Agent dictionary with metadata
+
+        Returns:
+            Canonical ID string for matching
+
+        Example:
+            Remote agent: "bobmatnyc/claude-mpm-agents:pm"
+            Legacy agent: "legacy:custom-agent"
+        """
+        # Priority 1: Existing canonical_id
+        if "canonical_id" in agent_info:
+            return agent_info["canonical_id"]
+
+        # Priority 2: Generate from collection_id + agent_id
+        collection_id = agent_info.get("collection_id")
+        agent_id = agent_info.get("agent_id")
+
+        if collection_id and agent_id:
+            canonical_id = f"{collection_id}:{agent_id}"
+            # Cache it in agent_info for future use
+            agent_info["canonical_id"] = canonical_id
+            return canonical_id
+
+        # Priority 3: Fallback to legacy format
+        # Use filename or agent name
+        agent_name = agent_info.get("name") or agent_info.get("metadata", {}).get(
+            "name", "unknown"
+        )
+
+        # Extract filename from path
+        path_str = (
+            agent_info.get("path")
+            or agent_info.get("file_path")
+            or agent_info.get("source_file")
+        )
+
+        if path_str:
+            filename = Path(path_str).stem
+            canonical_id = f"legacy:{filename}"
+        else:
+            canonical_id = f"legacy:{agent_name}"
+
+        # Cache it
+        agent_info["canonical_id"] = canonical_id
+        return canonical_id
+
     def discover_agents_from_all_sources(
         self,
         system_templates_dir: Optional[Path] = None,
         project_agents_dir: Optional[Path] = None,
         user_agents_dir: Optional[Path] = None,
+        agents_cache_dir: Optional[Path] = None,
         working_directory: Optional[Path] = None,
     ) -> Dict[str, List[Dict[str, Any]]]:
-        """Discover agents from all available sources.
+        """Discover agents from all 4 tiers (system, user, cache, project).
+
+        Priority hierarchy (highest to lowest):
+        4. Project agents - Highest priority, project-specific customizations
+        3. Cached agents - GitHub-synced agents from cache
+        2. User agents - DEPRECATED, user-level customizations
+        1. System templates - Lowest priority, built-in agents
 
         Args:
             system_templates_dir: Directory containing system agent templates
             project_agents_dir: Directory containing project-specific agents
-            user_agents_dir: Directory containing user custom agents
+            user_agents_dir: Directory containing user custom agents (DEPRECATED)
+            agents_cache_dir: Directory containing cached agents from Git sources
             working_directory: Current working directory for finding project agents
 
         Returns:
             Dictionary mapping agent names to list of agent info from different sources
+
+        Deprecation Warning:
+            User-level agents are deprecated and will show a warning if found.
+            Use 'claude-mpm agents migrate-to-project' to migrate them.
         """
         agents_by_name = {}
 
@@ -80,24 +225,53 @@ class MultiSourceAgentDeploymentService:
             if not user_agents_dir.exists():
                 user_agents_dir = None
 
-        # Discover agents from each source
+        if not agents_cache_dir:
+            # Check for agents in cache directory
+            cache_dir = Path.home() / ".claude-mpm" / "cache"
+            agents_cache_dir = cache_dir / "agents"
+            if not agents_cache_dir.exists():
+                agents_cache_dir = None
+
+        # Discover agents from each source in priority order
+        # Note: We process in reverse priority order (system first) and build up the dictionary
+        # The select_highest_version_agents() method will handle the actual prioritization
         sources = [
             ("system", system_templates_dir),
-            ("project", project_agents_dir),
             ("user", user_agents_dir),
+            ("remote", agents_cache_dir),
+            ("project", project_agents_dir),
         ]
 
+        # Track if we found user agents for deprecation warning
+        user_agents_found = False
+
         for source_name, source_dir in sources:
             if source_dir and source_dir.exists():
                 self.logger.debug(
                     f"Discovering agents from {source_name} source: {source_dir}"
                 )
-                discovery_service = AgentDiscoveryService(source_dir)
-                # Pass log_discovery=False to avoid duplicate logging
-                agents = discovery_service.list_available_agents(log_discovery=False)
+
+                # Use appropriate discovery service based on source type
+                if source_name == "remote":
+                    # Remote agents are Markdown, use RemoteAgentDiscoveryService
+                    remote_service = RemoteAgentDiscoveryService(source_dir)
+                    agents = remote_service.discover_remote_agents()
+                else:
+                    # Other sources are JSON, use AgentDiscoveryService
+                    discovery_service = AgentDiscoveryService(source_dir)
+                    # Pass log_discovery=False to avoid duplicate logging
+                    agents = discovery_service.list_available_agents(
+                        log_discovery=False
+                    )
+
+                # Track user agents for deprecation warning
+                if source_name == "user" and agents:
+                    user_agents_found = True
 
                 for agent_info in agents:
-                    agent_name = agent_info.get("name")
+                    agent_name = agent_info.get("name") or agent_info.get(
+                        "metadata", {}
+                    ).get("name")
                     if not agent_name:
                         continue
 
@@ -105,19 +279,87 @@ class MultiSourceAgentDeploymentService:
                     agent_info["source"] = source_name
                     agent_info["source_dir"] = str(source_dir)
 
+                    # NEW: Build canonical_id for enhanced matching
+                    canonical_id = self._build_canonical_id_for_agent(agent_info)
+
+                    # Group by canonical_id (PRIMARY) for enhanced matching
+                    # This allows matching agents from different sources with same canonical_id
+                    # while maintaining backward compatibility with name-based matching
+                    matching_key = canonical_id
+
                     # Initialize list if this is the first occurrence of this agent
-                    if agent_name not in agents_by_name:
-                        agents_by_name[agent_name] = []
+                    if matching_key not in agents_by_name:
+                        agents_by_name[matching_key] = []
 
-                    agents_by_name[agent_name].append(agent_info)
+                    agents_by_name[matching_key].append(agent_info)
 
                 # Use more specific log message
                 self.logger.info(
                     f"Discovered {len(agents)} {source_name} agent templates from {source_dir.name}"
                 )
 
+        # Show deprecation warning if user agents found
+        if user_agents_found:
+            self.logger.warning(
+                "\n"
+                "⚠️  DEPRECATION WARNING: User-level agents found in ~/.claude-mpm/agents/\n"
+                "   User-level agent deployment is deprecated and will be removed in v5.0.0\n"
+                "\n"
+                "   Why this change?\n"
+                "   - Project isolation: Agents should be project-specific\n"
+                "   - Version control: Project agents can be versioned with your code\n"
+                "   - Team consistency: All team members use the same agents\n"
+                "\n"
+                "   Migration:\n"
+                "   1. Run: claude-mpm agents migrate-to-project\n"
+                "   2. Verify agents work in .claude-mpm/agents/\n"
+                "   3. Remove: rm -rf ~/.claude-mpm/agents/\n"
+                "\n"
+                "   Learn more: https://docs.claude-mpm.dev/agents/migration\n"
+            )
+
         return agents_by_name
 
+    def get_agents_by_collection(
+        self,
+        collection_id: str,
+        agents_cache_dir: Optional[Path] = None,
+    ) -> List[Dict[str, Any]]:
+        """Get all agents from a specific collection.
+
+        NEW: Enables collection-based agent selection.
+
+        Args:
+            collection_id: Collection identifier (e.g., "bobmatnyc/claude-mpm-agents")
+            agents_cache_dir: Directory containing agents cache
+
+        Returns:
+            List of agent dictionaries from the specified collection
+
+        Example:
+            >>> service = MultiSourceAgentDeploymentService()
+            >>> agents = service.get_agents_by_collection("bobmatnyc/claude-mpm-agents")
+            >>> len(agents)
+            45
+        """
+        if not agents_cache_dir:
+            cache_dir = Path.home() / ".claude-mpm" / "cache"
+            agents_cache_dir = cache_dir / "agents"
+
+        if not agents_cache_dir.exists():
+            self.logger.warning(f"Agents cache directory not found: {agents_cache_dir}")
+            return []
+
+        # Use RemoteAgentDiscoveryService to get collection agents
+        remote_service = RemoteAgentDiscoveryService(agents_cache_dir)
+        collection_agents = remote_service.get_agents_by_collection(collection_id)
+
+        self.logger.info(
+            f"Retrieved {len(collection_agents)} agents from collection '{collection_id}'"
+        )
+
+        return collection_agents
+
     def select_highest_version_agents(
         self, agents_by_name: Dict[str, List[Dict[str, Any]]]
     ) -> Dict[str, Dict[str, Any]]:
@@ -226,17 +468,19 @@ class MultiSourceAgentDeploymentService:
         system_templates_dir: Optional[Path] = None,
         project_agents_dir: Optional[Path] = None,
         user_agents_dir: Optional[Path] = None,
+        agents_cache_dir: Optional[Path] = None,
         working_directory: Optional[Path] = None,
         excluded_agents: Optional[List[str]] = None,
         config: Optional[Config] = None,
         cleanup_outdated: bool = True,
     ) -> Tuple[Dict[str, Path], Dict[str, str], Dict[str, Any]]:
-        """Get the highest version agents from all sources for deployment.
+        """Get the highest version agents from all 4 tiers for deployment.
 
         Args:
             system_templates_dir: Directory containing system agent templates
             project_agents_dir: Directory containing project-specific agents
-            user_agents_dir: Directory containing user custom agents
+            user_agents_dir: Directory containing user custom agents (DEPRECATED)
+            agents_cache_dir: Directory containing cached agents from Git sources
             working_directory: Current working directory for finding project agents
             excluded_agents: List of agent names to exclude from deployment
             config: Configuration object for additional filtering
@@ -248,11 +492,12 @@ class MultiSourceAgentDeploymentService:
             - Dictionary mapping agent names to their source
             - Dictionary with cleanup results (removed, preserved, errors)
         """
-        # Discover all available agents
+        # Discover all available agents from 4 tiers
         agents_by_name = self.discover_agents_from_all_sources(
             system_templates_dir=system_templates_dir,
             project_agents_dir=project_agents_dir,
             user_agents_dir=user_agents_dir,
+            agents_cache_dir=agents_cache_dir,
             working_directory=working_directory,
         )
 
@@ -286,10 +531,28 @@ class MultiSourceAgentDeploymentService:
 
         # Apply exclusion filters
         if excluded_agents:
-            for agent_name in excluded_agents:
-                if agent_name in selected_agents:
-                    self.logger.info(f"Excluding agent '{agent_name}' from deployment")
-                    del selected_agents[agent_name]
+            # Find agents to remove by matching name field or agent_id portion of canonical_id
+            agents_to_remove = []
+            excluded_set = {name.lower() for name in excluded_agents}
+
+            for canonical_id, agent_info in list(selected_agents.items()):
+                # Check agent name field
+                agent_name = agent_info.get("name", "").lower()
+
+                # Also check the agent_id portion of canonical_id (after the colon)
+                # Example: "bobmatnyc/claude-mpm-agents:pm" -> "pm"
+                agent_id = canonical_id.split(":")[-1].lower() if ":" in canonical_id else canonical_id.lower()
+
+                if agent_name in excluded_set or agent_id in excluded_set:
+                    agents_to_remove.append(canonical_id)
+                    self.logger.info(
+                        f"Excluding agent '{agent_info.get('name', agent_id)}' "
+                        f"(canonical_id: {canonical_id}) from deployment"
+                    )
+
+            # Remove matched agents
+            for canonical_id in agents_to_remove:
+                del selected_agents[canonical_id]
 
         # Apply config-based filtering if provided
         if config:
@@ -300,7 +563,21 @@ class MultiSourceAgentDeploymentService:
         agent_sources = {}
 
         for agent_name, agent_info in selected_agents.items():
-            template_path = Path(agent_info["path"])
+            # Defensive: Try multiple path fields for backward compatibility (ticket 1M-480)
+            # Priority: 'path' -> 'file_path' -> 'source_file'
+            path_str = (
+                agent_info.get("path")
+                or agent_info.get("file_path")
+                or agent_info.get("source_file")
+            )
+
+            if not path_str:
+                self.logger.warning(
+                    f"Agent '{agent_name}' missing path information (no 'path', 'file_path', or 'source_file' field)"
+                )
+                continue
+
+            template_path = Path(path_str)
             if template_path.exists():
                 # Use the file stem as the key for consistency
                 file_stem = template_path.stem
@@ -324,6 +601,105 @@ class MultiSourceAgentDeploymentService:
 
         return agents_to_deploy, agent_sources, cleanup_results
 
+    def cleanup_excluded_agents(
+        self,
+        deployed_agents_dir: Path,
+        agents_to_deploy: Dict[str, Path],
+    ) -> Dict[str, Any]:
+        """Remove agents from deployed directory that aren't in the deployment list.
+
+        Similar to skill cleanup logic, this removes agents that were previously
+        deployed but are no longer in the enabled agents list (e.g., filtered out
+        by profile configuration).
+
+        Args:
+            deployed_agents_dir: Directory containing deployed agents (~/.claude/agents)
+            agents_to_deploy: Dictionary mapping agent file stems to template paths
+
+        Returns:
+            Dictionary with cleanup results:
+            - removed: List of removed agent names
+            - errors: List of errors during cleanup
+        """
+        cleanup_results = {"removed": [], "errors": []}
+
+        # Safety check - only operate on deployed agents directory
+        if not deployed_agents_dir.exists():
+            self.logger.debug("Deployed agents directory does not exist, no cleanup needed")
+            return cleanup_results
+
+        # Build set of agent names that should exist (file stems without .md extension)
+        expected_agents = set(agents_to_deploy.keys())
+
+        try:
+            # Check each file in deployed_agents_dir
+            for item in deployed_agents_dir.iterdir():
+                # Only process .md files
+                if not item.is_file() or item.suffix != ".md":
+                    continue
+
+                # Skip hidden files
+                if item.name.startswith("."):
+                    continue
+
+                # Get agent name (file stem)
+                agent_name = item.stem
+
+                # Check if this agent should be kept
+                if agent_name not in expected_agents:
+                    try:
+                        # Security: Validate path is within deployed_agents_dir
+                        resolved_item = item.resolve()
+                        resolved_target = deployed_agents_dir.resolve()
+
+                        if not str(resolved_item).startswith(str(resolved_target)):
+                            self.logger.error(
+                                f"Refusing to remove path outside target directory: {item}"
+                            )
+                            cleanup_results["errors"].append(
+                                {
+                                    "agent": agent_name,
+                                    "error": "Path outside target directory",
+                                }
+                            )
+                            continue
+
+                        # Remove the agent file
+                        item.unlink()
+                        cleanup_results["removed"].append(agent_name)
+                        self.logger.info(f"Removed excluded agent: {agent_name}")
+
+                    except PermissionError as e:
+                        error_msg = f"Permission denied removing {agent_name}: {e}"
+                        self.logger.error(error_msg)
+                        cleanup_results["errors"].append(
+                            {"agent": agent_name, "error": error_msg}
+                        )
+                    except Exception as e:
+                        error_msg = f"Error removing {agent_name}: {e}"
+                        self.logger.error(error_msg)
+                        cleanup_results["errors"].append(
+                            {"agent": agent_name, "error": error_msg}
+                        )
+
+        except Exception as e:
+            self.logger.error(f"Error during agent cleanup: {e}")
+            cleanup_results["errors"].append(
+                {"agent": "cleanup_process", "error": str(e)}
+            )
+
+        # Log cleanup summary
+        if cleanup_results["removed"]:
+            self.logger.info(
+                f"Cleanup complete: removed {len(cleanup_results['removed'])} excluded agents"
+            )
+        if cleanup_results["errors"]:
+            self.logger.warning(
+                f"Encountered {len(cleanup_results['errors'])} errors during cleanup"
+            )
+
+        return cleanup_results
+
     def cleanup_outdated_user_agents(
         self,
         agents_by_name: Dict[str, List[Dict[str, Any]]],
@@ -371,8 +747,20 @@ class MultiSourceAgentDeploymentService:
                 if agent_info["source"] != "user":
                     continue
 
+                # Defensive: Get path from agent_info (ticket 1M-480)
+                path_str = (
+                    agent_info.get("path")
+                    or agent_info.get("file_path")
+                    or agent_info.get("source_file")
+                )
+                if not path_str:
+                    self.logger.warning(
+                        f"User agent '{agent_name}' missing path information, skipping cleanup"
+                    )
+                    continue
+
                 # Safety check - ensure path is within user agents directory
-                user_agent_path = Path(agent_info["path"])
+                user_agent_path = Path(path_str)
                 try:
                     # Resolve paths to compare them safely
                     resolved_user_path = user_agent_path.resolve()
@@ -620,17 +1008,20 @@ class MultiSourceAgentDeploymentService:
                 comparison_results["needs_update"].append(agent_name)
                 continue
 
-            # Read template version
-            try:
-                template_data = json.loads(template_path.read_text())
-                metadata = template_data.get("metadata", {})
-                template_version = self.version_manager.parse_version(
-                    template_data.get("agent_version")
-                    or template_data.get("version")
-                    or metadata.get("version", "0.0.0")
+            # Read template version using format-aware helper
+            version_string = self._read_template_version(template_path)
+            if not version_string:
+                self.logger.warning(
+                    f"Could not extract version from template for '{agent_name}', skipping"
                 )
+                continue
+
+            try:
+                template_version = self.version_manager.parse_version(version_string)
             except Exception as e:
-                self.logger.warning(f"Error reading template for '{agent_name}': {e}")
+                self.logger.warning(
+                    f"Error parsing version '{version_string}' for '{agent_name}': {e}"
+                )
                 continue
 
             # Read deployed version

claude_mpm/services/agents/deployment/pipeline/pipeline_executor.py

@@ -51,11 +51,11 @@ class DeploymentPipelineExecutor:
             try:
                 # Check if step should be executed
                 if not step.should_execute(context):
-                    self.logger.info(f"Skipping step {i+1}/{len(steps)}: {step.name}")
+                    self.logger.info(f"Skipping step {i + 1}/{len(steps)}: {step.name}")
                     skipped_steps.append(step.name)
                     continue
 
-                self.logger.info(f"Executing step {i+1}/{len(steps)}: {step.name}")
+                self.logger.info(f"Executing step {i + 1}/{len(steps)}: {step.name}")
 
                 # Execute the step
                 result = step.execute(context)

claude_mpm/services/agents/deployment/refactored_agent_deployment_service.py

@@ -9,10 +9,7 @@ from claude_mpm.core.logger import get_logger
 
 from .config import DeploymentConfigManager
 from .facade import DeploymentFacade
-from .pipeline import (
-    DeploymentPipelineBuilder,
-    DeploymentPipelineExecutor,
-)
+from .pipeline import DeploymentPipelineBuilder, DeploymentPipelineExecutor
 from .results import DeploymentResultBuilder
 
 # Import refactored components