claude-mpm 4.21.3__py3-none-any.whl → 5.0.2__py3-none-any.whl

claude_mpm/services/agents/deployment/multi_source_deployment_service.py

@@ -21,19 +21,29 @@ from claude_mpm.core.logging_config import get_logger
 
 from .agent_discovery_service import AgentDiscoveryService
 from .agent_version_manager import AgentVersionManager
+from .remote_agent_discovery_service import RemoteAgentDiscoveryService
 
 
 class MultiSourceAgentDeploymentService:
     """Service for deploying agents from multiple sources with version comparison.
 
     This service ensures that the highest version of each agent is deployed,
-    regardless of whether it comes from system templates, project agents, or
-    user agents.
+    regardless of whether it comes from system templates, project agents,
+    user agents, or remote agents.
+
+    4-Tier Agent Discovery:
+    1. System templates (lowest priority) - Built-in agents
+    2. User agents (DEPRECATED) - User-level customizations (~/.claude-mpm/agents/)
+    3. Remote agents - Agents cached from GitHub
+    4. Project agents (highest priority) - Project-specific customizations
 
     WHY: The current system processes agents from a single source at a time,
     which can result in lower version agents being deployed if they exist in
     a higher priority source. This service fixes that by comparing versions
     across all sources.
+
+    DEPRECATION: User-level agents (~/.claude-mpm/agents/) are deprecated and
+    will be removed in v5.0.0. Use project-level agents instead.
     """
 
     def __init__(self):
@@ -46,18 +56,30 @@ class MultiSourceAgentDeploymentService:
         system_templates_dir: Optional[Path] = None,
         project_agents_dir: Optional[Path] = None,
         user_agents_dir: Optional[Path] = None,
+        remote_agents_dir: Optional[Path] = None,
         working_directory: Optional[Path] = None,
     ) -> Dict[str, List[Dict[str, Any]]]:
-        """Discover agents from all available sources.
+        """Discover agents from all 4 tiers (system, user, remote, project).
+
+        Priority hierarchy (highest to lowest):
+        4. Project agents - Highest priority, project-specific customizations
+        3. Remote agents - GitHub-synced agents from cache
+        2. User agents - DEPRECATED, user-level customizations
+        1. System templates - Lowest priority, built-in agents
 
         Args:
             system_templates_dir: Directory containing system agent templates
             project_agents_dir: Directory containing project-specific agents
-            user_agents_dir: Directory containing user custom agents
+            user_agents_dir: Directory containing user custom agents (DEPRECATED)
+            remote_agents_dir: Directory containing cached remote agents
             working_directory: Current working directory for finding project agents
 
         Returns:
             Dictionary mapping agent names to list of agent info from different sources
+
+        Deprecation Warning:
+            User-level agents are deprecated and will show a warning if found.
+            Use 'claude-mpm agents migrate-to-project' to migrate them.
         """
         agents_by_name = {}
 
@@ -80,24 +102,53 @@ class MultiSourceAgentDeploymentService:
             if not user_agents_dir.exists():
                 user_agents_dir = None
 
-        # Discover agents from each source
+        if not remote_agents_dir:
+            # Check for remote agents in cache directory
+            cache_dir = Path.home() / ".claude-mpm" / "cache"
+            remote_agents_dir = cache_dir / "remote-agents"
+            if not remote_agents_dir.exists():
+                remote_agents_dir = None
+
+        # Discover agents from each source in priority order
+        # Note: We process in reverse priority order (system first) and build up the dictionary
+        # The select_highest_version_agents() method will handle the actual prioritization
         sources = [
             ("system", system_templates_dir),
-            ("project", project_agents_dir),
             ("user", user_agents_dir),
+            ("remote", remote_agents_dir),
+            ("project", project_agents_dir),
         ]
 
+        # Track if we found user agents for deprecation warning
+        user_agents_found = False
+
         for source_name, source_dir in sources:
             if source_dir and source_dir.exists():
                 self.logger.debug(
                     f"Discovering agents from {source_name} source: {source_dir}"
                 )
-                discovery_service = AgentDiscoveryService(source_dir)
-                # Pass log_discovery=False to avoid duplicate logging
-                agents = discovery_service.list_available_agents(log_discovery=False)
+
+                # Use appropriate discovery service based on source type
+                if source_name == "remote":
+                    # Remote agents are Markdown, use RemoteAgentDiscoveryService
+                    remote_service = RemoteAgentDiscoveryService(source_dir)
+                    agents = remote_service.discover_remote_agents()
+                else:
+                    # Other sources are JSON, use AgentDiscoveryService
+                    discovery_service = AgentDiscoveryService(source_dir)
+                    # Pass log_discovery=False to avoid duplicate logging
+                    agents = discovery_service.list_available_agents(
+                        log_discovery=False
+                    )
+
+                # Track user agents for deprecation warning
+                if source_name == "user" and agents:
+                    user_agents_found = True
 
                 for agent_info in agents:
-                    agent_name = agent_info.get("name")
+                    agent_name = agent_info.get("name") or agent_info.get(
+                        "metadata", {}
+                    ).get("name")
                     if not agent_name:
                         continue
 
@@ -116,6 +167,26 @@ class MultiSourceAgentDeploymentService:
                     f"Discovered {len(agents)} {source_name} agent templates from {source_dir.name}"
                 )
 
+        # Show deprecation warning if user agents found
+        if user_agents_found:
+            self.logger.warning(
+                "\n"
+                "⚠️  DEPRECATION WARNING: User-level agents found in ~/.claude-mpm/agents/\n"
+                "   User-level agent deployment is deprecated and will be removed in v5.0.0\n"
+                "\n"
+                "   Why this change?\n"
+                "   - Project isolation: Agents should be project-specific\n"
+                "   - Version control: Project agents can be versioned with your code\n"
+                "   - Team consistency: All team members use the same agents\n"
+                "\n"
+                "   Migration:\n"
+                "   1. Run: claude-mpm agents migrate-to-project\n"
+                "   2. Verify agents work in .claude-mpm/agents/\n"
+                "   3. Remove: rm -rf ~/.claude-mpm/agents/\n"
+                "\n"
+                "   Learn more: https://docs.claude-mpm.dev/agents/migration\n"
+            )
+
         return agents_by_name
 
     def select_highest_version_agents(
@@ -226,17 +297,19 @@ class MultiSourceAgentDeploymentService:
         system_templates_dir: Optional[Path] = None,
         project_agents_dir: Optional[Path] = None,
         user_agents_dir: Optional[Path] = None,
+        remote_agents_dir: Optional[Path] = None,
         working_directory: Optional[Path] = None,
         excluded_agents: Optional[List[str]] = None,
         config: Optional[Config] = None,
         cleanup_outdated: bool = True,
     ) -> Tuple[Dict[str, Path], Dict[str, str], Dict[str, Any]]:
-        """Get the highest version agents from all sources for deployment.
+        """Get the highest version agents from all 4 tiers for deployment.
 
         Args:
             system_templates_dir: Directory containing system agent templates
             project_agents_dir: Directory containing project-specific agents
-            user_agents_dir: Directory containing user custom agents
+            user_agents_dir: Directory containing user custom agents (DEPRECATED)
+            remote_agents_dir: Directory containing cached remote agents
             working_directory: Current working directory for finding project agents
             excluded_agents: List of agent names to exclude from deployment
             config: Configuration object for additional filtering
@@ -248,11 +321,12 @@ class MultiSourceAgentDeploymentService:
             - Dictionary mapping agent names to their source
             - Dictionary with cleanup results (removed, preserved, errors)
         """
-        # Discover all available agents
+        # Discover all available agents from 4 tiers
         agents_by_name = self.discover_agents_from_all_sources(
             system_templates_dir=system_templates_dir,
             project_agents_dir=project_agents_dir,
             user_agents_dir=user_agents_dir,
+            remote_agents_dir=remote_agents_dir,
             working_directory=working_directory,
         )
 
@@ -300,7 +374,21 @@ class MultiSourceAgentDeploymentService:
         agent_sources = {}
 
         for agent_name, agent_info in selected_agents.items():
-            template_path = Path(agent_info["path"])
+            # Defensive: Try multiple path fields for backward compatibility (ticket 1M-480)
+            # Priority: 'path' -> 'file_path' -> 'source_file'
+            path_str = (
+                agent_info.get("path")
+                or agent_info.get("file_path")
+                or agent_info.get("source_file")
+            )
+
+            if not path_str:
+                self.logger.warning(
+                    f"Agent '{agent_name}' missing path information (no 'path', 'file_path', or 'source_file' field)"
+                )
+                continue
+
+            template_path = Path(path_str)
             if template_path.exists():
                 # Use the file stem as the key for consistency
                 file_stem = template_path.stem
@@ -371,8 +459,20 @@ class MultiSourceAgentDeploymentService:
                 if agent_info["source"] != "user":
                     continue
 
+                # Defensive: Get path from agent_info (ticket 1M-480)
+                path_str = (
+                    agent_info.get("path")
+                    or agent_info.get("file_path")
+                    or agent_info.get("source_file")
+                )
+                if not path_str:
+                    self.logger.warning(
+                        f"User agent '{agent_name}' missing path information, skipping cleanup"
+                    )
+                    continue
+
                 # Safety check - ensure path is within user agents directory
-                user_agent_path = Path(agent_info["path"])
+                user_agent_path = Path(path_str)
                 try:
                     # Resolve paths to compare them safely
                     resolved_user_path = user_agent_path.resolve()

claude_mpm/services/agents/deployment/pipeline/pipeline_executor.py

@@ -51,11 +51,11 @@ class DeploymentPipelineExecutor:
             try:
                 # Check if step should be executed
                 if not step.should_execute(context):
-                    self.logger.info(f"Skipping step {i+1}/{len(steps)}: {step.name}")
+                    self.logger.info(f"Skipping step {i + 1}/{len(steps)}: {step.name}")
                     skipped_steps.append(step.name)
                     continue
 
-                self.logger.info(f"Executing step {i+1}/{len(steps)}: {step.name}")
+                self.logger.info(f"Executing step {i + 1}/{len(steps)}: {step.name}")
 
                 # Execute the step
                 result = step.execute(context)

claude_mpm/services/agents/deployment/refactored_agent_deployment_service.py

@@ -9,10 +9,7 @@ from claude_mpm.core.logger import get_logger
 
 from .config import DeploymentConfigManager
 from .facade import DeploymentFacade
-from .pipeline import (
-    DeploymentPipelineBuilder,
-    DeploymentPipelineExecutor,
-)
+from .pipeline import DeploymentPipelineBuilder, DeploymentPipelineExecutor
 from .results import DeploymentResultBuilder
 
 # Import refactored components

claude_mpm/services/agents/deployment/remote_agent_discovery_service.py

@@ -0,0 +1,363 @@
+"""Service for discovering and converting remote Markdown agents to JSON format.
+
+This service handles the 4th tier of agent discovery: remote agents cached from GitHub.
+Remote agents are stored as Markdown files with YAML frontmatter and need to be converted
+to the JSON template format expected by the deployment system.
+
+WHY: Remote agents from GitHub are cached as Markdown but the deployment system expects
+JSON templates. This service bridges that gap and integrates remote agents into the
+multi-tier discovery system.
+"""
+
+import json
+import re
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+from claude_mpm.core.logging_config import get_logger
+
+logger = get_logger(__name__)
+
+
+@dataclass
+class RemoteAgentMetadata:
+    """Metadata extracted from remote agent Markdown file."""
+
+    name: str
+    description: str
+    model: str
+    routing_keywords: List[str]
+    routing_paths: List[str]
+    routing_priority: int
+    source_file: Path
+    version: str  # SHA-256 hash from cache metadata
+
+
+class RemoteAgentDiscoveryService:
+    """Discovers and converts remote Markdown agents to JSON format.
+
+    Remote agents are discovered from the cache directory (~/.claude-mpm/cache/remote-agents/)
+    where they are stored as Markdown files. This service:
+    1. Discovers all *.md files in the remote agents cache
+    2. Parses Markdown frontmatter and content to extract metadata
+    3. Converts to JSON template format for deployment
+    4. Retrieves version (SHA-256 hash) from cache metadata
+
+    Design Decision: Markdown Parsing Strategy
+    - Use regex for simple frontmatter extraction (fast, no dependencies)
+    - Parse key-value pairs from Configuration section
+    - Extract routing info from Routing section
+    - Fallback to sensible defaults when sections are missing
+
+    Trade-offs:
+    - Performance: Regex parsing is fast for our simple format
+    - Maintainability: Clear regex patterns are easy to understand
+    - Flexibility: Supports optional sections with defaults
+    """
+
+    def __init__(self, remote_agents_dir: Path):
+        """Initialize the remote agent discovery service.
+
+        Args:
+            remote_agents_dir: Directory containing cached remote agent Markdown files
+        """
+        self.remote_agents_dir = remote_agents_dir
+        self.logger = get_logger(__name__)
+
+    def _generate_hierarchical_id(self, file_path: Path) -> str:
+        """Generate hierarchical agent ID from file path.
+
+        Converts file path relative to agents subdirectory into hierarchical ID.
+
+        Design Decision: Path-based IDs for hierarchy preservation
+
+        Rationale: Agent IDs must reflect directory hierarchy to enable:
+        - Category-based filtering (engineer/backend/python-engineer)
+        - Preset matching against AUTO-DEPLOY-INDEX.md
+        - Multi-level organization without name collisions
+
+        Bug #4 Fix: Calculate relative to /agents/ subdirectory, not repository root
+        This ensures agent IDs are based on their position within the agents directory.
+
+        Example:
+            Input:  /cache/bobmatnyc/claude-mpm-agents/agents/engineer/backend/python-engineer.md
+            Root:   /cache/bobmatnyc/claude-mpm-agents/agents
+            Output: engineer/backend/python-engineer
+
+        Args:
+            file_path: Absolute path to agent Markdown file
+
+        Returns:
+            Hierarchical agent ID with forward slashes
+        """
+        try:
+            # Calculate relative to /agents/ subdirectory (Bug #4 fix)
+            agents_dir = self.remote_agents_dir / "agents"
+            relative_path = file_path.relative_to(agents_dir)
+
+            # Remove .md extension and convert to forward slashes
+            return str(relative_path.with_suffix("")).replace("\\", "/")
+        except ValueError:
+            # File is not under agents subdirectory, fall back to filename
+            self.logger.warning(
+                f"File {file_path} not under agents directory, using filename"
+            )
+            return file_path.stem
+
+    def _detect_category_from_path(self, file_path: Path) -> str:
+        """Detect category from file path hierarchy.
+
+        Extracts category from directory structure. Category is the path
+        from agents subdirectory to the file, excluding the filename.
+
+        Bug #4 Fix: Calculate relative to /agents/ subdirectory, not repository root
+        This ensures categories are based on agent organization within /agents/.
+
+        Example:
+            Input:  /cache/bobmatnyc/claude-mpm-agents/agents/engineer/backend/python-engineer.md
+            Root:   /cache/bobmatnyc/claude-mpm-agents/agents
+            Output: engineer/backend
+
+        Args:
+            file_path: Absolute path to agent Markdown file
+
+        Returns:
+            Category path with forward slashes, or "universal" if in root
+        """
+        try:
+            # Calculate relative to /agents/ subdirectory (Bug #4 fix)
+            agents_dir = self.remote_agents_dir / "agents"
+            relative_path = file_path.relative_to(agents_dir)
+            parts = relative_path.parts[:-1]  # Exclude filename
+            return "/".join(parts) if parts else "universal"
+        except ValueError:
+            return "universal"
+
+    def discover_remote_agents(self) -> List[Dict[str, Any]]:
+        """Discover all remote agents from cache directory.
+
+        Scans the remote agents directory for *.md files recursively and converts each
+        to JSON template format. Skips files that can't be parsed.
+
+        Bug #4 Fix: Only scan /agents/ subdirectory, not entire repository
+        This prevents README.md, CHANGELOG.md, etc. from being treated as agents.
+
+        Returns:
+            List of agent dictionaries in JSON template format
+
+        Example:
+            >>> service = RemoteAgentDiscoveryService(Path("~/.claude-mpm/cache/remote-agents"))
+            >>> agents = service.discover_remote_agents()
+            >>> len(agents)
+            5
+            >>> agents[0]['name']
+            'Security Scanner Agent'
+        """
+        agents = []
+
+        if not self.remote_agents_dir.exists():
+            self.logger.debug(
+                f"Remote agents directory does not exist: {self.remote_agents_dir}"
+            )
+            return agents
+
+        # Bug #4 Fix: Only scan /agents/ subdirectory, not entire repository
+        # This prevents non-agent files (README.md, CHANGELOG.md, etc.) from polluting results
+        agents_dir = self.remote_agents_dir / "agents"
+
+        if not agents_dir.exists():
+            self.logger.warning(
+                f"Agents subdirectory not found: {agents_dir}. "
+                f"Expected agents to be in /agents/ subdirectory."
+            )
+            return agents
+
+        # Find all Markdown files recursively in /agents/ subdirectory only
+        md_files = list(agents_dir.rglob("*.md"))
+        self.logger.debug(f"Found {len(md_files)} Markdown files in {agents_dir}")
+
+        for md_file in md_files:
+            try:
+                agent_dict = self._parse_markdown_agent(md_file)
+                if agent_dict:
+                    agents.append(agent_dict)
+                    self.logger.debug(
+                        f"Successfully parsed remote agent: {md_file.name}"
+                    )
+                else:
+                    self.logger.warning(
+                        f"Failed to parse remote agent (no name found): {md_file.name}"
+                    )
+            except Exception as e:
+                self.logger.warning(f"Failed to parse remote agent {md_file.name}: {e}")
+
+        self.logger.info(
+            f"Discovered {len(agents)} remote agents from {self.remote_agents_dir.name}"
+        )
+        return agents
+
+    def _parse_markdown_agent(self, md_file: Path) -> Optional[Dict[str, Any]]:
+        """Parse Markdown agent file and convert to JSON template format.
+
+        Expected Markdown format:
+        ```markdown
+        # Agent Name
+
+        Description paragraph (first paragraph after heading)
+
+        ## Configuration
+        - Model: sonnet
+        - Priority: 100
+
+        ## Routing
+        - Keywords: keyword1, keyword2
+        - Paths: /path1/, /path2/
+        ```
+
+        Args:
+            md_file: Path to Markdown agent file
+
+        Returns:
+            Agent dictionary in JSON template format, or None if parsing fails
+
+        Error Handling:
+        - Returns None if agent name (first heading) is missing
+        - Uses defaults for missing sections (model=sonnet, priority=50)
+        - Empty routing keywords/paths if Routing section missing
+        """
+        try:
+            content = md_file.read_text(encoding="utf-8")
+        except Exception as e:
+            self.logger.error(f"Failed to read file {md_file}: {e}")
+            return None
+
+        # Extract agent name from first heading
+        name_match = re.search(r"^#\s+(.+?)$", content, re.MULTILINE)
+        if not name_match:
+            self.logger.debug(f"No agent name heading found in {md_file.name}")
+            return None
+        name = name_match.group(1).strip()
+
+        # Extract description (first paragraph after heading, before next heading)
+        desc_match = re.search(
+            r"^#.+?\n\n(.+?)(?:\n\n##|\Z)", content, re.DOTALL | re.MULTILINE
+        )
+        description = desc_match.group(1).strip() if desc_match else ""
+
+        # Extract model from Configuration section
+        model_match = re.search(r"Model:\s*(\w+)", content, re.IGNORECASE)
+        model = model_match.group(1) if model_match else "sonnet"
+
+        # Extract priority from Configuration section
+        priority_match = re.search(r"Priority:\s*(\d+)", content, re.IGNORECASE)
+        priority = int(priority_match.group(1)) if priority_match else 50
+
+        # Extract routing keywords
+        keywords_match = re.search(r"Keywords:\s*(.+?)(?:\n|$)", content, re.IGNORECASE)
+        keywords = []
+        if keywords_match:
+            keywords = [k.strip() for k in keywords_match.group(1).split(",")]
+
+        # Extract routing paths
+        paths_match = re.search(r"Paths:\s*(.+?)(?:\n|$)", content, re.IGNORECASE)
+        paths = []
+        if paths_match:
+            paths = [p.strip() for p in paths_match.group(1).split(",")]
+
+        # Get version (SHA-256 hash) from cache metadata
+        version = self._get_agent_version(md_file)
+
+        # Bug #3 fix: Generate hierarchical agent_id from file path
+        # This preserves directory structure for category filtering and preset matching
+        agent_id = self._generate_hierarchical_id(md_file)
+
+        # Bug #1 fix: Detect category from directory path
+        category = self._detect_category_from_path(md_file)
+
+        # Convert to JSON template format and return
+        # IMPORTANT: Include 'path' field for compatibility with deployment validation (ticket 1M-480)
+        # Git-sourced agents must have 'path' field to match structure from AgentDiscoveryService
+        return {
+            "agent_id": agent_id,
+            "metadata": {
+                "name": name,
+                "description": description,
+                "version": version,
+                "author": "remote",  # Mark as remote agent
+                "category": category,  # Use detected category from path
+            },
+            "model": model,
+            "source": "remote",  # Mark as remote agent
+            "source_file": str(md_file),
+            "path": str(
+                md_file
+            ),  # Add 'path' field for deployment compatibility (1M-480)
+            "file_path": str(md_file),  # Keep for backward compatibility
+            "version": version,  # Include at root level for version comparison
+            "category": category,  # Add category at root level for filtering
+            "routing": {"keywords": keywords, "paths": paths, "priority": priority},
+        }
+
+    def _get_agent_version(self, md_file: Path) -> str:
+        """Get version (SHA-256 hash) from cache metadata.
+
+        Looks for corresponding .meta.json file in cache directory that contains
+        the SHA-256 hash of the agent content.
+
+        Args:
+            md_file: Path to Markdown agent file
+
+        Returns:
+            SHA-256 hash from metadata, or 'unknown' if not found
+
+        Example metadata file:
+            {
+                "content_hash": "abc123...",
+                "etag": "W/\"abc123\"",
+                "last_modified": "2025-11-30T10:00:00Z"
+            }
+        """
+        # Look for .meta.json file
+        meta_file = md_file.with_suffix(".md.meta.json")
+
+        if not meta_file.exists():
+            self.logger.debug(f"No metadata file found for {md_file.name}")
+            return "unknown"
+
+        try:
+            meta_data = json.loads(meta_file.read_text(encoding="utf-8"))
+            content_hash = meta_data.get("content_hash", "unknown")
+            self.logger.debug(
+                f"Retrieved version {content_hash[:8]}... for {md_file.name}"
+            )
+            return content_hash
+        except Exception as e:
+            self.logger.warning(f"Failed to read metadata for {md_file.name}: {e}")
+            return "unknown"
+
+    def get_remote_agent_metadata(
+        self, agent_name: str
+    ) -> Optional[RemoteAgentMetadata]:
+        """Get metadata for a specific remote agent.
+
+        Args:
+            agent_name: Name of the agent to retrieve
+
+        Returns:
+            RemoteAgentMetadata if found, None otherwise
+        """
+        for md_file in self.remote_agents_dir.glob("*.md"):
+            agent_dict = self._parse_markdown_agent(md_file)
+            if agent_dict and agent_dict["metadata"]["name"] == agent_name:
+                return RemoteAgentMetadata(
+                    name=agent_dict["metadata"]["name"],
+                    description=agent_dict["metadata"]["description"],
+                    model=agent_dict["model"],
+                    routing_keywords=agent_dict["routing"]["keywords"],
+                    routing_paths=agent_dict["routing"]["paths"],
+                    routing_priority=agent_dict["routing"]["priority"],
+                    source_file=Path(agent_dict["source_file"]),
+                    version=agent_dict["version"],
+                )
+        return None

claude_mpm/services/agents/deployment/single_agent_deployer.py

@@ -200,8 +200,8 @@ class SingleAgentDeployer:
         - Enables selective agent management in projects
         """
         try:
-            # Find the template file
-            template_file = templates_dir / f"{agent_name}.json"
+            # Find the template file (templates migrated to .md in v4.26.0+)
+            template_file = templates_dir / f"{agent_name}.md"
             if not template_file.exists():
                 self.logger.error(f"Agent template not found: {agent_name}")
                 return False