claude-mpm 4.24.0__py3-none-any.whl → 5.4.41__py3-none-any.whl

claude_mpm/services/agents/deployment/agent_deployment.py

@@ -31,6 +31,7 @@ import time
 from pathlib import Path
 from typing import Any, Dict, List, Optional, Tuple
 
+from claude_mpm.config.agent_sources import AgentSourceConfiguration
 from claude_mpm.config.paths import paths
 from claude_mpm.constants import Paths
 from claude_mpm.core.config import Config
@@ -38,6 +39,8 @@ from claude_mpm.core.exceptions import AgentDeploymentError
 from claude_mpm.core.interfaces import AgentDeploymentInterface
 from claude_mpm.services.shared import ConfigServiceBase
 
+# Import git source management for remote agent sync
+from ..git_source_manager import GitSourceManager
 from .agent_configuration_manager import AgentConfigurationManager
 from .agent_discovery_service import AgentDiscoveryService
 from .agent_environment_manager import AgentEnvironmentManager
@@ -184,9 +187,110 @@ class AgentDeploymentService(ConfigServiceBase, AgentDeploymentInterface):
         # Initialize format converter service
         self.format_converter = AgentFormatConverter()
 
+        # Initialize git source manager for remote agent sync
+        self.git_source_manager = GitSourceManager()
+        self.agent_source_config = AgentSourceConfiguration.load()
+
         self.logger.info(f"Templates directory: {self.templates_dir}")
         self.logger.info(f"Base agent path: {self.base_agent_path}")
 
+    def _sync_remote_agent_sources(self, timeout_seconds: int = 30) -> Dict[str, Any]:
+        """Sync git-based agent sources before deployment.
+
+        This method follows the skills system pattern: sync configured git repositories
+        to cache before discovery. Network failures are logged but don't block deployment.
+
+        Args:
+            timeout_seconds: Timeout for git operations (default: 30 seconds)
+
+        Returns:
+            Dictionary with sync results:
+            {
+                "synced_count": int,
+                "failed_count": int,
+                "repositories": Dict[str, Dict],  # repo_id -> sync result
+                "duration_ms": float
+            }
+
+        Error Handling:
+            - Network failures: Logged as warnings, sync continues
+            - Invalid repos: Logged as errors, sync continues
+            - Timeout: Individual repo timeouts don't stop overall sync
+            - Missing cache dir: Created automatically
+        """
+        import time
+
+        start_time = time.time()
+
+        results = {
+            "synced_count": 0,
+            "failed_count": 0,
+            "repositories": {},
+            "duration_ms": 0,
+        }
+
+        # Load agent sources configuration
+        try:
+            config = AgentSourceConfiguration.load()
+            enabled_repos = [r for r in config.repositories if r.enabled]
+
+            if not enabled_repos:
+                self.logger.debug("No enabled agent sources configured")
+                return results
+
+            self.logger.info(f"Syncing {len(enabled_repos)} agent git sources...")
+
+            # Sync each enabled repository
+            for repo in enabled_repos:
+                repo_id = repo.identifier
+                try:
+                    # Sync with timeout (individual repo sync)
+                    # NOTE: show_progress=False to avoid duplicate progress bars
+                    # (startup sync already showed progress to user)
+                    sync_result = self.git_source_manager.sync_repository(
+                        repo,
+                        force=False,  # Use ETag-based caching
+                        show_progress=False,  # Suppress progress (startup already synced)
+                    )
+
+                    results["repositories"][repo_id] = sync_result
+
+                    if sync_result.get("synced"):
+                        results["synced_count"] += 1
+                        agents_discovered = sync_result.get("agents_discovered", [])
+                        self.logger.info(
+                            f"Synced {repo_id}: {sync_result.get('files_updated', 0)} files, "
+                            f"{len(agents_discovered)} agents"
+                        )
+                    else:
+                        results["failed_count"] += 1
+                        error = sync_result.get("error", "Unknown error")
+                        self.logger.warning(f"Failed to sync {repo_id}: {error}")
+
+                except Exception as e:
+                    # Don't let individual repo failures stop deployment
+                    results["failed_count"] += 1
+                    results["repositories"][repo_id] = {
+                        "synced": False,
+                        "error": str(e),
+                    }
+                    self.logger.warning(f"Exception syncing {repo_id}: {e}")
+
+        except Exception as e:
+            # Configuration loading failure - log but don't crash
+            self.logger.warning(f"Failed to load agent sources config: {e}")
+            results["failed_count"] = -1  # Indicates config failure
+
+        results["duration_ms"] = (time.time() - start_time) * 1000
+
+        if results["synced_count"] > 0:
+            self.logger.info(
+                f"Agent source sync complete: {results['synced_count']} succeeded, "
+                f"{results['failed_count']} failed ({results['duration_ms']:.0f}ms)"
+            )
+
+        return results
+
     def deploy_agents(
         self,
         target_dir: Optional[Path] = None,
@@ -267,6 +371,10 @@ class AgentDeploymentService(ConfigServiceBase, AgentDeploymentInterface):
         # METRICS: Record deployment start time for performance tracking
         deployment_start_time = time.time()
 
+        # PHASE 2 (1M-442): Sync git-based agent sources before deployment
+        # This ensures remote agents from configured sources are cached and discoverable
+        sync_results = self._sync_remote_agent_sources()
+
         # Try async deployment for better performance if requested
         if use_async:
             async_results = self._try_async_deployment(
@@ -292,6 +400,13 @@ class AgentDeploymentService(ConfigServiceBase, AgentDeploymentInterface):
             agents_dir, deployment_start_time
         )
 
+        # Add git source sync results to deployment results
+        if (
+            sync_results.get("synced_count", 0) > 0
+            or sync_results.get("failed_count", 0) > 0
+        ):
+            results["remote_sources"] = sync_results
+
         try:
             # Create agents directory if needed
             agents_dir.mkdir(parents=True, exist_ok=True)
@@ -754,19 +869,28 @@ class AgentDeploymentService(ConfigServiceBase, AgentDeploymentInterface):
                 project_agents_dir = potential_project_dir
                 self.logger.info(f"Found project agents at: {project_agents_dir}")
 
-        # Check for user agents
+        # Check for user agents (DEPRECATED)
         user_home = Path.home()
         potential_user_dir = user_home / ".claude-mpm" / "agents"
         if potential_user_dir.exists():
             user_agents_dir = potential_user_dir
             self.logger.info(f"Found user agents at: {user_agents_dir}")
 
-        # Get agents with version comparison and cleanup
+        # Check for agents cache (from Git sources)
+        agents_cache_dir = None
+        cache_dir = user_home / ".claude-mpm" / "cache"
+        potential_cache_dir = cache_dir / "agents"
+        if potential_cache_dir.exists():
+            agents_cache_dir = potential_cache_dir
+            self.logger.info(f"Found agents cache at: {agents_cache_dir}")
+
+        # Get agents with version comparison and cleanup (4-tier discovery)
         agents_to_deploy, agent_sources, cleanup_results = (
             self.multi_source_service.get_agents_for_deployment(
                 system_templates_dir=system_templates_dir,
                 project_agents_dir=project_agents_dir,
                 user_agents_dir=user_agents_dir,
+                agents_cache_dir=agents_cache_dir,  # NEW: 4th tier
                 working_directory=self.working_directory,
                 excluded_agents=excluded_agents,
                 config=config,
@@ -774,6 +898,9 @@ class AgentDeploymentService(ConfigServiceBase, AgentDeploymentInterface):
             )
         )
 
+        # Keep track of all enabled agents before filtering (for cleanup)
+        all_enabled_agents = agents_to_deploy.copy()
+
         # Compare with deployed versions if agents directory exists
         if agents_dir.exists():
             comparison_results = self.multi_source_service.compare_deployed_versions(
@@ -830,6 +957,25 @@ class AgentDeploymentService(ConfigServiceBase, AgentDeploymentInterface):
                         f"All {len(comparison_results.get('up_to_date', []))} agents are up to date"
                     )
 
+        # Cleanup excluded agents (remove agents not in deployment list)
+        # CRITICAL: Use all_enabled_agents (before filtering for updates) to preserve up-to-date agents
+        # Bug fix (1M-XXX): Previously used filtered agents_to_deploy which could be empty,
+        # causing all agents to be removed when everything was up-to-date
+        exclusion_cleanup_results = self.multi_source_service.cleanup_excluded_agents(
+            deployed_agents_dir=agents_dir,
+            agents_to_deploy=all_enabled_agents,
+        )
+
+        # Add exclusion cleanup results to main cleanup results
+        if exclusion_cleanup_results.get("removed"):
+            cleanup_results.setdefault("excluded_removed", []).extend(
+                exclusion_cleanup_results["removed"]
+            )
+            self.logger.info(
+                f"Removed {len(exclusion_cleanup_results['removed'])} excluded agents: "
+                f"{', '.join(exclusion_cleanup_results['removed'])}"
+            )
+
         # Convert to list of Path objects
         template_files = list(agents_to_deploy.values())
 

claude_mpm/services/agents/deployment/agent_discovery_service.py

@@ -7,10 +7,11 @@ Extracted from AgentDeploymentService as part of the refactoring to improve
 maintainability and testability.
 """
 
-import json
 from pathlib import Path
 from typing import Any, Dict, List, Optional
 
+import yaml
+
 from claude_mpm.core.config import Config
 from claude_mpm.core.logging_config import get_logger
 
@@ -59,8 +60,8 @@ class AgentDiscoveryService:
             )
             return agents
 
-        # Find all JSON template files
-        template_files = list(self.templates_dir.glob("*.json"))
+        # Find all markdown template files with YAML frontmatter
+        template_files = list(self.templates_dir.glob("*.md"))
 
         for template_file in template_files:
             try:
@@ -105,8 +106,8 @@ class AgentDiscoveryService:
             self.logger.error(f"Templates directory not found: {self.templates_dir}")
             return []
 
-        # Get all template files
-        template_files = list(self.templates_dir.glob("*.json"))
+        # Get all markdown template files
+        template_files = list(self.templates_dir.glob("*.md"))
 
         if not template_files:
             self.logger.warning(f"No agent templates found in {self.templates_dir}")
@@ -157,7 +158,7 @@ class AgentDiscoveryService:
         Returns:
             Path to template file if found, None otherwise
         """
-        template_file = self.templates_dir / f"{agent_name}.json"
+        template_file = self.templates_dir / f"{agent_name}.md"
 
         if template_file.exists():
             if self._validate_template_file(template_file):
@@ -199,59 +200,44 @@ class AgentDiscoveryService:
 
     def _extract_agent_metadata(self, template_file: Path) -> Optional[Dict[str, Any]]:
         """
-        Extract metadata from an agent template file.
+        Extract metadata from an agent template file with YAML frontmatter.
 
         Args:
-            template_file: Path to the template file
+            template_file: Path to the markdown template file
 
         Returns:
             Dictionary with agent metadata or None if extraction fails
         """
         try:
-            # Read and parse template file
+            # Read template file content
             template_content = template_file.read_text()
-            template_data = json.loads(template_content)
-
-            # Extract basic metadata from the metadata section (per agent schema)
-            metadata = template_data.get("metadata", {})
-            capabilities = template_data.get("capabilities", {})
-
-            # Handle capabilities as either dict or list
-            if isinstance(capabilities, list):
-                # If capabilities is a list (like in php-engineer.json), treat it as capabilities list
-                tools_list = template_data.get(
-                    "tools", []
-                )  # Look for tools at root level
-                model_value = template_data.get("model", "sonnet")
-            else:
-                # If capabilities is a dict, extract tools and model from it
-                tools_list = capabilities.get("tools", [])
-                model_value = capabilities.get("model", "sonnet")
 
+            # Extract YAML frontmatter
+            frontmatter = self._extract_yaml_frontmatter(template_content)
+            if not frontmatter:
+                # Silently return None for files without frontmatter
+                # (e.g., PM instruction templates in templates/ directory)
+                return None
+
+            # Extract metadata directly from frontmatter (flat structure)
+            # Markdown templates use flat YAML structure, not nested "metadata" section
             agent_info = {
-                "name": metadata.get("name", template_file.stem),
-                "description": metadata.get(
-                    "description",
-                    template_data.get("description", "No description available"),
+                "name": frontmatter.get("name", template_file.stem),
+                "description": frontmatter.get(
+                    "description", "No description available"
                 ),
-                "type": template_data.get(
-                    "agent_type",
-                    metadata.get("category", template_data.get("category", "agent")),
-                ),  # Extract agent type
-                "version": template_data.get(
-                    "agent_version",
-                    template_data.get("version", metadata.get("version", "1.0.0")),
+                "type": frontmatter.get(
+                    "agent_type", frontmatter.get("category", "agent")
                 ),
-                "tools": tools_list,
-                "specializations": metadata.get(
-                    "tags", template_data.get("tags", [])
-                ),  # Use tags as specializations, fallback to root-level tags
+                "version": frontmatter.get("version", "1.0.0"),
+                "tools": frontmatter.get("tools", []),
+                "specializations": frontmatter.get("tags", []),
                 "file": template_file.name,
                 "path": str(template_file),
                 "file_path": str(template_file),  # Keep for backward compatibility
                 "size": template_file.stat().st_size,
-                "model": model_value,
-                "author": metadata.get("author", "unknown"),
+                "model": frontmatter.get("model", "sonnet"),
+                "author": frontmatter.get("author", "unknown"),
             }
 
             # Validate required fields
@@ -261,8 +247,8 @@ class AgentDiscoveryService:
 
             return agent_info
 
-        except json.JSONDecodeError as e:
-            self.logger.error(f"Invalid JSON in template {template_file.name}: {e}")
+        except yaml.YAMLError as e:
+            self.logger.warning(f"Invalid YAML frontmatter in {template_file.name}: {e}")
             return None
         except Exception as e:
             self.logger.error(
@@ -270,37 +256,73 @@ class AgentDiscoveryService:
             )
             return None
 
+    def _extract_yaml_frontmatter(self, content: str) -> Optional[Dict[str, Any]]:
+        """
+        Extract and parse YAML frontmatter from markdown.
+
+        Frontmatter must be at the start of the file, delimited by '---'.
+        Example:
+            ---
+            name: agent_name
+            description: Agent description
+            version: 1.0.0
+            ---
+            # Agent content...
+
+        Args:
+            content: File content to parse
+
+        Returns:
+            Parsed YAML frontmatter as dict, or None if not found/invalid
+        """
+        if not content.strip().startswith("---"):
+            return None
+
+        # Split on --- delimiters
+        parts = content.split("---", 2)
+        if len(parts) < 3:
+            return None
+
+        try:
+            return yaml.safe_load(parts[1])
+        except yaml.YAMLError as e:
+            self.logger.warning(f"Failed to parse YAML frontmatter: {e}")
+            return None
+
     def _is_mpm_agent(self, template_file: Path) -> bool:
         """Check if agent is authored by Claude MPM team.
 
         MPM agents must have:
         - An author field containing 'claude mpm', 'claude-mpm', or 'anthropic'
-        - A valid agent_version field
+        - A valid version field
 
         Args:
-            template_file: Path to the agent template JSON file
+            template_file: Path to the agent template markdown file
 
         Returns:
             True if this is an MPM agent, False otherwise
         """
         try:
-            template_data = json.loads(template_file.read_text())
-            metadata = template_data.get("metadata", {})
+            # Extract YAML frontmatter
+            content = template_file.read_text()
+            frontmatter = self._extract_yaml_frontmatter(content)
+            if not frontmatter:
+                return False
 
             # Check for author field
-            author = metadata.get("author", "").lower()
+            author = frontmatter.get("author", "").lower()
             has_valid_author = any(
                 pattern in author
                 for pattern in ["claude mpm", "claude-mpm", "anthropic"]
             )
 
             # Check for version field
-            has_version = bool(template_data.get("agent_version"))
+            has_version = bool(frontmatter.get("version"))
 
             if not has_valid_author or not has_version:
                 self.logger.debug(
                     f"Filtered non-MPM agent {template_file.name}: "
-                    f"author='{metadata.get('author', 'missing')}', "
+                    f"author='{frontmatter.get('author', 'missing')}', "
                     f"version={'present' if has_version else 'missing'}"
                 )
 
@@ -366,10 +388,10 @@ class AgentDiscoveryService:
 
     def _validate_template_file(self, template_file: Path) -> bool:
         """
-        Validate that a template file is properly formatted.
+        Validate that a template file is properly formatted with YAML frontmatter.
 
         Args:
-            template_file: Path to template file to validate
+            template_file: Path to markdown template file to validate
 
         Returns:
             True if template is valid, False otherwise
@@ -379,23 +401,27 @@ class AgentDiscoveryService:
             if not template_file.exists():
                 return False
 
-            # Parse JSON content
+            # Read and parse YAML frontmatter
             content = template_file.read_text()
-            template_data = json.loads(content)
+            frontmatter = self._extract_yaml_frontmatter(content)
+            if not frontmatter:
+                self.logger.warning(
+                    f"Template {template_file.name} has no valid YAML frontmatter"
+                )
+                return False
 
-            # Check required fields in metadata section (per agent schema)
-            metadata = template_data.get("metadata", {})
+            # Check required fields (flat structure in markdown templates)
             required_fields = ["name", "description"]
             for field in required_fields:
-                if field not in metadata:
+                if field not in frontmatter:
                     self.logger.warning(
-                        f"Template {template_file.name} missing required field in metadata: {field}"
+                        f"Template {template_file.name} missing required field: {field}"
                     )
                     return False
 
             # Validate agent ID format (Claude Code requirements)
             # Use agent_id for validation, not the display name
-            agent_id = template_data.get("agent_id", "")
+            agent_id = frontmatter.get("agent_id", "")
             if not self._is_valid_agent_name(agent_id):
                 self.logger.warning(
                     f"Invalid agent ID format in {template_file.name}: {agent_id}"
@@ -404,8 +430,10 @@ class AgentDiscoveryService:
 
             return True
 
-        except json.JSONDecodeError:
-            self.logger.error(f"Invalid JSON in template: {template_file.name}")
+        except yaml.YAMLError:
+            self.logger.warning(
+                f"Invalid YAML frontmatter in template: {template_file.name}"
+            )
             return False
         except Exception as e:
             self.logger.error(
@@ -431,25 +459,28 @@ class AgentDiscoveryService:
 
         return bool(re.match(pattern, agent_name))
 
-    def _is_mpm_agent(
+    def _is_mpm_agent_with_config(
         self, template_file: Path, config: Optional[Config] = None
     ) -> bool:
-        """Check if agent is authored by Claude MPM team.
+        """Check if agent is authored by Claude MPM team with configurable patterns.
 
         MPM agents must have:
         - An author field containing configurable MPM patterns (default: 'claude mpm', 'claude-mpm', 'anthropic')
-        - A valid agent_version field
+        - A valid version field
 
         Args:
-            template_file: Path to the agent template JSON file
+            template_file: Path to the agent template markdown file
             config: Configuration object for MPM patterns
 
         Returns:
             True if this is an MPM agent, False otherwise
         """
         try:
-            template_data = json.loads(template_file.read_text())
-            metadata = template_data.get("metadata", {})
+            # Extract YAML frontmatter
+            content = template_file.read_text()
+            frontmatter = self._extract_yaml_frontmatter(content)
+            if not frontmatter:
+                return False
 
             # Get MPM author patterns from config
             if config:
@@ -461,13 +492,13 @@ class AgentDiscoveryService:
                 mpm_patterns = ["claude mpm", "claude-mpm", "anthropic"]
 
             # Check for author field
-            author = metadata.get("author", "").lower()
+            author = frontmatter.get("author", "").lower()
             has_valid_author = any(
                 pattern.lower() in author for pattern in mpm_patterns
             )
 
             # Check for version field
-            has_version = bool(template_data.get("agent_version"))
+            has_version = bool(frontmatter.get("version"))
 
             return has_valid_author and has_version
 
@@ -494,8 +525,8 @@ class AgentDiscoveryService:
         if not self.templates_dir.exists():
             return stats
 
-        # Count template files
-        template_files = list(self.templates_dir.glob("*.json"))
+        # Count markdown template files
+        template_files = list(self.templates_dir.glob("*.md"))
         stats["total_templates"] = len(template_files)
 
         # Validate each template

claude_mpm/services/agents/deployment/agent_format_converter.py

@@ -166,7 +166,7 @@ class AgentFormatConverter:
 
 <example>
 Context: When you need specialized assistance from the {name} agent.
-user: "I need help with {agent_name.replace('_', ' ').replace('-', ' ')} tasks"
+user: "I need help with {agent_name.replace("_", " ").replace("-", " ")} tasks"
 assistant: "I'll use the {name} agent to provide specialized assistance."
 </example>"""
 

claude_mpm/services/agents/deployment/agent_lifecycle_manager.py

@@ -63,11 +63,7 @@ from .agent_operation_service import (
     LifecycleOperationResult,
 )
 from .agent_record_service import AgentRecordService
-from .agent_state_service import (
-    AgentLifecycleRecord,
-    AgentStateService,
-    LifecycleState,
-)
+from .agent_state_service import AgentLifecycleRecord, AgentStateService, LifecycleState
 
 # Re-export for backward compatibility
 __all__ = [

claude_mpm/services/agents/deployment/agent_metrics_collector.py

@@ -176,9 +176,9 @@ class AgentMetricsCollector:
             agent_name: Name of the agent being validated
             validation_time_ms: Validation time in milliseconds
         """
-        self._deployment_metrics["template_validation_times"][
-            agent_name
-        ] = validation_time_ms
+        self._deployment_metrics["template_validation_times"][agent_name] = (
+            validation_time_ms
+        )
 
     def get_performance_summary(self) -> Dict[str, Any]:
         """

claude_mpm/services/agents/deployment/agent_restore_handler.py

@@ -10,10 +10,7 @@ from typing import TYPE_CHECKING, Optional
 from claude_mpm.core.path_ops import path_ops
 
 if TYPE_CHECKING:
-    from .agent_lifecycle_manager import (
-        AgentLifecycleManager,
-        LifecycleOperationResult,
-    )
+    from .agent_lifecycle_manager import AgentLifecycleManager, LifecycleOperationResult
 
 
 class AgentRestoreHandler: