claude-mpm 4.25.10__py3-none-any.whl → 5.1.8__py3-none-any.whl

claude_mpm/services/agents/deployment/agent_template_builder.py

@@ -12,6 +12,8 @@ import re
 from pathlib import Path
 from typing import Any, Dict, List
 
+import yaml
+
 from claude_mpm.core.logging_config import get_logger
 
 
@@ -76,9 +78,190 @@ class AgentTemplateBuilder:
 
         return tool_list
 
+    def _discover_base_agent_templates(self, agent_file: Path) -> List[Path]:
+        """Discover BASE-AGENT.md files in hierarchy from agent file to repository root.
+
+        This method implements hierarchical BASE template discovery by walking up the
+        directory tree from the agent file location and collecting all BASE-AGENT.md
+        files found along the way.
+
+        Composition Order (closest to farthest):
+        1. Local BASE-AGENT.md (same directory as agent)
+        2. Parent BASE-AGENT.md (parent directory)
+        3. Grandparent BASE-AGENT.md (grandparent directory)
+        ... continuing to repository root
+
+        Args:
+            agent_file: Path to the agent template file
+
+        Returns:
+            List of BASE-AGENT.md paths ordered from closest to farthest
+            (same directory to root)
+
+        Example:
+            Given structure:
+            repo/
+              BASE-AGENT.md           # Root (index 2)
+              engineering/
+                BASE-AGENT.md         # Parent (index 1)
+                python/
+                  BASE-AGENT.md       # Local (index 0)
+                  fastapi-engineer.md # Agent file
+
+            Returns: [
+                repo/engineering/python/BASE-AGENT.md,
+                repo/engineering/BASE-AGENT.md,
+                repo/BASE-AGENT.md
+            ]
+        """
+        base_templates = []
+        current_dir = agent_file.parent
+
+        # Walk up directory tree until we reach root or a reasonable limit
+        # Stop at repository root or after 10 levels (safety limit)
+        max_depth = 10
+        depth = 0
+
+        while current_dir and depth < max_depth:
+            # Check for BASE-AGENT.md in current directory
+            base_agent_file = current_dir / "BASE-AGENT.md"
+            if base_agent_file.exists() and base_agent_file.is_file():
+                base_templates.append(base_agent_file)
+                self.logger.debug(f"Found BASE-AGENT.md at: {base_agent_file}")
+
+            # Stop at git repository root if detected
+            if (current_dir / ".git").exists():
+                self.logger.debug(f"Reached git repository root at: {current_dir}")
+                break
+
+            # Stop at common repository root indicators
+            if current_dir.name in [".claude-mpm", "remote-agents", "cache"]:
+                self.logger.debug(
+                    f"Reached repository root indicator at: {current_dir}"
+                )
+                break
+
+            # Move to parent directory
+            parent = current_dir.parent
+            if parent == current_dir:  # Reached filesystem root
+                break
+
+            current_dir = parent
+            depth += 1
+
+        if base_templates:
+            self.logger.info(
+                f"Discovered {len(base_templates)} BASE-AGENT.md file(s) for {agent_file.name}"
+            )
+
+        return base_templates
+
+    def _parse_markdown_template(self, template_path: Path) -> dict:
+        """Parse Markdown template with YAML frontmatter.
+
+        Extracts metadata from YAML frontmatter and content from Markdown body.
+        Supports the new agent template format with YAML frontmatter between --- delimiters.
+
+        Args:
+            template_path: Path to the Markdown template file
+
+        Returns:
+            Dictionary containing metadata and instructions
+
+        Raises:
+            ValueError: If frontmatter is missing or malformed
+            yaml.YAMLError: If YAML parsing fails
+        """
+        content = template_path.read_text(encoding="utf-8")
+
+        # Split frontmatter and body
+        # Format: ---\n<yaml>\n---\n<markdown>
+        if not content.startswith("---"):
+            raise ValueError(
+                f"Markdown template missing YAML frontmatter: {template_path}"
+            )
+
+        # Split by --- delimiters
+        parts = content.split("---", 2)
+        if len(parts) < 3:
+            raise ValueError(f"Malformed YAML frontmatter in template: {template_path}")
+
+        # parts[0] is empty (before first ---)
+        # parts[1] is YAML frontmatter
+        # parts[2] is Markdown content
+        yaml_content = parts[1].strip()
+        markdown_content = parts[2].strip()
+
+        # Parse YAML frontmatter
+        try:
+            metadata = yaml.safe_load(yaml_content)
+            if not isinstance(metadata, dict):
+                raise ValueError(
+                    f"YAML frontmatter must be a dictionary: {template_path}"
+                )
+        except yaml.YAMLError as e:
+            self.logger.error(
+                f"Failed to parse YAML frontmatter in {template_path}: {e}"
+            )
+            raise
+
+        # Validate required fields
+        required_fields = ["name", "description", "version"]
+        missing_fields = [field for field in required_fields if field not in metadata]
+        if missing_fields:
+            raise ValueError(
+                f"Missing required fields in template {template_path}: {missing_fields}"
+            )
+
+        # Add the markdown content as instructions field
+        metadata["instructions"] = markdown_content
+
+        # Normalize metadata structure to match JSON template format
+        # JSON templates have these fields at top level, Markdown may have them nested
+        self._normalize_metadata_structure(metadata)
+
+        return metadata
+
+    def _normalize_metadata_structure(self, metadata: dict) -> None:
+        """Normalize metadata structure to match expected JSON template format.
+
+        This ensures both Markdown and JSON templates produce the same metadata structure
+        for downstream processing.
+
+        Args:
+            metadata: Metadata dictionary to normalize (modified in-place)
+        """
+        # Map Markdown frontmatter fields to JSON template structure
+        # Handle tags: YAML list vs JSON comma-separated string
+        if "tags" in metadata and isinstance(metadata["tags"], list):
+            # Keep as list for now, normalize_tools_input will handle both formats
+            pass
+
+        # Map agent_id to name if name is missing
+        if "agent_id" in metadata and "name" not in metadata:
+            metadata["name"] = metadata["agent_id"]
+
+        # Ensure capabilities dict exists
+        if "capabilities" not in metadata:
+            metadata["capabilities"] = {}
+
+        # Merge top-level capability fields into capabilities dict
+        capability_fields = ["memory_limit", "cpu_limit", "network_access"]
+        for field in capability_fields:
+            if field in metadata:
+                metadata["capabilities"][field] = metadata.pop(field)
+
+        # Add model to capabilities if present at top level
+        if "model" in metadata and "model" not in metadata["capabilities"]:
+            metadata["capabilities"]["model"] = metadata["model"]
+
     def _load_base_agent_instructions(self, agent_type: str) -> str:
         """Load BASE instructions for a specific agent type.
 
+        DEPRECATED: This method loads BASE_{TYPE}.md files (old pattern).
+        New pattern uses hierarchical BASE-AGENT.md files discovered via
+        _discover_base_agent_templates() and composed in build_agent_markdown().
+
         Args:
             agent_type: The type of agent (engineer, qa, ops, research, documentation)
 
@@ -156,7 +339,7 @@ class AgentTemplateBuilder:
 
         Args:
             agent_name: Name of the agent
-            template_path: Path to the agent template JSON file
+            template_path: Path to the agent template (JSON or Markdown file)
             base_agent_data: Base agent configuration data
             source_info: Source of the agent (system/project/user)
 
@@ -166,16 +349,34 @@ class AgentTemplateBuilder:
         Raises:
             FileNotFoundError: If template file doesn't exist
             json.JSONDecodeError: If template JSON is invalid
+            yaml.YAMLError: If template YAML is invalid
+            ValueError: If template format is unsupported or malformed
         """
         if not template_path.exists():
             raise FileNotFoundError(f"Template file not found: {template_path}")
 
+        # Format detection: route to appropriate parser
         try:
-            template_content = template_path.read_text()
-            template_data = json.loads(template_content)
+            if template_path.suffix == ".md":
+                # Parse Markdown template with YAML frontmatter
+                self.logger.debug(f"Parsing Markdown template: {template_path}")
+                template_data = self._parse_markdown_template(template_path)
+            elif template_path.suffix == ".json":
+                # Parse JSON template (legacy format)
+                self.logger.debug(f"Parsing JSON template: {template_path}")
+                template_content = template_path.read_text()
+                template_data = json.loads(template_content)
+            else:
+                raise ValueError(
+                    f"Unsupported template format: {template_path.suffix}. "
+                    f"Expected .md or .json"
+                )
         except json.JSONDecodeError as e:
             self.logger.error(f"Invalid JSON in template {template_path}: {e}")
             raise
+        except (yaml.YAMLError, ValueError) as e:
+            self.logger.error(f"Invalid template {template_path}: {e}")
+            raise
 
         # Extract tools from template with fallback
         # Handle both dict and list formats for capabilities (backward compatibility)
@@ -392,9 +593,6 @@ class AgentTemplateBuilder:
 
         frontmatter = "\n".join(frontmatter_lines)
 
-        # Load BASE instructions for this agent type
-        base_instructions = self._load_base_agent_instructions(agent_type)
-
         # Get agent instructions from template data (primary) or base agent data (fallback)
         raw_instructions = template_data.get("instructions")
 
@@ -411,15 +609,38 @@ class AgentTemplateBuilder:
                 or "# Agent Instructions\n\nThis agent provides specialized assistance."
             )
 
-        # Combine BASE instructions with agent-specific instructions
-        if base_instructions:
-            # Create a combined instruction set
-            content = f"{base_instructions}\n\n---\n\n{agent_specific_instructions}"
-            self.logger.debug(
-                f"Combined BASE instructions with agent-specific instructions for {agent_type}"
-            )
-        else:
-            content = agent_specific_instructions
+        # Compose hierarchical BASE-AGENT.md templates
+        # Order: agent-specific + local BASE + parent BASE + ... + root BASE
+        content_parts = [agent_specific_instructions]
+
+        # Discover BASE-AGENT.md files in directory hierarchy
+        base_templates = self._discover_base_agent_templates(template_path)
+
+        # Append each BASE template (order: closest to farthest)
+        for base_template_path in base_templates:
+            try:
+                base_content = base_template_path.read_text(encoding="utf-8")
+                if base_content.strip():
+                    content_parts.append(base_content)
+                    self.logger.debug(
+                        f"Composed BASE template: {base_template_path.relative_to(template_path.parent.parent) if template_path.parent.parent in base_template_path.parents else base_template_path.name}"
+                    )
+            except Exception as e:
+                self.logger.warning(
+                    f"Failed to read BASE template {base_template_path}: {e}"
+                )
+
+        # Fallback: Load legacy BASE_{TYPE}.md if no hierarchical templates found
+        if len(content_parts) == 1:  # Only agent-specific instructions
+            legacy_base_instructions = self._load_base_agent_instructions(agent_type)
+            if legacy_base_instructions:
+                content_parts.append(legacy_base_instructions)
+                self.logger.debug(
+                    f"Using legacy BASE_{agent_type.upper()}.md (no hierarchical BASE-AGENT.md found)"
+                )
+
+        # Join all parts with separator
+        content = "\n\n---\n\n".join(content_parts)
 
         # Add memory update instructions if not already present
         if "memory-update" not in content and "Remember" not in content:

claude_mpm/services/agents/deployment/agents_directory_resolver.py

@@ -2,6 +2,47 @@
 
 This module handles determining the correct agents directory for deployment
 based on different deployment scenarios and target directories.
+
+DEPLOYMENT ARCHITECTURE:
+=======================
+
+Agent Source Locations (Discovery):
+-----------------------------------
+1. System Agents:  ~/.claude-mpm/cache/remote-agents/bobmatnyc/claude-mpm-agents/
+   - Synced from GitHub repository
+   - Read-only (managed by git pull)
+   - 44+ agents organized by category
+
+2. User Agents:    ~/.claude-mpm/agents/
+   - User-created custom agents
+   - Personal agent definitions
+
+3. Project Agents: .claude-mpm/agents/
+   - Project-specific agents
+   - Overrides for system/user agents
+
+Agent Deployment Target (Runtime):
+----------------------------------
+ALL AGENTS DEPLOY TO: <project_root>/.claude/agents/
+   - Single deployment location for ALL agents (system, user, project)
+   - Claude Code discovers agents from this directory
+   - Ensures consistent agent availability across sessions
+   - Simplifies agent management and version control
+
+Why Project-Level Deployment?
+------------------------------
+1. **Consistency**: All team members work with same agents
+2. **Versioning**: Agent deployments tracked in git
+3. **Isolation**: Different projects can have different agent versions
+4. **Performance**: No global agent conflicts or version mismatches
+
+Example Flow:
+-------------
+1. User runs: claude-mpm agents deploy
+2. Agents synced from GitHub → ~/.claude-mpm/cache/remote-agents/
+3. Agents deployed FROM cache → .claude/agents/
+4. Claude Code discovers agents FROM .claude/agents/
+
 Extracted from AgentDeploymentService to reduce complexity.
 """
 
@@ -28,36 +69,81 @@ class AgentsDirectoryResolver:
         """
         Determine the correct agents directory based on input.
 
-        MODIFIED: Always deploy to project .claude/agents directory
-        regardless of agent source (system, user, or project).
+        DEPLOYMENT STRATEGY:
+        ====================
+        Always deploy to project .claude/agents directory regardless of agent
+        source (system, user, or project). This ensures:
+        - All agents available in single location
+        - Claude Code agent discovery works consistently
+        - Version control tracks deployed agents
+        - No global agent conflicts between projects
 
-        This ensures all agents are deployed at the project level while
-        maintaining discovery from both user (~/.claude-mpm) and project
-        (.claude-mpm) directories.
+        LOGIC FLOW:
+        -----------
+        1. If target_dir is None (default case):
+           → Deploy to: <working_directory>/.claude/agents
+           → This is 99% of use cases (standard deployment)
+
+        2. If target_dir is provided explicitly:
+           → Normalize the path based on directory name:
+             - Already "agents" directory? → Use as-is
+             - Is ".claude-mpm" directory? → Add /agents
+             - Is ".claude" directory? → Add /agents
+             - Otherwise? → Treat as project root, add /.claude/agents
+
+        EXAMPLES:
+        ---------
+        Standard deployment (target_dir=None):
+          working_directory = /Users/masa/Projects/my-app
+          → Returns: /Users/masa/Projects/my-app/.claude/agents
+
+        Explicit target (target_dir="/some/path/.claude"):
+          → Returns: /some/path/.claude/agents
+
+        Explicit target (target_dir="/some/path/.claude/agents"):
+          → Returns: /some/path/.claude/agents (no modification)
 
         Args:
-            target_dir: Optional target directory
+            target_dir: Optional target directory override
+                       None = use default project .claude/agents (recommended)
+                       Path = explicitly specify deployment location (advanced)
 
         Returns:
-            Path to agents directory
+            Path: Resolved agents directory path
+
+        Note:
+            The default behavior (target_dir=None) is the recommended approach.
+            Only provide target_dir for advanced use cases or testing.
         """
         if not target_dir:
-            # Always deploy to project directory
-            # This is the key change - all agents go to project .claude/agents
+            # DEFAULT CASE: Deploy to project .claude/agents directory
+            # This is the standard behavior for all normal deployments
+            # Example: /Users/masa/Projects/my-app → /Users/masa/Projects/my-app/.claude/agents
             return self.working_directory / ".claude" / "agents"
 
-        # If target_dir provided, use it directly (caller decides structure)
+        # EXPLICIT TARGET CASE: User/caller provided specific target directory
+        # Normalize target_dir to proper agents directory based on structure
         target_dir = Path(target_dir)
 
-        # Check if this is already an agents directory
+        # Pattern 1: Already an agents directory
         if target_dir.name == "agents":
-            # Already an agents directory, use as-is
+            # Already targeting agents dir explicitly, use as-is
+            # Example: /some/path/.claude/agents → /some/path/.claude/agents
             return target_dir
+
+        # Pattern 2: Targeting .claude-mpm directory
         if target_dir.name == ".claude-mpm":
-            # .claude-mpm directory, add agents subdirectory
+            # Add agents subdirectory to .claude-mpm
+            # Example: /some/path/.claude-mpm → /some/path/.claude-mpm/agents
             return target_dir / "agents"
+
+        # Pattern 3: Targeting .claude directory
         if target_dir.name == ".claude":
-            # .claude directory, add agents subdirectory
+            # Add agents subdirectory to .claude
+            # Example: /some/path/.claude → /some/path/.claude/agents
             return target_dir / "agents"
-        # Assume it's a project directory, add .claude/agents
+
+        # Pattern 4: Targeting project root or other directory
+        # Assume it's a project directory, add full .claude/agents path
+        # Example: /some/project → /some/project/.claude/agents
         return target_dir / ".claude" / "agents"

claude_mpm/services/agents/deployment/async_agent_deployment.py

@@ -200,9 +200,10 @@ class AsyncAgentDeploymentService:
                 return str(directory), []
 
             # Use asyncio to run glob in executor (since Path.glob is blocking)
+            # Agent templates migrated to Markdown with YAML frontmatter (v4.26.0+)
             loop = asyncio.get_event_loop()
             files = await loop.run_in_executor(
-                self.executor, lambda: list(directory.glob("*.json"))
+                self.executor, lambda: list(directory.glob("*.md"))
             )
 
             self.logger.debug(f"Found {len(files)} agents in {directory}")