claude-mpm 5.1.9__py3-none-any.whl → 5.4.3__py3-none-any.whl

claude_mpm/services/agents/deployment/remote_agent_discovery_service.py

@@ -15,6 +15,8 @@ from dataclasses import dataclass
 from pathlib import Path
 from typing import Any, Dict, List, Optional
 
+import yaml
+
 from claude_mpm.core.logging_config import get_logger
 
 logger = get_logger(__name__)
@@ -153,6 +155,110 @@ class RemoteAgentDiscoveryService:
             self.logger.warning(f"Failed to extract source_path from {file_path}: {e}")
             return None
 
+    def _parse_yaml_frontmatter(self, content: str) -> Optional[Dict[str, Any]]:
+        """Parse YAML frontmatter from Markdown content.
+
+        Extracts YAML frontmatter delimited by --- markers at the start of the file.
+        Uses a tolerant approach: attempts full YAML parsing first, falls back to
+        simple key-value extraction for malformed YAML.
+
+        Design Decision: Tolerant YAML Parsing
+
+        Rationale: Some agent markdown files have malformed YAML (incorrect indentation
+        in nested structures). Rather than failing completely, we:
+        1. Try full YAML parsing first (handles well-formed YAML)
+        2. Fall back to regex extraction for critical fields (agent_id, name, etc.)
+        3. Log warnings but continue processing
+
+        This ensures we can still extract agent_id even if complex nested structures
+        (like template_changelog) have indentation issues.
+
+        Args:
+            content: Full Markdown file content
+
+        Returns:
+            Dictionary of parsed YAML frontmatter, or None if not found
+
+        Example:
+            Input:
+                ---
+                agent_id: python-engineer
+                name: Python Engineer
+                version: 2.3.0
+                ---
+                # Agent content...
+
+            Output:
+                {"agent_id": "python-engineer", "name": "Python Engineer", "version": "2.3.0"}
+        """
+        try:
+            # Check if content starts with YAML frontmatter
+            if not content.startswith("---"):
+                self.logger.debug("No YAML frontmatter found (doesn't start with ---)")
+                return None
+
+            # Extract frontmatter content between --- markers
+            frontmatter_match = re.match(r"^---\n(.*?)\n---\s*\n", content, re.DOTALL)
+            if not frontmatter_match:
+                self.logger.debug("No closing --- marker found for YAML frontmatter")
+                return None
+
+            yaml_content = frontmatter_match.group(1)
+
+            # Try full YAML parsing first
+            try:
+                parsed = yaml.safe_load(yaml_content)
+                if isinstance(parsed, dict):
+                    return parsed
+                self.logger.warning(
+                    f"YAML frontmatter is not a dictionary: {type(parsed)}"
+                )
+            except yaml.YAMLError as e:
+                # Malformed YAML (e.g., indentation errors) - fall back to regex extraction
+                self.logger.debug(
+                    f"Full YAML parse failed, using fallback extraction: {e}"
+                )
+
+                # Extract key fields using regex (tolerant of malformed nested structures)
+                result = {}
+
+                # Extract simple key-value pairs (no nested structures)
+                simple_keys = [
+                    "agent_id",
+                    "name",
+                    "description",
+                    "version",
+                    "model",
+                    "agent_type",
+                    "category",
+                    "author",
+                    "schema_version",
+                ]
+
+                for key in simple_keys:
+                    # Match key: value on a line (not indented, so it's top-level)
+                    pattern = rf"^{key}:\s*(.+?)$"
+                    match = re.search(pattern, yaml_content, re.MULTILINE)
+                    if match:
+                        value = match.group(1).strip()
+                        # Remove quotes if present
+                        if value.startswith(("'", '"')) and value.endswith(("'", '"')):
+                            value = value[1:-1]
+                        result[key] = value
+
+                if result:
+                    self.logger.debug(
+                        f"Extracted {len(result)} fields using fallback method"
+                    )
+                    return result
+                return None
+
+        except Exception as e:
+            self.logger.warning(f"Unexpected error parsing frontmatter: {e}")
+            return None
+
+        return None
+
     def _generate_hierarchical_id(self, file_path: Path) -> str:
         """Generate hierarchical agent ID from file path.
 
@@ -165,14 +271,20 @@ class RemoteAgentDiscoveryService:
         - Preset matching against AUTO-DEPLOY-INDEX.md
         - Multi-level organization without name collisions
 
-        Bug #4 Fix: Calculate relative to /agents/ subdirectory, not repository root
-        This ensures agent IDs are based on their position within the agents directory.
+        Supports both cache structures:
+        1. Git repo: Calculate relative to /agents/ subdirectory
+        2. Flattened cache: Calculate relative to remote_agents_dir directly
 
-        Example:
+        Example (Git repo):
             Input:  /cache/bobmatnyc/claude-mpm-agents/agents/engineer/backend/python-engineer.md
             Root:   /cache/bobmatnyc/claude-mpm-agents/agents
             Output: engineer/backend/python-engineer
 
+        Example (Flattened cache):
+            Input:  /cache/remote-agents/engineer/python-engineer.md
+            Root:   /cache/remote-agents
+            Output: engineer/python-engineer
+
         Args:
             file_path: Absolute path to agent Markdown file
 
@@ -180,16 +292,30 @@ class RemoteAgentDiscoveryService:
             Hierarchical agent ID with forward slashes
         """
         try:
-            # Calculate relative to /agents/ subdirectory (Bug #4 fix)
+            # Try git repo structure first: /agents/ subdirectory
             agents_dir = self.remote_agents_dir / "agents"
-            relative_path = file_path.relative_to(agents_dir)
+            if agents_dir.exists():
+                try:
+                    relative_path = file_path.relative_to(agents_dir)
+                    return str(relative_path.with_suffix("")).replace("\\", "/")
+                except ValueError:
+                    pass  # Not under agents_dir, try flattened structure
+
+            # Try flattened cache structure: calculate relative to remote_agents_dir
+            try:
+                relative_path = file_path.relative_to(self.remote_agents_dir)
+                return str(relative_path.with_suffix("")).replace("\\", "/")
+            except ValueError:
+                pass  # Not under remote_agents_dir either
 
-            # Remove .md extension and convert to forward slashes
-            return str(relative_path.with_suffix("")).replace("\\", "/")
-        except ValueError:
-            # File is not under agents subdirectory, fall back to filename
+            # Fall back to filename
+            self.logger.warning(
+                f"File {file_path} not under expected directories, using filename"
+            )
+            return file_path.stem
+        except Exception as e:
             self.logger.warning(
-                f"File {file_path} not under agents directory, using filename"
+                f"Error generating hierarchical ID for {file_path}: {e}"
             )
             return file_path.stem
 
@@ -199,14 +325,20 @@ class RemoteAgentDiscoveryService:
         Extracts category from directory structure. Category is the path
         from agents subdirectory to the file, excluding the filename.
 
-        Bug #4 Fix: Calculate relative to /agents/ subdirectory, not repository root
-        This ensures categories are based on agent organization within /agents/.
+        Supports both cache structures:
+        1. Git repo: Calculate relative to /agents/ subdirectory
+        2. Flattened cache: Calculate relative to remote_agents_dir directly
 
-        Example:
+        Example (Git repo):
             Input:  /cache/bobmatnyc/claude-mpm-agents/agents/engineer/backend/python-engineer.md
             Root:   /cache/bobmatnyc/claude-mpm-agents/agents
             Output: engineer/backend
 
+        Example (Flattened cache):
+            Input:  /cache/remote-agents/engineer/python-engineer.md
+            Root:   /cache/remote-agents
+            Output: engineer
+
         Args:
             file_path: Absolute path to agent Markdown file
 
@@ -214,12 +346,26 @@ class RemoteAgentDiscoveryService:
             Category path with forward slashes, or "universal" if in root
         """
         try:
-            # Calculate relative to /agents/ subdirectory (Bug #4 fix)
+            # Try git repo structure first: /agents/ subdirectory
             agents_dir = self.remote_agents_dir / "agents"
-            relative_path = file_path.relative_to(agents_dir)
-            parts = relative_path.parts[:-1]  # Exclude filename
-            return "/".join(parts) if parts else "universal"
-        except ValueError:
+            if agents_dir.exists():
+                try:
+                    relative_path = file_path.relative_to(agents_dir)
+                    parts = relative_path.parts[:-1]  # Exclude filename
+                    return "/".join(parts) if parts else "universal"
+                except ValueError:
+                    pass  # Not under agents_dir, try flattened structure
+
+            # Try flattened cache structure: calculate relative to remote_agents_dir
+            try:
+                relative_path = file_path.relative_to(self.remote_agents_dir)
+                parts = relative_path.parts[:-1]  # Exclude filename
+                return "/".join(parts) if parts else "universal"
+            except ValueError:
+                pass  # Not under remote_agents_dir either
+
+            return "universal"
+        except Exception:
             return "universal"
 
     def discover_remote_agents(self) -> List[Dict[str, Any]]:
@@ -228,8 +374,12 @@ class RemoteAgentDiscoveryService:
         Scans the remote agents directory for *.md files recursively and converts each
         to JSON template format. Skips files that can't be parsed.
 
-        Bug #4 Fix: Only scan /agents/ subdirectory, not entire repository
-        This prevents README.md, CHANGELOG.md, etc. from being treated as agents.
+        Supports two cache structures:
+        1. Git repo path: {path}/agents/ - has /agents/ subdirectory
+        2. Flattened cache: {path}/ - directly contains category directories
+
+        Bug #4 Fix: Only scan /agents/ subdirectory when it exists to prevent
+        README.md, CHANGELOG.md, etc. from being treated as agents.
 
         Returns:
             List of agent dictionaries in JSON template format
@@ -250,20 +400,94 @@ class RemoteAgentDiscoveryService:
             )
             return agents
 
-        # Bug #4 Fix: Only scan /agents/ subdirectory, not entire repository
-        # This prevents non-agent files (README.md, CHANGELOG.md, etc.) from polluting results
+        # Support three cache structures (PRIORITY ORDER):
+        # 1. Built output: {path}/dist/agents/ - PREFERRED (built with BASE-AGENT composition)
+        # 2. Git repo path: {path}/agents/ - source files (fallback)
+        # 3. Flattened cache: {path}/ - directly contains category directories (legacy)
+
+        # Priority 1: Check for dist/agents/ (built output with BASE-AGENT composition)
+        dist_agents_dir = self.remote_agents_dir / "dist" / "agents"
         agents_dir = self.remote_agents_dir / "agents"
 
-        if not agents_dir.exists():
-            self.logger.warning(
-                f"Agents subdirectory not found: {agents_dir}. "
-                f"Expected agents to be in /agents/ subdirectory."
+        if dist_agents_dir.exists():
+            # PREFERRED: Use built agents from dist/agents/
+            # These have BASE-AGENT.md files properly composed by build-agent.py
+            self.logger.debug(f"Using built agents from dist: {dist_agents_dir}")
+            scan_dir = dist_agents_dir
+        elif agents_dir.exists():
+            # FALLBACK: Git repo structure - scan /agents/ subdirectory (source files)
+            # This path is used when dist/agents/ hasn't been built yet
+            self.logger.debug(f"Using source agents (no dist/ found): {agents_dir}")
+            scan_dir = agents_dir
+        else:
+            # LEGACY: Flattened cache structure - scan root directly
+            # Check if this looks like the flattened cache (has category subdirectories)
+            category_dirs = [
+                "universal",
+                "engineer",
+                "ops",
+                "qa",
+                "security",
+                "documentation",
+            ]
+            has_categories = any(
+                (self.remote_agents_dir / cat).exists() for cat in category_dirs
             )
-            return agents
 
-        # Find all Markdown files recursively in /agents/ subdirectory only
-        md_files = list(agents_dir.rglob("*.md"))
-        self.logger.debug(f"Found {len(md_files)} Markdown files in {agents_dir}")
+            if has_categories:
+                self.logger.debug(
+                    f"Using flattened cache structure: {self.remote_agents_dir}"
+                )
+                scan_dir = self.remote_agents_dir
+            else:
+                self.logger.warning(
+                    f"No agent directories found. Checked: {dist_agents_dir}, {agents_dir}, "
+                    f"and category directories in {self.remote_agents_dir}. "
+                    f"Expected agents in /dist/agents/, /agents/, or category directories."
+                )
+                return agents
+
+        # Find all Markdown files recursively
+        md_files = list(scan_dir.rglob("*.md"))
+
+        # Filter out non-agent files and git repository files
+        excluded_files = {
+            "README.md",
+            "CHANGELOG.md",
+            "CONTRIBUTING.md",
+            "LICENSE.md",
+            "BASE-AGENT.md",
+            "SUMMARY.md",
+            "IMPLEMENTATION-SUMMARY.md",
+            "REFACTORING_REPORT.md",
+            "REORGANIZATION-PLAN.md",
+            "AUTO-DEPLOY-INDEX.md",
+            "PHASE1_COMPLETE.md",
+            "AGENTS.md",
+        }
+        md_files = [f for f in md_files if f.name not in excluded_files]
+
+        # In flattened cache mode, also exclude files from git repository subdirectories
+        # (files under directories that contain .git folder)
+        if scan_dir == self.remote_agents_dir:
+            filtered_files = []
+            for f in md_files:
+                # Check if this file is inside a git repository (has .git in path)
+                # Git repos are at {remote_agents_dir}/{owner}/{repo}/.git
+                path_parts = f.relative_to(self.remote_agents_dir).parts
+                if len(path_parts) >= 2:
+                    # Check if this looks like a git repo path (owner/repo)
+                    potential_repo = (
+                        self.remote_agents_dir / path_parts[0] / path_parts[1]
+                    )
+                    if (potential_repo / ".git").exists():
+                        # This file is in a git repo, skip it (we'll handle git repos separately)
+                        self.logger.debug(f"Skipping file in git repo: {f}")
+                        continue
+                filtered_files.append(f)
+            md_files = filtered_files
+
+        self.logger.debug(f"Found {len(md_files)} Markdown files in {scan_dir}")
 
         for md_file in md_files:
             try:
@@ -288,8 +512,14 @@ class RemoteAgentDiscoveryService:
     def _parse_markdown_agent(self, md_file: Path) -> Optional[Dict[str, Any]]:
         """Parse Markdown agent file and convert to JSON template format.
 
-        Expected Markdown format:
+        Expected Markdown format with YAML frontmatter:
         ```markdown
+        ---
+        agent_id: python-engineer
+        name: Python Engineer
+        version: 2.3.0
+        model: sonnet
+        ---
         # Agent Name
 
         Description paragraph (first paragraph after heading)
@@ -303,6 +533,11 @@ class RemoteAgentDiscoveryService:
         - Paths: /path1/, /path2/
         ```
 
+        Agent ID Priority (Mismatch Fix):
+        1. Use agent_id from YAML frontmatter if present (e.g., "python-engineer")
+        2. Fall back to leaf filename if no YAML frontmatter (e.g., "python-engineer.md" -> "python-engineer")
+        3. Store hierarchical path separately as category_path for categorization
+
         Args:
             md_file: Path to Markdown agent file
 
@@ -320,22 +555,54 @@ class RemoteAgentDiscoveryService:
             self.logger.error(f"Failed to read file {md_file}: {e}")
             return None
 
-        # Extract agent name from first heading
-        name_match = re.search(r"^#\s+(.+?)$", content, re.MULTILINE)
-        if not name_match:
-            self.logger.debug(f"No agent name heading found in {md_file.name}")
-            return None
-        name = name_match.group(1).strip()
+        # MISMATCH FIX: Parse YAML frontmatter to extract agent_id
+        frontmatter = self._parse_yaml_frontmatter(content)
 
-        # Extract description (first paragraph after heading, before next heading)
-        desc_match = re.search(
-            r"^#.+?\n\n(.+?)(?:\n\n##|\Z)", content, re.DOTALL | re.MULTILINE
-        )
-        description = desc_match.group(1).strip() if desc_match else ""
+        # MISMATCH FIX: Use agent_id from YAML frontmatter if present, otherwise fall back to filename
+        if frontmatter and "agent_id" in frontmatter:
+            agent_id = frontmatter["agent_id"]
+            self.logger.debug(f"Using agent_id from YAML frontmatter: {agent_id}")
+        else:
+            # Fallback: Use leaf filename without extension
+            agent_id = md_file.stem
+            self.logger.debug(f"No agent_id in YAML, using filename: {agent_id}")
+
+        # Store hierarchical path separately for categorization (not as primary ID)
+        hierarchical_path = self._generate_hierarchical_id(md_file)
 
-        # Extract model from Configuration section
-        model_match = re.search(r"Model:\s*(\w+)", content, re.IGNORECASE)
-        model = model_match.group(1) if model_match else "sonnet"
+        # Extract agent name - prioritize frontmatter over markdown heading
+        # Frontmatter is intentional metadata, headings may be arbitrary content
+        if frontmatter and "name" in frontmatter:
+            name = frontmatter["name"]
+        else:
+            # Fallback to first heading or filename
+            name_match = re.search(r"^#\s+(.+?)$", content, re.MULTILINE)
+            if name_match:
+                name = name_match.group(1).strip()
+            else:
+                # Last resort: derive from filename
+                name = md_file.stem.replace("-", " ").replace("_", " ").title()
+
+        # Extract description - prioritize frontmatter over markdown content
+        # Frontmatter is intentional metadata, paragraphs may be arbitrary content
+        if frontmatter and "description" in frontmatter:
+            description = frontmatter["description"]
+        else:
+            # Fallback to first paragraph after heading
+            desc_match = re.search(
+                r"^#.+?\n\n(.+?)(?:\n\n##|\Z)", content, re.DOTALL | re.MULTILINE
+            )
+            if desc_match:
+                description = desc_match.group(1).strip()
+            else:
+                description = ""
+
+        # Extract model from YAML frontmatter or Configuration section
+        if frontmatter and "model" in frontmatter:
+            model = frontmatter["model"]
+        else:
+            model_match = re.search(r"Model:\s*(\w+)", content, re.IGNORECASE)
+            model = model_match.group(1) if model_match else "sonnet"
 
         # Extract priority from Configuration section
         priority_match = re.search(r"Priority:\s*(\d+)", content, re.IGNORECASE)
@@ -353,12 +620,11 @@ class RemoteAgentDiscoveryService:
         if paths_match:
             paths = [p.strip() for p in paths_match.group(1).split(",")]
 
-        # Get version (SHA-256 hash) from cache metadata
-        version = self._get_agent_version(md_file)
-
-        # Bug #3 fix: Generate hierarchical agent_id from file path
-        # This preserves directory structure for category filtering and preset matching
-        agent_id = self._generate_hierarchical_id(md_file)
+        # Get version (SHA-256 hash) from cache metadata or YAML frontmatter
+        if frontmatter and "version" in frontmatter:
+            version = frontmatter["version"]
+        else:
+            version = self._get_agent_version(md_file)
 
         # Bug #1 fix: Detect category from directory path
         category = self._detect_category_from_path(md_file)
@@ -368,6 +634,7 @@ class RemoteAgentDiscoveryService:
         source_path = self._extract_source_path_from_file(md_file)
 
         # NEW: Generate canonical_id (collection_id:agent_id)
+        # Use leaf agent_id (not hierarchical path) for canonical_id
         if collection_id:
             canonical_id = f"{collection_id}:{agent_id}"
         else:
@@ -378,8 +645,9 @@ class RemoteAgentDiscoveryService:
         # IMPORTANT: Include 'path' field for compatibility with deployment validation (ticket 1M-480)
         # Git-sourced agents must have 'path' field to match structure from AgentDiscoveryService
         return {
-            "agent_id": agent_id,
-            "canonical_id": canonical_id,  # NEW: Primary matching key
+            "agent_id": agent_id,  # MISMATCH FIX: Use leaf name from YAML, not hierarchical path
+            "hierarchical_path": hierarchical_path,  # Store hierarchical path separately
+            "canonical_id": canonical_id,  # NEW: Primary matching key (uses leaf agent_id)
             "collection_id": collection_id,  # NEW: Collection identifier
             "source_path": source_path,  # NEW: Path within repository
             "metadata": {
@@ -388,6 +656,7 @@ class RemoteAgentDiscoveryService:
                 "version": version,
                 "author": "remote",  # Mark as remote agent
                 "category": category,  # Use detected category from path
+                "hierarchical_path": hierarchical_path,  # For categorization/filtering
                 "collection_id": collection_id,  # NEW: Also in metadata
                 "source_path": source_path,  # NEW: Also in metadata
                 "canonical_id": canonical_id,  # NEW: Also in metadata

claude_mpm/services/agents/git_source_manager.py

@@ -339,9 +339,29 @@ class GitSourceManager:
 
             # Walk cache directory structure
             logger.debug(f"[DEBUG] Walking cache root: {self.cache_root}")
+
+            # Known legacy category directories to skip (flat cache structure)
+            LEGACY_CATEGORIES = {
+                "universal",
+                "engineer",
+                "ops",
+                "qa",
+                "security",
+                "documentation",
+                "claude-mpm",
+            }
+
             for owner_dir in self.cache_root.iterdir():
                 if not owner_dir.is_dir():
                     continue
+
+                # Skip legacy category directories (they're not GitHub owners)
+                if owner_dir.name.lower() in LEGACY_CATEGORIES:
+                    logger.debug(
+                        f"[DEBUG] Skipping legacy category directory: {owner_dir.name}"
+                    )
+                    continue
+
                 logger.debug(f"[DEBUG] Processing owner_dir: {owner_dir.name}")
 
                 for repo_dir in owner_dir.iterdir():

claude_mpm/services/agents/sources/git_source_sync_service.py

@@ -851,6 +851,12 @@ class GitSourceSyncService:
             if base_path and not path.startswith(base_path + "/"):
                 continue
 
+            # Exclude build/dist directories (prevents double-counting)
+            # e.g., both "agents/engineer.md" and "dist/agents/engineer.md"
+            path_parts = path.split("/")
+            if any(excluded in path_parts for excluded in ["dist", "build", ".cache"]):
+                continue
+
             # Remove base_path prefix for relative paths
             if base_path:
                 relative_path = path[len(base_path) + 1 :]
@@ -971,7 +977,8 @@ class GitSourceSyncService:
         """
         import shutil
 
-        deployment_dir = project_dir / ".claude-mpm" / "agents"
+        # Deploy to .claude/agents/ where Claude Code expects them
+        deployment_dir = project_dir / ".claude" / "agents"
         deployment_dir.mkdir(parents=True, exist_ok=True)
 
         results = {

claude_mpm/services/agents/toolchain_detector.py

@@ -158,12 +158,13 @@ class ToolchainDetector:
         "make": ["ops"],
     }
 
-    # Core agents always included
+    # Core agents always included (use exact agent IDs from repository)
     CORE_AGENTS = [
-        "qa",
-        "research",
-        "documentation",
+        "qa-agent",
+        "research-agent",
+        "documentation-agent",
         "ticketing",
+        "local-ops-agent",
     ]
 
     # Directories to exclude from scanning
@@ -365,7 +366,7 @@ class ToolchainDetector:
         """Map detected toolchain to recommended agents.
 
         Combines language-specific, framework-specific, and ops agents with
-        core agents (qa, research, documentation, ticketing).
+        core agents (qa-agent, research-agent, documentation-agent, ticketing, local-ops-agent).
 
         Args:
             toolchain: Detected toolchain dictionary with languages, frameworks, tools

claude_mpm/services/analysis/__init__.py

@@ -1,9 +1,15 @@
 """
 Analysis services for Claude MPM.
 
-Provides postmortem analysis and error improvement suggestions.
+Provides postmortem analysis, error improvement suggestions, and code clone detection.
 """
 
+from .clone_detector import (
+    CloneDetector,
+    CloneReport,
+    RefactoringSuggestion,
+    SimilarityReport,
+)
 from .postmortem_service import (
     ActionType,
     ErrorAnalysis,
@@ -16,10 +22,14 @@ from .postmortem_service import (
 
 __all__ = [
     "ActionType",
+    "CloneDetector",
+    "CloneReport",
     "ErrorAnalysis",
     "ErrorCategory",
     "ImprovementAction",
     "PostmortemReport",
     "PostmortemService",
+    "RefactoringSuggestion",
+    "SimilarityReport",
     "get_postmortem_service",
 ]