claude-mpm 4.1.26__py3-none-any.whl → 5.0.9__py3-none-any.whl

claude_mpm/utils/agent_dependency_loader.py

@@ -14,8 +14,9 @@ import logging
 import subprocess
 import sys
 import time
-from typing import Dict, List, Optional, Set, Tuple
+from typing import Any, Dict, List, Optional, Set, Tuple
 
+import yaml
 from packaging.requirements import InvalidRequirement, Requirement
 
 from ..core.logger import get_logger
@@ -31,6 +32,22 @@ class AgentDependencyLoader:
     and being used, rather than all possible agents.
     """
 
+    # Optional database packages - if one fails, try alternatives
+    OPTIONAL_DB_PACKAGES = {
+        "mysqlclient": ["pymysql"],  # PyMySQL is a pure Python alternative
+        "psycopg2": ["psycopg2-binary"],  # Binary version doesn't require compilation
+        "cx_Oracle": [],  # No good alternative, mark as optional
+    }
+
+    # Packages that commonly fail compilation on certain platforms
+    COMPILATION_PRONE = [
+        "mysqlclient",  # Requires MySQL development headers
+        "psycopg2",  # Requires PostgreSQL development headers (use psycopg2-binary instead)
+        "cx_Oracle",  # Requires Oracle client libraries
+        "pycairo",  # Requires Cairo development headers
+        "lxml",  # Can fail if libxml2 dev headers missing
+    ]
+
     def __init__(self, auto_install: bool = False):
         """
         Initialize the agent dependency loader.
@@ -44,6 +61,7 @@ class AgentDependencyLoader:
         self.agent_dependencies: Dict[str, Dict] = {}
         self.missing_dependencies: Dict[str, List[str]] = {}
         self.checked_packages: Set[str] = set()
+        self.optional_failed: Dict[str, str] = {}  # Track optional packages that failed
         self.deployment_state_file = (
             Path.cwd() / ".claude" / "agents" / ".mpm_deployment_state"
         )
@@ -55,7 +73,7 @@ class AgentDependencyLoader:
         Returns:
             Dictionary mapping agent IDs to their file paths
         """
-        deployed_agents = {}
+        deployed_agents: Dict[str, Path] = {}
         claude_agents_dir = Path.cwd() / ".claude" / "agents"
 
         if not claude_agents_dir.exists():
@@ -68,14 +86,52 @@ class AgentDependencyLoader:
             deployed_agents[agent_id] = agent_file
             logger.debug(f"Found deployed agent: {agent_id}")
 
-        logger.info(f"Discovered {len(deployed_agents)} deployed agents")
+        logger.debug(f"Discovered {len(deployed_agents)} deployed agents")
         self.deployed_agents = deployed_agents
         return deployed_agents
 
+    def _extract_yaml_frontmatter(self, content: str) -> Optional[Dict[str, Any]]:
+        """
+        Extract and parse YAML frontmatter from markdown.
+
+        Frontmatter must be at the start of the file, delimited by '---'.
+        Example:
+            ---
+            name: agent_name
+            dependencies:
+              python:
+                - package>=1.0.0
+            ---
+            # Agent content...
+
+        Args:
+            content: File content to parse
+
+        Returns:
+            Parsed YAML frontmatter as dict, or None if not found/invalid
+        """
+        if not content.strip().startswith("---"):
+            return None
+
+        # Split on --- delimiters
+        parts = content.split("---", 2)
+        if len(parts) < 3:
+            return None
+
+        try:
+            return yaml.safe_load(parts[1])
+        except yaml.YAMLError as e:
+            logger.warning(f"Failed to parse YAML frontmatter: {e}")
+            return None
+
     def load_agent_dependencies(self) -> Dict[str, Dict]:
         """
         Load dependency information for deployed agents from their source configs.
 
+        Searches for agent configuration in both markdown (.md) and JSON (.json) formats.
+        Markdown files with YAML frontmatter are searched first for better maintainability.
+        Falls back to JSON format for backward compatibility.
+
         Returns:
             Dictionary mapping agent IDs to their dependency requirements
         """
@@ -89,22 +145,50 @@ class AgentDependencyLoader:
         ]
 
         for agent_id in self.deployed_agents:
-            # Try to find the agent's JSON config
+            found = False
+
+            # Try to find the agent's config (markdown first, then JSON)
             for config_dir in config_paths:
-                config_file = config_dir / f"{agent_id}.json"
-                if config_file.exists():
+                if found:
+                    break
+
+                # Try markdown first (current format with YAML frontmatter)
+                md_file = config_dir / f"{agent_id}.md"
+                if md_file.exists():
                     try:
-                        with open(config_file) as f:
-                            config = json.load(f)
-                            if "dependencies" in config:
-                                agent_dependencies[agent_id] = config["dependencies"]
-                                logger.debug(f"Loaded dependencies for {agent_id}")
-                                break
+                        content = md_file.read_text(encoding="utf-8")
+                        frontmatter = self._extract_yaml_frontmatter(content)
+                        if frontmatter and "dependencies" in frontmatter:
+                            agent_dependencies[agent_id] = frontmatter["dependencies"]
+                            logger.debug(
+                                f"Loaded dependencies for {agent_id} from markdown"
+                            )
+                            found = True
+                            break
                     except Exception as e:
-                        logger.warning(f"Failed to load config for {agent_id}: {e}")
+                        logger.warning(f"Failed to load markdown for {agent_id}: {e}")
+
+                # Fall back to JSON for backward compatibility
+                if not found:
+                    json_file = config_dir / f"{agent_id}.json"
+                    if json_file.exists():
+                        try:
+                            with json_file.open() as f:
+                                config = json.load(f)
+                                if "dependencies" in config:
+                                    agent_dependencies[agent_id] = config[
+                                        "dependencies"
+                                    ]
+                                    logger.debug(
+                                        f"Loaded dependencies for {agent_id} from JSON"
+                                    )
+                                    found = True
+                                    break
+                        except Exception as e:
+                            logger.warning(f"Failed to load JSON for {agent_id}: {e}")
 
         self.agent_dependencies = agent_dependencies
-        logger.info(f"Loaded dependencies for {len(agent_dependencies)} agents")
+        logger.debug(f"Loaded dependencies for {len(agent_dependencies)} agents")
         return agent_dependencies
 
     def check_python_dependency(self, package_spec: str) -> Tuple[bool, Optional[str]]:
@@ -130,6 +214,13 @@ class AgentDependencyLoader:
                 self.checked_packages.add(package_name)
                 return True, "built-in"
 
+            # Check if this is an optional package that already failed
+            if package_name in self.optional_failed:
+                logger.debug(
+                    f"Skipping optional package {package_name} (previously failed)"
+                )
+                return True, "optional-skipped"
+
             # Try to import and check version
             try:
                 import importlib.metadata
@@ -147,6 +238,24 @@ class AgentDependencyLoader:
                     return False, version
 
                 except importlib.metadata.PackageNotFoundError:
+                    # Check if there's an alternative for this optional package
+                    if package_name in self.OPTIONAL_DB_PACKAGES:
+                        for alternative in self.OPTIONAL_DB_PACKAGES[package_name]:
+                            try:
+                                alt_version = importlib.metadata.version(alternative)
+                                logger.info(
+                                    f"Using {alternative} as alternative to {package_name}"
+                                )
+                                self.checked_packages.add(package_name)
+                                return True, f"{alternative}:{alt_version}"
+                            except importlib.metadata.PackageNotFoundError:
+                                continue
+                        # If no alternatives work, mark as optional failure
+                        self.optional_failed[package_name] = "No alternatives available"
+                        logger.warning(
+                            f"Optional package {package_name} not found, marking as optional"
+                        )
+                        return True, "optional-not-found"
                     return False, None
 
             except ImportError:
@@ -162,6 +271,25 @@ class AgentDependencyLoader:
                     return False, version
 
                 except pkg_resources.DistributionNotFound:
+                    # Check alternatives for optional packages
+                    if package_name in self.OPTIONAL_DB_PACKAGES:
+                        for alternative in self.OPTIONAL_DB_PACKAGES[package_name]:
+                            try:
+                                alt_version = pkg_resources.get_distribution(
+                                    alternative
+                                ).version
+                                logger.info(
+                                    f"Using {alternative} as alternative to {package_name}"
+                                )
+                                self.checked_packages.add(package_name)
+                                return True, f"{alternative}:{alt_version}"
+                            except pkg_resources.DistributionNotFound:
+                                continue
+                        self.optional_failed[package_name] = "No alternatives available"
+                        logger.warning(
+                            f"Optional package {package_name} not found, marking as optional"
+                        )
+                        return True, "optional-not-found"
                     return False, None
 
         except InvalidRequirement as e:
@@ -348,7 +476,7 @@ class AgentDependencyLoader:
         Returns:
             Analysis results including missing and satisfied dependencies
         """
-        results = {
+        results: Dict[str, Any] = {
             "agents": {},
             "summary": {
                 "total_agents": len(self.deployed_agents),
@@ -361,7 +489,7 @@ class AgentDependencyLoader:
         }
 
         for agent_id, deps in self.agent_dependencies.items():
-            agent_result = {
+            agent_result: Dict[str, Dict[str, List[str]]] = {
                 "python": {"satisfied": [], "missing": [], "outdated": []},
                 "system": {"satisfied": [], "missing": []},
             }
@@ -415,8 +543,8 @@ class AgentDependencyLoader:
         """
         import sys
 
-        compatible = []
-        incompatible = []
+        compatible: List[str] = []
+        incompatible: List[str] = []
 
         for dep in dependencies:
             try:
@@ -532,20 +660,37 @@ class AgentDependencyLoader:
             try:
                 cmd = [sys.executable, "-m", "pip", "install"]
 
-                # Check for PEP 668 managed environment
+                # Check environment and add appropriate flags
+                import os
                 import sysconfig
 
-                stdlib_path = sysconfig.get_path("stdlib")
-                marker_file = Path(stdlib_path) / "EXTERNALLY-MANAGED"
-                parent_marker = marker_file.parent.parent / "EXTERNALLY-MANAGED"
+                # Check if in virtualenv
+                in_virtualenv = (
+                    (hasattr(sys, "base_prefix") and sys.base_prefix != sys.prefix)
+                    or (hasattr(sys, "real_prefix"))
+                    or (os.environ.get("VIRTUAL_ENV") is not None)
+                )
 
-                if marker_file.exists() or parent_marker.exists():
-                    logger.warning(
-                        "PEP 668 managed environment detected. "
-                        "Installing with --break-system-packages --user flags. "
-                        "Consider using a virtual environment instead."
-                    )
-                    cmd.extend(["--break-system-packages", "--user"])
+                if in_virtualenv:
+                    # In virtualenv - no special flags needed
+                    logger.debug("Installing in virtualenv (no special flags)")
+                else:
+                    # Check for PEP 668 managed environment
+                    stdlib_path = sysconfig.get_path("stdlib")
+                    marker_file = Path(stdlib_path) / "EXTERNALLY-MANAGED"
+                    parent_marker = marker_file.parent.parent / "EXTERNALLY-MANAGED"
+
+                    if marker_file.exists() or parent_marker.exists():
+                        logger.warning(
+                            "PEP 668 managed environment detected. "
+                            "Installing with --break-system-packages flag. "
+                            "Consider using a virtual environment instead."
+                        )
+                        cmd.append("--break-system-packages")
+                    else:
+                        # Normal system Python - use --user
+                        cmd.append("--user")
+                        logger.debug("Installing with --user flag")
 
                 cmd.extend(compatible)
 
@@ -600,7 +745,7 @@ class AgentDependencyLoader:
             logger.info(
                 f"Auto-installing {len(results['summary']['missing_python'])} missing dependencies..."
             )
-            success, error = self.install_missing_dependencies(
+            success, _error = self.install_missing_dependencies(
                 results["summary"]["missing_python"]
             )
             if success:
@@ -699,7 +844,7 @@ class AgentDependencyLoader:
             import sys
 
             if sys.version_info >= (3, 13):
-                compatible, incompatible = self.check_python_compatibility(
+                _compatible, incompatible = self.check_python_compatibility(
                     summary["missing_python"]
                 )
                 if incompatible:
@@ -774,7 +919,7 @@ class AgentDependencyLoader:
                 hash_obj.update(str(stat.st_size).encode("utf-8"))
 
                 # Include file content for comprehensive change detection
-                with open(agent_path, "rb") as f:
+                with agent_path.open("rb") as f:
                     hash_obj.update(f.read())
             except Exception as e:
                 logger.debug(f"Could not hash agent file {agent_path}: {e}")
@@ -794,7 +939,7 @@ class AgentDependencyLoader:
             return {}
 
         try:
-            with open(self.deployment_state_file) as f:
+            with self.deployment_state_file.open() as f:
                 return json.load(f)
         except Exception as e:
             logger.debug(f"Could not load deployment state: {e}")
@@ -811,7 +956,7 @@ class AgentDependencyLoader:
             # Ensure directory exists
             self.deployment_state_file.parent.mkdir(parents=True, exist_ok=True)
 
-            with open(self.deployment_state_file, "w") as f:
+            with self.deployment_state_file.open("w") as f:
                 json.dump(state, f, indent=2)
         except Exception as e:
             logger.debug(f"Could not save deployment state: {e}")
@@ -871,7 +1016,7 @@ class AgentDependencyLoader:
         Returns:
             Cached results or None if not available/valid.
         """
-        has_changed, current_hash = self.has_agents_changed()
+        has_changed, _current_hash = self.has_agents_changed()
 
         if not has_changed:
             state = self.load_deployment_state()
@@ -885,13 +1030,16 @@ class AgentDependencyLoader:
 
 def check_deployed_agent_dependencies(
     auto_install: bool = False, verbose: bool = False
-) -> None:
+) -> int:
     """
     Check dependencies for currently deployed agents.
 
     Args:
         auto_install: If True, automatically install missing Python dependencies
         verbose: If True, enable verbose logging
+
+    Returns:
+        Status code: 0 if all dependencies satisfied, 1 if missing dependencies
     """
     if verbose:
         logging.getLogger().setLevel(logging.DEBUG)

claude_mpm/utils/agent_filters.py

@@ -0,0 +1,288 @@
+"""
+Agent filtering utilities for claude-mpm.
+
+WHY: This module provides centralized filtering logic to remove non-deployable
+agents (BASE_AGENT) and already-deployed agents from user-facing displays.
+
+DESIGN DECISIONS:
+- BASE_AGENT is a build tool, not a deployable agent - filter everywhere
+- Deployed agent detection supports both new (.claude-mpm/agents/) and
+  legacy (.claude/agents/)
+- Case-insensitive BASE_AGENT detection for robustness
+- Pure functions for easy testing and reuse
+
+IMPLEMENTATION NOTES:
+- Related to ticket 1M-502 Phase 1: UX improvements for agent filtering
+- Addresses user confusion from seeing BASE_AGENT and deployed agents in lists
+"""
+
+from pathlib import Path
+from typing import Dict, List, Optional, Set
+
+
+def is_base_agent(agent_id: str) -> bool:
+    """Check if agent is BASE_AGENT (build tool, not deployable).
+
+    BASE_AGENT is an internal build tool used to construct other agents.
+    It should never appear in user-facing agent lists or deployment menus.
+
+    Args:
+        agent_id: Agent identifier to check (may include path like "qa/BASE-AGENT")
+
+    Returns:
+        True if agent is BASE_AGENT (case-insensitive), False otherwise
+
+    Examples:
+        >>> is_base_agent("BASE_AGENT")
+        True
+        >>> is_base_agent("base-agent")
+        True
+        >>> is_base_agent("qa/BASE-AGENT")
+        True
+        >>> is_base_agent("ENGINEER")
+        False
+    """
+    if not agent_id:
+        return False
+
+    # Extract filename from path (handle cases like "qa/BASE-AGENT")
+    # 1M-502: Remote agents may have path prefixes like "qa/", "pm/", etc.
+    agent_name = agent_id.split("/")[-1]
+
+    normalized_id = agent_name.lower().replace("-", "").replace("_", "")
+    return normalized_id == "baseagent"
+
+
+def filter_base_agents(agents: List[Dict]) -> List[Dict]:
+    """Remove BASE_AGENT from agent list.
+
+    Filters out any agent with agent_id matching BASE_AGENT (case-insensitive).
+    This prevents users from seeing or selecting the internal build tool.
+
+    Args:
+        agents: List of agent dictionaries, each containing at least 'agent_id' key
+
+    Returns:
+        Filtered list with BASE_AGENT removed
+
+    Examples:
+        >>> agents = [
+        ...     {"agent_id": "ENGINEER", "name": "Engineer"},
+        ...     {"agent_id": "BASE_AGENT", "name": "Base Agent"},
+        ...     {"agent_id": "PM", "name": "PM"}
+        ... ]
+        >>> filtered = filter_base_agents(agents)
+        >>> len(filtered)
+        2
+        >>> "BASE_AGENT" in [a["agent_id"] for a in filtered]
+        False
+    """
+    return [a for a in agents if not is_base_agent(a.get("agent_id", ""))]
+
+
+def get_deployed_agent_ids(project_dir: Optional[Path] = None) -> Set[str]:
+    """Get set of currently deployed agent IDs.
+
+    Checks virtual deployment state (.mpm_deployment_state) first, then falls back
+    to physical .md files for backward compatibility. This ensures agents are detected
+    whether deployed virtually or as physical files.
+
+    Args:
+        project_dir: Project directory to check, defaults to current working directory
+
+    Returns:
+        Set of deployed agent IDs (leaf names like "python-engineer", "qa")
+
+    Examples:
+        >>> deployed = get_deployed_agent_ids()
+        >>> "python-engineer" in deployed  # If agent exists in deployment state
+        True
+        >>> "ENGINEER" in deployed  # If ENGINEER.md exists
+        True
+
+    Design Rationale:
+        - Primary detection: Virtual deployment state (.mpm_deployment_state)
+        - Fallback detection: Physical .md files (.claude-mpm/agents/, .claude/agents/)
+        - Returns leaf names for consistent comparison with agent_id formats
+        - Combines both detection methods for complete coverage
+        - Graceful error handling for malformed or missing state files
+
+    Related:
+        - Fixes checkbox interface showing all agents as "○ [Available]" instead of "● [Installed]"
+        - Matches detection logic from _is_agent_deployed() in agent_state_manager.py
+        - Related to ticket 1M-502: Virtual deployment state detection
+    """
+    deployed = set()
+
+    # Track if project_dir was explicitly provided
+    explicit_project_dir = project_dir is not None
+
+    if project_dir is None:
+        project_dir = Path.cwd()
+
+    # NEW: Check virtual deployment state (primary method)
+    # This is the current deployment model used by Claude Code
+    deployment_state_paths = [
+        project_dir / ".claude" / "agents" / ".mpm_deployment_state",
+    ]
+
+    # Only check user-level state if using default project directory
+    # This prevents test isolation issues when explicit project_dir is provided
+    if not explicit_project_dir:
+        deployment_state_paths.append(
+            Path.home() / ".claude" / "agents" / ".mpm_deployment_state"
+        )
+
+    for state_path in deployment_state_paths:
+        if state_path.exists():
+            try:
+                import json
+
+                with state_path.open() as f:
+                    state = json.load(f)
+
+                # Extract agent IDs from deployment state
+                # Agent IDs are leaf names (e.g., "python-engineer", "qa")
+                agents = state.get("last_check_results", {}).get("agents", {})
+                deployed.update(agents.keys())
+
+            except (json.JSONDecodeError, KeyError) as e:
+                # Log error but continue - don't break if state file is malformed
+                import logging
+
+                logger = logging.getLogger(__name__)
+                logger.debug(f"Failed to read deployment state from {state_path}: {e}")
+                continue
+            except Exception as e:
+                # Catch unexpected errors - fail gracefully
+                import logging
+
+                logger = logging.getLogger(__name__)
+                logger.debug(f"Unexpected error reading deployment state: {e}")
+                continue
+
+    # EXISTING: Check physical .md files (fallback for backward compatibility)
+    # Check new architecture
+    new_agents_dir = project_dir / ".claude-mpm" / "agents"
+    if new_agents_dir.exists():
+        for file in new_agents_dir.glob("*.md"):
+            if file.stem not in {"BASE-AGENT", ".DS_Store"}:
+                deployed.add(file.stem)
+
+    # Check legacy architecture
+    legacy_agents_dir = project_dir / ".claude" / "agents"
+    if legacy_agents_dir.exists():
+        for file in legacy_agents_dir.glob("*.md"):
+            if file.stem not in {"BASE-AGENT", ".DS_Store"}:
+                deployed.add(file.stem)
+
+    # Check .claude/templates/ directory (where agents are actually deployed)
+    templates_dir = project_dir / ".claude" / "templates"
+    if templates_dir.exists():
+        for file in templates_dir.glob("*.md"):
+            if file.stem not in {
+                "BASE-AGENT",
+                ".DS_Store",
+                "README",
+                "circuit-breakers",
+            }:
+                # Skip template/example files
+                if not any(x in file.stem for x in ["example", "template", "pm-"]):
+                    deployed.add(file.stem)
+
+    # Check user-level directory only if using default project directory
+    # This prevents test isolation issues when explicit project_dir is provided
+    if not explicit_project_dir:
+        user_agents_dir = Path.home() / ".claude" / "agents"
+        if user_agents_dir.exists():
+            for file in user_agents_dir.glob("*.md"):
+                if file.stem not in {"BASE-AGENT", ".DS_Store"}:
+                    deployed.add(file.stem)
+
+    return deployed
+
+
+def filter_deployed_agents(
+    agents: List[Dict], project_dir: Optional[Path] = None
+) -> List[Dict]:
+    """Remove already-deployed agents from list.
+
+    Filters agent list to show only agents that are not currently deployed.
+    This prevents users from attempting to re-deploy existing agents and
+    reduces confusion in deployment menus.
+
+    Args:
+        agents: List of agent dictionaries, each containing at least 'agent_id' key
+        project_dir: Project directory to check, defaults to current working directory
+
+    Returns:
+        Filtered list containing only non-deployed agents
+
+    Examples:
+        >>> agents = [
+        ...     {"agent_id": "ENGINEER", "name": "Engineer"},
+        ...     {"agent_id": "PM", "name": "PM"},
+        ...     {"agent_id": "QA", "name": "QA"}
+        ... ]
+        >>> # Assuming ENGINEER is deployed
+        >>> filtered = filter_deployed_agents(agents)
+        >>> "ENGINEER" not in [a["agent_id"] for a in filtered]
+        True
+
+    Design Rationale:
+        - Checks filesystem for actual deployed files (source of truth)
+        - Supports both new and legacy agent directory structures
+        - Preserves agent order for consistent UX
+    """
+    deployed_ids = get_deployed_agent_ids(project_dir)
+    return [a for a in agents if a.get("agent_id") not in deployed_ids]
+
+
+def apply_all_filters(
+    agents: List[Dict],
+    project_dir: Optional[Path] = None,
+    filter_base: bool = True,
+    filter_deployed: bool = False,
+) -> List[Dict]:
+    """Apply multiple filters to agent list in correct order.
+
+    Convenience function to apply common filtering combinations. Filters are
+    applied in this order:
+    1. BASE_AGENT filtering (if enabled)
+    2. Deployed agent filtering (if enabled)
+
+    Args:
+        agents: List of agent dictionaries to filter
+        project_dir: Project directory for deployment checks
+        filter_base: Remove BASE_AGENT from list (default: True)
+        filter_deployed: Remove deployed agents from list (default: False)
+
+    Returns:
+        Filtered agent list
+
+    Examples:
+        >>> agents = get_all_agents()
+        >>> # For display/info purposes - remove only BASE_AGENT
+        >>> filtered = apply_all_filters(
+        ...     agents, filter_base=True, filter_deployed=False
+        ... )
+        >>> # For deployment menus - remove BASE_AGENT and deployed agents
+        >>> deployable = apply_all_filters(
+        ...     agents, filter_base=True, filter_deployed=True
+        ... )
+
+    Usage Guidelines:
+        - Use filter_base=True (default) for all user-facing displays
+        - Use filter_deployed=True when showing deployment options
+        - Use filter_deployed=False when showing all available agents
+          (info/list commands)
+    """
+    result = agents
+
+    if filter_base:
+        result = filter_base_agents(result)
+
+    if filter_deployed:
+        result = filter_deployed_agents(result, project_dir)
+
+    return result