claude-mpm 5.0.9__py3-none-any.whl → 5.6.23__py3-none-any.whl

claude_mpm/services/skills_deployer.py

@@ -28,7 +28,7 @@ References:
 import json
 import platform
 import shutil
-import subprocess
+import subprocess  # nosec B404 - subprocess needed for safe git operations
 from pathlib import Path
 from typing import Any, Dict, List, Optional
 
@@ -82,6 +82,8 @@ class SkillsDeployerService(LoggerMixin):
         toolchain: Optional[List[str]] = None,
         categories: Optional[List[str]] = None,
         force: bool = False,
+        selective: bool = True,
+        project_root: Optional[Path] = None,
     ) -> Dict:
         """Deploy skills from GitHub repository.
 
@@ -89,14 +91,17 @@ class SkillsDeployerService(LoggerMixin):
         1. Downloads skills from GitHub collection
         2. Parses manifest for metadata
         3. Filters by toolchain and categories
-        4. Deploys to ~/.claude/skills/
-        5. Warns about Claude Code restart
+        4. (If selective=True) Filters to only agent-referenced skills
+        5. Deploys to ~/.claude/skills/
+        6. Warns about Claude Code restart
 
         Args:
             collection: Collection name to deploy from (default: uses default collection)
             toolchain: Filter by toolchain (e.g., ['python', 'javascript'])
             categories: Filter by categories (e.g., ['testing', 'debugging'])
             force: Overwrite existing skills
+            selective: If True, only deploy skills referenced by agents (default)
+            project_root: Project root directory (for finding agents, auto-detected if None)
 
         Returns:
             Dict containing:
@@ -107,10 +112,14 @@ class SkillsDeployerService(LoggerMixin):
             - restart_required: True if Claude Code needs restart
             - restart_instructions: Message about restarting
             - collection: Collection name used for deployment
+            - selective_mode: True if selective deployment was used
+            - total_available: Total skills available before filtering
 
         Example:
             >>> result = deployer.deploy_skills(collection="obra-superpowers")
             >>> result = deployer.deploy_skills(toolchain=['python'])  # Uses default
+            >>> # Deploy all skills (not just agent-referenced)
+            >>> result = deployer.deploy_skills(selective=False)
             >>> if result['restart_required']:
             >>>     print(result['restart_instructions'])
         """
@@ -152,7 +161,7 @@ class SkillsDeployerService(LoggerMixin):
 
         self.logger.info(f"Found {len(skills)} skills in repository")
 
-        # Step 3: Filter skills
+        # Step 3: Filter skills by toolchain and categories
         filtered_skills = self._filter_skills(skills, toolchain, categories)
 
         self.logger.info(
@@ -160,11 +169,124 @@ class SkillsDeployerService(LoggerMixin):
             f" (toolchain={toolchain}, categories={categories})"
         )
 
+        # Step 3.5: Apply selective filtering (only agent-referenced skills)
+        total_available = len(filtered_skills)
+        if selective:
+            # Auto-detect project root if not provided
+            if project_root is None:
+                # Try to find project root by looking for .claude-mpm directory
+                # Start from current directory and walk up
+                current = Path.cwd()
+                while current != current.parent:
+                    if (current / ".claude-mpm").exists():
+                        project_root = current
+                        break
+                    current = current.parent
+
+            # Read skills from configuration.yaml instead of agent frontmatter
+            if project_root:
+                config_path = Path(project_root) / ".claude-mpm" / "configuration.yaml"
+            else:
+                # Fallback to current directory's configuration
+                config_path = Path.cwd() / ".claude-mpm" / "configuration.yaml"
+
+            from claude_mpm.services.skills.selective_skill_deployer import (
+                get_required_skills_from_agents,
+                get_skills_to_deploy,
+                save_agent_skills_to_config,
+            )
+
+            # Check if agent_referenced is empty and needs to be populated
+            required_skill_names, source = get_skills_to_deploy(config_path)
+
+            if not required_skill_names and project_root:
+                # agent_referenced is empty, scan deployed agents to populate it
+                agents_dir = Path(project_root) / ".claude" / "agents"
+                if agents_dir.exists():
+                    self.logger.info(
+                        "agent_referenced is empty in configuration.yaml, scanning deployed agents..."
+                    )
+                    agent_skills = get_required_skills_from_agents(agents_dir)
+                    if agent_skills:
+                        save_agent_skills_to_config(list(agent_skills), config_path)
+                        self.logger.info(
+                            f"Populated agent_referenced with {len(agent_skills)} skills from deployed agents"
+                        )
+                        # Re-read configuration after update
+                        required_skill_names, source = get_skills_to_deploy(config_path)
+                    else:
+                        self.logger.warning(
+                            "No skills found in deployed agents - configuration.yaml remains empty"
+                        )
+                else:
+                    self.logger.warning(
+                        f"Agents directory not found at {agents_dir} - cannot scan for skills"
+                    )
+
+            if required_skill_names:
+                # Convert required_skill_names to a set for O(1) lookup
+                required_set = set(required_skill_names)
+
+                # Filter to only required skills
+                # Match on: 'name', 'skill_id', or normalized 'source_path'
+                # source_path example: "universal/web/api-design-patterns/SKILL.md"
+                # normalized: "universal-web-api-design-patterns"
+                def skill_matches_requirement(skill):
+                    # Check basic name and skill_id
+                    if skill.get("name") in required_set:
+                        return True
+                    if skill.get("skill_id") in required_set:
+                        return True
+
+                    # Check normalized source_path
+                    source_path = skill.get("source_path", "")
+                    if source_path:
+                        # Remove /SKILL.md suffix and replace / with -
+                        normalized = source_path.replace("/SKILL.md", "").replace(
+                            "/", "-"
+                        )
+                        if normalized in required_set:
+                            return True
+
+                    return False
+
+                filtered_skills = [
+                    s for s in filtered_skills if skill_matches_requirement(s)
+                ]
+
+                self.logger.info(
+                    f"Selective deployment: {len(filtered_skills)}/{total_available} skills "
+                    f"(source: {source})"
+                )
+            else:
+                self.logger.warning(
+                    f"No skills found in configuration at {config_path}. "
+                    f"Deploying all {total_available} skills."
+                )
+        else:
+            self.logger.info(
+                f"Selective mode disabled: deploying all {total_available} skills"
+            )
+
         # Step 4: Deploy skills
         deployed = []
         skipped = []
         errors = []
 
+        # Extract normalized skill names for cleanup (needed regardless of deployment outcome)
+        # Must match the names used during deployment (normalized from source_path)
+        filtered_skills_names = []
+        for skill in filtered_skills:
+            if isinstance(skill, dict) and "name" in skill:
+                source_path = skill.get("source_path", "")
+                if source_path:
+                    # Normalize: "universal/web/api-design-patterns/SKILL.md" -> "universal-web-api-design-patterns"
+                    normalized = source_path.replace("/SKILL.md", "").replace("/", "-")
+                    filtered_skills_names.append(normalized)
+                else:
+                    # Fallback to skill name
+                    filtered_skills_names.append(skill["name"])
+
         for skill in filtered_skills:
             try:
                 # Validate skill is a dictionary
@@ -173,11 +295,29 @@ class SkillsDeployerService(LoggerMixin):
                     errors.append(f"Invalid skill format: {skill}")
                     continue
 
-                result = self._deploy_skill(skill, skills_data["temp_dir"], force=force)
+                result = self._deploy_skill(
+                    skill, skills_data["temp_dir"], collection_name, force=force
+                )
                 if result["deployed"]:
-                    deployed.append(skill["name"])
+                    # Use normalized name for reporting
+                    source_path = skill.get("source_path", "")
+                    if source_path:
+                        normalized = source_path.replace("/SKILL.md", "").replace(
+                            "/", "-"
+                        )
+                        deployed.append(normalized)
+                    else:
+                        deployed.append(skill["name"])
                 elif result["skipped"]:
-                    skipped.append(skill["name"])
+                    # Use normalized name for reporting
+                    source_path = skill.get("source_path", "")
+                    if source_path:
+                        normalized = source_path.replace("/SKILL.md", "").replace(
+                            "/", "-"
+                        )
+                        skipped.append(normalized)
+                    else:
+                        skipped.append(skill["name"])
                 if result["error"]:
                     errors.append(result["error"])
             except Exception as e:
@@ -189,10 +329,34 @@ class SkillsDeployerService(LoggerMixin):
                 self.logger.error(f"Failed to deploy {skill_name}: {e}")
                 errors.append(f"{skill_name}: {e}")
 
-        # Step 5: Cleanup
+        # Step 5: Cleanup orphaned skills (always run in selective mode)
+        cleanup_result = {"removed_count": 0, "removed_skills": []}
+        if selective:
+            # Get the set of skills that should remain deployed
+            # This is the union of what we just deployed and what was already there
+            try:
+                from claude_mpm.services.skills.selective_skill_deployer import (
+                    cleanup_orphan_skills,
+                )
+
+                # Cleanup orphaned skills not referenced by agents
+                # This runs even if nothing new was deployed to remove stale skills
+                cleanup_result = cleanup_orphan_skills(
+                    self.CLAUDE_SKILLS_DIR, set(filtered_skills_names)
+                )
+
+                if cleanup_result["removed_count"] > 0:
+                    self.logger.info(
+                        f"Removed {cleanup_result['removed_count']} orphaned skills: "
+                        f"{', '.join(cleanup_result['removed_skills'])}"
+                    )
+            except Exception as e:
+                self.logger.warning(f"Failed to cleanup orphaned skills: {e}")
+
+        # Step 6: Cleanup temp directory
         self._cleanup(skills_data["temp_dir"])
 
-        # Step 6: Check if Claude Code restart needed
+        # Step 7: Check if Claude Code restart needed
         restart_required = len(deployed) > 0
         restart_instructions = ""
 
@@ -216,7 +380,8 @@ class SkillsDeployerService(LoggerMixin):
 
         self.logger.info(
             f"Deployment complete: {len(deployed)} deployed, "
-            f"{len(skipped)} skipped, {len(errors)} errors"
+            f"{len(skipped)} skipped, {len(errors)} errors, "
+            f"{cleanup_result['removed_count']} orphaned skills removed"
         )
 
         return {
@@ -228,6 +393,9 @@ class SkillsDeployerService(LoggerMixin):
             "restart_required": restart_required,
             "restart_instructions": restart_instructions,
             "collection": collection_name,
+            "selective_mode": selective,
+            "total_available": total_available,
+            "cleanup": cleanup_result,
         }
 
     def list_available_skills(self, collection: Optional[str] = None) -> Dict:
@@ -412,6 +580,13 @@ class SkillsDeployerService(LoggerMixin):
                 removed.append(skill_name)
                 self.logger.info(f"Removed skill: {skill_name}")
 
+                # Untrack skill from deployment index
+                from claude_mpm.services.skills.selective_skill_deployer import (
+                    untrack_skill,
+                )
+
+                untrack_skill(self.CLAUDE_SKILLS_DIR, skill_name)
+
             except Exception as e:
                 self.logger.error(f"Failed to remove {skill_name}: {e}")
                 errors.append(f"{skill_name}: {e}")
@@ -478,7 +653,7 @@ class SkillsDeployerService(LoggerMixin):
                     f"Updating existing collection '{collection_name}' at {target_dir}"
                 )
                 try:
-                    result = subprocess.run(
+                    result = subprocess.run(  # nosec B603 B607 - Safe: hardcoded git command
                         ["git", "pull"],
                         cwd=target_dir,
                         capture_output=True,
@@ -509,7 +684,7 @@ class SkillsDeployerService(LoggerMixin):
                 f"Installing new collection '{collection_name}' to {target_dir}"
             )
             try:
-                result = subprocess.run(
+                result = subprocess.run(  # nosec B603 B607 - Safe: hardcoded git command
                     ["git", "clone", repo_url, str(target_dir)],
                     capture_output=True,
                     text=True,
@@ -598,6 +773,32 @@ class SkillsDeployerService(LoggerMixin):
         if isinstance(skills_data, dict):
             flat_skills = []
 
+            # Define valid top-level categories
+            VALID_CATEGORIES = {"universal", "toolchains"}
+
+            # Check for unknown categories and warn user
+            unknown_categories = set(skills_data.keys()) - VALID_CATEGORIES
+            if unknown_categories:
+                # Count skills in unknown categories
+                skipped_count = 0
+                for cat in unknown_categories:
+                    cat_data = skills_data.get(cat, [])
+                    if isinstance(cat_data, list):
+                        skipped_count += len(cat_data)
+                    elif isinstance(cat_data, dict):
+                        # If it's a dict like toolchains, count nested skills
+                        for skills_list in cat_data.values():
+                            if isinstance(skills_list, list):
+                                skipped_count += len(skills_list)
+
+                self.logger.warning(
+                    f"Unknown categories in manifest will be skipped: "
+                    f"{', '.join(sorted(unknown_categories))} ({skipped_count} skills)"
+                )
+                self.logger.info(
+                    f"Valid top-level categories: {', '.join(sorted(VALID_CATEGORIES))}"
+                )
+
             # Add universal skills
             universal_skills = skills_data.get("universal", [])
             if isinstance(universal_skills, list):
@@ -677,72 +878,101 @@ class SkillsDeployerService(LoggerMixin):
         return filtered
 
     def _deploy_skill(
-        self, skill: Dict, collection_dir: Path, force: bool = False
+        self,
+        skill: Dict,
+        collection_dir: Path,
+        collection_name: str,
+        force: bool = False,
     ) -> Dict:
-        """Deploy a single skill to ~/.claude/skills/.
+        """Deploy a single skill to ~/.claude/skills/ and track deployment.
 
         NOTE: With multi-collection support, skills are now stored in collection
         subdirectories. This method creates symlinks or copies to maintain the
         flat structure that Claude Code expects in ~/.claude/skills/.
 
+        Additionally tracks deployed skills in .mpm-deployed-skills.json index
+        for orphan cleanup functionality.
+
         Args:
             skill: Skill metadata dict
             collection_dir: Collection directory containing skills
+            collection_name: Name of collection (for tracking)
             force: Overwrite if already exists
 
         Returns:
             Dict with deployed, skipped, error flags
         """
         skill_name = skill["name"]
-        target_dir = self.CLAUDE_SKILLS_DIR / skill_name
+
+        # Use normalized source_path for both target directory and deployment tracking
+        # This ensures consistency with configuration.yaml skill names
+        source_path = skill.get("source_path", "")
+        if source_path:
+            # Normalize: "universal/web/api-design-patterns/SKILL.md" -> "universal-web-api-design-patterns"
+            normalized_name = source_path.replace("/SKILL.md", "").replace("/", "-")
+            target_dir = self.CLAUDE_SKILLS_DIR / normalized_name
+        else:
+            # Fallback to skill name if no source_path
+            target_dir = self.CLAUDE_SKILLS_DIR / skill_name
 
         # Check if already deployed
         if target_dir.exists() and not force:
             self.logger.debug(f"Skipped {skill_name} (already deployed)")
             return {"deployed": False, "skipped": True, "error": None}
 
-        # Find skill source in collection directory
-        # Updated structure: collection_dir / skills / category / skill-name
-        # OR: collection_dir / universal / skill-name
-        # OR: collection_dir / toolchains / toolchain-name / skill-name
+        # Find skill source using source_path from manifest
+        source_dir = None
+
+        if source_path:
+            # Direct lookup using source_path (most reliable)
+            # Example: "universal/web/api-design-patterns/SKILL.md" -> "universal/web/api-design-patterns"
+            skill_dir_path = source_path.replace("/SKILL.md", "")
+            potential_source = collection_dir / skill_dir_path
+            if potential_source.exists():
+                source_dir = potential_source
+            else:
+                self.logger.debug(
+                    f"Source path {skill_dir_path} not found, trying fallback search"
+                )
+
+        # Fallback: search using old logic (for backward compatibility)
+        if not source_dir:
+            skills_base = collection_dir / "skills"
+            category = skill.get("category", "")
 
-        skills_base = collection_dir / "skills"
-        category = skill.get("category", "")
+            # Try multiple possible locations
+            search_paths = []
 
-        # Try multiple possible locations
-        source_dir = None
-        search_paths = []
-
-        # Try category-based path
-        if category and skills_base.exists():
-            search_paths.append(skills_base / category / skill_name)
-
-        # Try universal/toolchains structure
-        if (collection_dir / "universal").exists():
-            search_paths.append(collection_dir / "universal" / skill_name)
-
-        if (collection_dir / "toolchains").exists():
-            toolchain_dir = collection_dir / "toolchains"
-            for tc in toolchain_dir.iterdir():
-                if tc.is_dir():
-                    search_paths.append(tc / skill_name)
-
-        # Search in all possible locations
-        for path in search_paths:
-            if path.exists():
-                source_dir = path
-                break
-
-        # Fallback: search recursively for skill in skills directory
-        if not source_dir and skills_base.exists():
-            for cat_dir in skills_base.iterdir():
-                if not cat_dir.is_dir():
-                    continue
-                potential = cat_dir / skill_name
-                if potential.exists():
-                    source_dir = potential
+            # Try category-based path
+            if category and skills_base.exists():
+                search_paths.append(skills_base / category / skill_name)
+
+            # Try universal/toolchains structure
+            if (collection_dir / "universal").exists():
+                search_paths.append(collection_dir / "universal" / skill_name)
+
+            if (collection_dir / "toolchains").exists():
+                toolchain_dir = collection_dir / "toolchains"
+                for tc in toolchain_dir.iterdir():
+                    if tc.is_dir():
+                        search_paths.append(tc / skill_name)
+
+            # Search in all possible locations
+            for path in search_paths:
+                if path.exists():
+                    source_dir = path
                     break
 
+            # Final fallback: search recursively for skill in skills directory
+            if not source_dir and skills_base.exists():
+                for cat_dir in skills_base.iterdir():
+                    if not cat_dir.is_dir():
+                        continue
+                    potential = cat_dir / skill_name
+                    if potential.exists():
+                        source_dir = potential
+                        break
+
         if not source_dir or not source_dir.exists():
             return {
                 "deployed": False,
@@ -777,6 +1007,15 @@ class SkillsDeployerService(LoggerMixin):
             # NOTE: We use copy instead of symlink to maintain Claude Code compatibility
             shutil.copytree(source_dir, target_dir)
 
+            # Track deployment in index using normalized name
+            from claude_mpm.services.skills.selective_skill_deployer import (
+                track_deployed_skill,
+            )
+
+            # Use normalized name for tracking (matches configuration.yaml format)
+            track_name = normalized_name if source_path else skill_name
+            track_deployed_skill(self.CLAUDE_SKILLS_DIR, track_name, collection_name)
+
             self.logger.debug(
                 f"Deployed {skill_name} from {source_dir} to {target_dir}"
             )
@@ -809,12 +1048,12 @@ class SkillsDeployerService(LoggerMixin):
         """
         try:
             if platform.system() == "Windows":
-                result = subprocess.run(
+                result = subprocess.run(  # nosec B603 B607 - Safe: hardcoded tasklist command
                     ["tasklist"], check=False, capture_output=True, text=True, timeout=5
                 )
                 return "claude" in result.stdout.lower()
             # macOS and Linux
-            result = subprocess.run(
+            result = subprocess.run(  # nosec B603 B607 - Safe: hardcoded ps command
                 ["ps", "aux"], check=False, capture_output=True, text=True, timeout=5
             )
             # Look for "Claude Code" or "claude-code" process

claude_mpm/services/socketio/dashboard_server.py

@@ -152,6 +152,7 @@ class DashboardServer(SocketIOServiceInterface):
 
         # Register handlers for all events we want to relay from monitor to dashboard
         relay_events = [
+            "claude_event",  # Tool events from Claude Code hooks
             "session_started",
             "session_ended",
             "claude_status",

claude_mpm/services/socketio/event_normalizer.py

@@ -6,7 +6,7 @@ This normalizer ensures all events follow a consistent schema before broadcastin
 providing backward compatibility while establishing a standard format.
 
 DESIGN DECISION: Transform all events to a consistent schema:
-- event: Socket.IO event name (always "claude_event")
+- event: Socket.IO event name (always "mpm_event")
 - type: Main category (hook, system, session, file, connection)
 - subtype: Specific event type (pre_tool, heartbeat, started, etc.)
 - timestamp: ISO format timestamp
@@ -72,16 +72,21 @@ class NormalizedEvent:
     structure explicit and self-documenting.
     """
 
-    event: str = "claude_event"  # Socket.IO event name
+    event: str = "mpm_event"  # Socket.IO event name
     source: str = ""  # WHERE the event comes from
     type: str = ""  # WHAT category of event
     subtype: str = ""  # Specific event type
     timestamp: str = ""  # ISO format timestamp
     data: Dict[str, Any] = field(default_factory=dict)  # Event payload
+    correlation_id: Optional[str] = (
+        None  # For correlating related events (e.g., pre_tool/post_tool)
+    )
+    session_id: Optional[str] = None  # Session identifier for stream grouping
+    cwd: Optional[str] = None  # Working directory for project identification
 
     def to_dict(self) -> Dict[str, Any]:
         """Convert to dictionary for emission."""
-        return {
+        result = {
             "event": self.event,
             "source": self.source,
             "type": self.type,
@@ -89,6 +94,16 @@ class NormalizedEvent:
             "timestamp": self.timestamp,
             "data": self.data,
         }
+        # Include correlation_id if present
+        if self.correlation_id:
+            result["correlation_id"] = self.correlation_id
+        # Include session_id if present
+        if self.session_id:
+            result["session_id"] = self.session_id
+        # Include cwd if present
+        if self.cwd:
+            result["cwd"] = self.cwd
+        return result
 
 
 class EventNormalizer:
@@ -106,6 +121,7 @@ class EventNormalizer:
         "pre_response": (EventType.HOOK, "pre_response"),
         "post_response": (EventType.HOOK, "post_response"),
         "hook_event": (EventType.HOOK, "generic"),
+        "hook_execution": (EventType.HOOK, "execution"),  # Hook execution metadata
         "UserPrompt": (EventType.HOOK, "user_prompt"),  # Legacy format
         # Test events (legacy format)
         "TestStart": (EventType.TEST, "start"),
@@ -218,14 +234,32 @@ class EventNormalizer:
             # Get or generate timestamp
             timestamp = self._extract_timestamp(event_data)
 
+            # Extract correlation_id, session_id, and cwd if present
+            correlation_id = None
+            session_id = None
+            cwd = None
+            if isinstance(event_data, dict):
+                correlation_id = event_data.get("correlation_id")
+                # Try both naming conventions for session_id
+                session_id = event_data.get("session_id") or event_data.get("sessionId")
+                # Try multiple field names for working directory
+                cwd = (
+                    event_data.get("cwd")
+                    or event_data.get("working_directory")
+                    or event_data.get("workingDirectory")
+                )
+
             # Create normalized event
             normalized = NormalizedEvent(
-                event="claude_event",
+                event="mpm_event",
                 source=event_source,
                 type=event_type,
                 subtype=subtype,
                 timestamp=timestamp,
                 data=data,
+                correlation_id=correlation_id,
+                session_id=session_id,
+                cwd=cwd,
             )
 
             self.stats["normalized"] += 1
@@ -239,7 +273,7 @@ class EventNormalizer:
 
             # Return a generic event on error
             return NormalizedEvent(
-                event="claude_event",
+                event="mpm_event",
                 source="system",
                 type="unknown",
                 subtype="error",
@@ -272,8 +306,16 @@ class EventNormalizer:
             # If source is not a valid EventSource value, keep it as-is
             pass
 
+        # Extract session_id and cwd, trying multiple naming conventions
+        session_id = event_data.get("session_id") or event_data.get("sessionId")
+        cwd = (
+            event_data.get("cwd")
+            or event_data.get("working_directory")
+            or event_data.get("workingDirectory")
+        )
+
         return NormalizedEvent(
-            event="claude_event",  # Always use standard event name
+            event="mpm_event",  # Always use standard event name
             source=source,
             type=event_data.get("type", "unknown"),
             subtype=event_data.get("subtype", "generic"),
@@ -281,6 +323,9 @@ class EventNormalizer:
                 "timestamp", datetime.now(timezone.utc).isoformat()
             ),
             data=event_data.get("data", {}),
+            correlation_id=event_data.get("correlation_id"),
+            session_id=session_id,
+            cwd=cwd,
         )
 
     def _extract_event_info(self, event_data: Any) -> Tuple[str, str, Dict[str, Any]]: