claude-mpm 5.4.36__py3-none-any.whl → 5.4.59__py3-none-any.whl

claude_mpm/services/skills/selective_skill_deployer.py

@@ -51,6 +51,59 @@ logger = get_logger(__name__)
 # Deployment tracking index file
 DEPLOYED_INDEX_FILE = ".mpm-deployed-skills.json"
 
+# Core skills that are universally useful across all projects
+# These are deployed when skill mapping returns too many skills (>60)
+# Target: ~25-30 core skills for balanced functionality
+CORE_SKILLS = {
+    # Universal debugging and verification (4 skills)
+    "universal-debugging-systematic-debugging",
+    "universal-debugging-verification-before-completion",
+    "universal-verification-pre-merge",
+    "universal-verification-screenshot",
+
+    # Universal testing patterns (2 skills)
+    "universal-testing-test-driven-development",
+    "universal-testing-testing-anti-patterns",
+
+    # Universal architecture and design (1 skill)
+    "universal-architecture-software-patterns",
+
+    # Universal infrastructure (3 skills)
+    "universal-infrastructure-env-manager",
+    "universal-infrastructure-docker",
+    "universal-infrastructure-github-actions",
+
+    # Universal collaboration (1 skill)
+    "universal-collaboration-stacked-prs",
+
+    # Universal emergency/operations (1 skill)
+    "toolchains-universal-emergency-release",
+    "toolchains-universal-dependency-audit",
+
+    # Common language toolchains (6 skills)
+    "toolchains-typescript-core",
+    "toolchains-python-core",
+    "toolchains-javascript-tooling-biome",
+    "toolchains-python-tooling-mypy",
+    "toolchains-typescript-testing-vitest",
+    "toolchains-python-frameworks-flask",
+
+    # Common web frameworks (4 skills)
+    "toolchains-javascript-frameworks-nextjs",
+    "toolchains-nextjs-core",
+    "toolchains-typescript-frameworks-nodejs-backend",
+    "toolchains-javascript-frameworks-react-state-machine",
+
+    # Common testing tools (2 skills)
+    "toolchains-javascript-testing-playwright",
+    "toolchains-typescript-testing-jest",
+
+    # Common data/UI tools (3 skills)
+    "universal-data-xlsx",
+    "toolchains-ui-styling-tailwind",
+    "toolchains-ui-components-headlessui",
+}
+
 
 def parse_agent_frontmatter(agent_file: Path) -> Dict[str, Any]:
     """Parse YAML frontmatter from agent markdown file.
@@ -140,50 +193,49 @@ def get_skills_from_agent(frontmatter: Dict[str, Any]) -> Set[str]:
 def get_skills_from_mapping(agent_ids: List[str]) -> Set[str]:
     """Get skills for agents using SkillToAgentMapper inference.
 
-    Uses SkillToAgentMapper to find all skills associated with given agent IDs.
-    This provides pattern-based skill discovery beyond explicit frontmatter declarations.
+    DEPRECATED: This function is deprecated as of Phase 3 refactor.
+    Skills are now declared exclusively in agent frontmatter.
+
+    The static skill_to_agent_mapping.yaml is no longer used for skill deployment.
+    Each agent must declare its skills in frontmatter or it gets zero skills.
+
+    This function remains for backward compatibility but is NO LONGER CALLED
+    by get_required_skills_from_agents().
 
     Args:
-        agent_ids: List of agent identifiers (e.g., ["python-engineer", "typescript-engineer"])
+        agent_ids: List of DEPLOYED agent identifiers (e.g., ["python-engineer", "typescript-engineer"])
+                  These should be extracted from ~/.claude/agents/*.md files only.
 
     Returns:
-        Set of unique skill names inferred from mapping configuration
+        Set of unique skill names inferred from mapping configuration for DEPLOYED agents only
+        NOTE: This is now an empty set as the function is deprecated.
 
     Example:
-        >>> agent_ids = ["python-engineer", "typescript-engineer"]
-        >>> skills = get_skills_from_mapping(agent_ids)
-        >>> print(f"Found {len(skills)} skills from mapping")
+        >>> # DEPRECATED - use frontmatter instead
+        >>> deployed_agent_ids = ["python-engineer", "typescript-engineer", "qa"]
+        >>> skills = get_skills_from_mapping(deployed_agent_ids)  # Returns empty set
     """
-    try:
-        mapper = SkillToAgentMapper()
-        all_skills = set()
-
-        for agent_id in agent_ids:
-            agent_skills = mapper.get_skills_for_agent(agent_id)
-            if agent_skills:
-                all_skills.update(agent_skills)
-                logger.debug(f"Mapped {len(agent_skills)} skills to {agent_id}")
-
-        logger.info(
-            f"Mapped {len(all_skills)} unique skills for {len(agent_ids)} agents"
-        )
-        return all_skills
-
-    except Exception as e:
-        logger.warning(f"Failed to load SkillToAgentMapper: {e}")
-        logger.info("Falling back to frontmatter-only skill discovery")
-        return set()
+    # DEPRECATED: Return empty set
+    logger.warning(
+        "get_skills_from_mapping() is DEPRECATED and returns empty set. "
+        "Skills are now declared in agent frontmatter only. "
+        "Update your agents with 'skills:' field in frontmatter."
+    )
+    return set()
 
 
 def get_required_skills_from_agents(agents_dir: Path) -> Set[str]:
     """Extract all skills referenced by deployed agents.
 
-    Combines skills from two sources:
-    1. Explicit frontmatter declarations (skills: field in agent .md files)
-    2. SkillToAgentMapper inference (pattern-based skill discovery)
+    MAJOR CHANGE (Phase 3): Now ONLY uses frontmatter-declared skills.
+    The static skill_to_agent_mapping.yaml is DEPRECATED. Each agent must
+    declare its skills in frontmatter or it gets zero skills deployed.
 
-    This dual-source approach ensures agents get both explicitly declared skills
-    and skills inferred from their domain/toolchain patterns.
+    This change:
+    - Eliminates dual-source complexity (frontmatter + mapping)
+    - Makes skill requirements explicit per agent
+    - Enables per-agent customization via frontmatter
+    - Removes dependency on static YAML mapping
 
     Args:
         agents_dir: Path to deployed agents directory (e.g., .claude/agents/)
@@ -204,13 +256,11 @@ def get_required_skills_from_agents(agents_dir: Path) -> Set[str]:
     agent_files = list(agents_dir.glob("*.md"))
     logger.debug(f"Scanning {len(agent_files)} agent files in {agents_dir}")
 
-    # Source 1: Extract skills from frontmatter
+    # ONLY use frontmatter skills - no more mapping inference
     frontmatter_skills = set()
-    agent_ids = []
 
     for agent_file in agent_files:
         agent_id = agent_file.stem
-        agent_ids.append(agent_id)
 
         frontmatter = parse_agent_frontmatter(agent_file)
         agent_skills = get_skills_from_agent(frontmatter)
@@ -220,24 +270,23 @@ def get_required_skills_from_agents(agents_dir: Path) -> Set[str]:
             logger.debug(
                 f"Agent {agent_id}: {len(agent_skills)} skills from frontmatter"
             )
+        else:
+            logger.debug(f"Agent {agent_id}: No skills declared in frontmatter")
 
-    logger.info(f"Found {len(frontmatter_skills)} unique skills from frontmatter")
-
-    # Source 2: Get skills from SkillToAgentMapper
-    mapped_skills = get_skills_from_mapping(agent_ids)
-
-    # Combine both sources
-    required_skills = frontmatter_skills | mapped_skills
+    logger.info(
+        f"Found {len(frontmatter_skills)} unique skills from agent frontmatter "
+        f"(static mapping no longer used)"
+    )
 
     # Normalize skill paths: convert slashes to dashes for compatibility with deployment
-    # SkillToAgentMapper returns paths like "toolchains/python/frameworks/django"
-    # but deployment expects "toolchains-python-frameworks-django"
-    normalized_skills = {skill.replace("/", "-") for skill in required_skills}
+    # Some skills may use slash format, normalize to dashes
+    normalized_skills = {skill.replace("/", "-") for skill in frontmatter_skills}
 
-    logger.info(
-        f"Combined {len(frontmatter_skills)} frontmatter + {len(mapped_skills)} mapped "
-        f"= {len(required_skills)} total unique skills (normalized to {len(normalized_skills)})"
-    )
+    if normalized_skills != frontmatter_skills:
+        logger.debug(
+            f"Normalized {len(frontmatter_skills)} skills to {len(normalized_skills)} "
+            "(converted slashes to dashes)"
+        )
 
     return normalized_skills
 

claude_mpm/services/skills_deployer.py

@@ -174,44 +174,93 @@ class SkillsDeployerService(LoggerMixin):
         if selective:
             # Auto-detect project root if not provided
             if project_root is None:
-                # Try to find project root by looking for .claude directory
+                # Try to find project root by looking for .claude-mpm directory
                 # Start from current directory and walk up
                 current = Path.cwd()
                 while current != current.parent:
-                    if (current / ".claude").exists():
+                    if (current / ".claude-mpm").exists():
                         project_root = current
                         break
                     current = current.parent
 
+            # Read skills from configuration.yaml instead of agent frontmatter
             if project_root:
-                agents_dir = Path(project_root) / ".claude" / "agents"
+                config_path = Path(project_root) / ".claude-mpm" / "configuration.yaml"
             else:
-                # Fallback to current directory's .claude/agents
-                agents_dir = Path.cwd() / ".claude" / "agents"
+                # Fallback to current directory's configuration
+                config_path = Path.cwd() / ".claude-mpm" / "configuration.yaml"
 
             from claude_mpm.services.skills.selective_skill_deployer import (
                 get_required_skills_from_agents,
+                get_skills_to_deploy,
+                save_agent_skills_to_config,
             )
 
-            required_skill_names = get_required_skills_from_agents(agents_dir)
+            # Check if agent_referenced is empty and needs to be populated
+            required_skill_names, source = get_skills_to_deploy(config_path)
+
+            if not required_skill_names and project_root:
+                # agent_referenced is empty, scan deployed agents to populate it
+                agents_dir = Path(project_root) / ".claude" / "agents"
+                if agents_dir.exists():
+                    self.logger.info(
+                        "agent_referenced is empty in configuration.yaml, scanning deployed agents..."
+                    )
+                    agent_skills = get_required_skills_from_agents(agents_dir)
+                    if agent_skills:
+                        save_agent_skills_to_config(list(agent_skills), config_path)
+                        self.logger.info(
+                            f"Populated agent_referenced with {len(agent_skills)} skills from deployed agents"
+                        )
+                        # Re-read configuration after update
+                        required_skill_names, source = get_skills_to_deploy(config_path)
+                    else:
+                        self.logger.warning(
+                            "No skills found in deployed agents - configuration.yaml remains empty"
+                        )
+                else:
+                    self.logger.warning(
+                        f"Agents directory not found at {agents_dir} - cannot scan for skills"
+                    )
 
             if required_skill_names:
+                # Convert required_skill_names to a set for O(1) lookup
+                required_set = set(required_skill_names)
+
                 # Filter to only required skills
-                # Match on either 'name' or 'skill_id' field
+                # Match on: 'name', 'skill_id', or normalized 'source_path'
+                # source_path example: "universal/web/api-design-patterns/SKILL.md"
+                # normalized: "universal-web-api-design-patterns"
+                def skill_matches_requirement(skill):
+                    # Check basic name and skill_id
+                    if skill.get("name") in required_set:
+                        return True
+                    if skill.get("skill_id") in required_set:
+                        return True
+
+                    # Check normalized source_path
+                    source_path = skill.get("source_path", "")
+                    if source_path:
+                        # Remove /SKILL.md suffix and replace / with -
+                        normalized = source_path.replace("/SKILL.md", "").replace(
+                            "/", "-"
+                        )
+                        if normalized in required_set:
+                            return True
+
+                    return False
+
                 filtered_skills = [
-                    s
-                    for s in filtered_skills
-                    if s.get("name") in required_skill_names
-                    or s.get("skill_id") in required_skill_names
+                    s for s in filtered_skills if skill_matches_requirement(s)
                 ]
 
                 self.logger.info(
                     f"Selective deployment: {len(filtered_skills)}/{total_available} skills "
-                    f"(agent-referenced only)"
+                    f"(source: {source})"
                 )
             else:
                 self.logger.warning(
-                    f"No skills found in agent frontmatter at {agents_dir}. "
+                    f"No skills found in configuration at {config_path}. "
                     f"Deploying all {total_available} skills."
                 )
         else:
@@ -224,12 +273,19 @@ class SkillsDeployerService(LoggerMixin):
         skipped = []
         errors = []
 
-        # Extract skill names for cleanup (needed regardless of deployment outcome)
-        filtered_skills_names = [
-            skill["name"]
-            for skill in filtered_skills
-            if isinstance(skill, dict) and "name" in skill
-        ]
+        # Extract normalized skill names for cleanup (needed regardless of deployment outcome)
+        # Must match the names used during deployment (normalized from source_path)
+        filtered_skills_names = []
+        for skill in filtered_skills:
+            if isinstance(skill, dict) and "name" in skill:
+                source_path = skill.get("source_path", "")
+                if source_path:
+                    # Normalize: "universal/web/api-design-patterns/SKILL.md" -> "universal-web-api-design-patterns"
+                    normalized = source_path.replace("/SKILL.md", "").replace("/", "-")
+                    filtered_skills_names.append(normalized)
+                else:
+                    # Fallback to skill name
+                    filtered_skills_names.append(skill["name"])
 
         for skill in filtered_skills:
             try:
@@ -243,9 +299,25 @@ class SkillsDeployerService(LoggerMixin):
                     skill, skills_data["temp_dir"], collection_name, force=force
                 )
                 if result["deployed"]:
-                    deployed.append(skill["name"])
+                    # Use normalized name for reporting
+                    source_path = skill.get("source_path", "")
+                    if source_path:
+                        normalized = source_path.replace("/SKILL.md", "").replace(
+                            "/", "-"
+                        )
+                        deployed.append(normalized)
+                    else:
+                        deployed.append(skill["name"])
                 elif result["skipped"]:
-                    skipped.append(skill["name"])
+                    # Use normalized name for reporting
+                    source_path = skill.get("source_path", "")
+                    if source_path:
+                        normalized = source_path.replace("/SKILL.md", "").replace(
+                            "/", "-"
+                        )
+                        skipped.append(normalized)
+                    else:
+                        skipped.append(skill["name"])
                 if result["error"]:
                     errors.append(result["error"])
             except Exception as e:
@@ -257,9 +329,9 @@ class SkillsDeployerService(LoggerMixin):
                 self.logger.error(f"Failed to deploy {skill_name}: {e}")
                 errors.append(f"{skill_name}: {e}")
 
-        # Step 5: Cleanup orphaned skills (if selective mode enabled)
+        # Step 5: Cleanup orphaned skills (always run in selective mode)
         cleanup_result = {"removed_count": 0, "removed_skills": []}
-        if selective and len(deployed) > 0:
+        if selective:
             # Get the set of skills that should remain deployed
             # This is the union of what we just deployed and what was already there
             try:
@@ -267,7 +339,8 @@ class SkillsDeployerService(LoggerMixin):
                     cleanup_orphan_skills,
                 )
 
-                # Only cleanup if we're in selective mode
+                # Cleanup orphaned skills not referenced by agents
+                # This runs even if nothing new was deployed to remove stale skills
                 cleanup_result = cleanup_orphan_skills(
                     self.CLAUDE_SKILLS_DIR, set(filtered_skills_names)
                 )
@@ -804,55 +877,76 @@ class SkillsDeployerService(LoggerMixin):
             Dict with deployed, skipped, error flags
         """
         skill_name = skill["name"]
-        target_dir = self.CLAUDE_SKILLS_DIR / skill_name
+
+        # Use normalized source_path for both target directory and deployment tracking
+        # This ensures consistency with configuration.yaml skill names
+        source_path = skill.get("source_path", "")
+        if source_path:
+            # Normalize: "universal/web/api-design-patterns/SKILL.md" -> "universal-web-api-design-patterns"
+            normalized_name = source_path.replace("/SKILL.md", "").replace("/", "-")
+            target_dir = self.CLAUDE_SKILLS_DIR / normalized_name
+        else:
+            # Fallback to skill name if no source_path
+            target_dir = self.CLAUDE_SKILLS_DIR / skill_name
 
         # Check if already deployed
         if target_dir.exists() and not force:
             self.logger.debug(f"Skipped {skill_name} (already deployed)")
             return {"deployed": False, "skipped": True, "error": None}
 
-        # Find skill source in collection directory
-        # Updated structure: collection_dir / skills / category / skill-name
-        # OR: collection_dir / universal / skill-name
-        # OR: collection_dir / toolchains / toolchain-name / skill-name
+        # Find skill source using source_path from manifest
+        source_dir = None
 
-        skills_base = collection_dir / "skills"
-        category = skill.get("category", "")
+        if source_path:
+            # Direct lookup using source_path (most reliable)
+            # Example: "universal/web/api-design-patterns/SKILL.md" -> "universal/web/api-design-patterns"
+            skill_dir_path = source_path.replace("/SKILL.md", "")
+            potential_source = collection_dir / skill_dir_path
+            if potential_source.exists():
+                source_dir = potential_source
+            else:
+                self.logger.debug(
+                    f"Source path {skill_dir_path} not found, trying fallback search"
+                )
 
-        # Try multiple possible locations
-        source_dir = None
-        search_paths = []
-
-        # Try category-based path
-        if category and skills_base.exists():
-            search_paths.append(skills_base / category / skill_name)
-
-        # Try universal/toolchains structure
-        if (collection_dir / "universal").exists():
-            search_paths.append(collection_dir / "universal" / skill_name)
-
-        if (collection_dir / "toolchains").exists():
-            toolchain_dir = collection_dir / "toolchains"
-            for tc in toolchain_dir.iterdir():
-                if tc.is_dir():
-                    search_paths.append(tc / skill_name)
-
-        # Search in all possible locations
-        for path in search_paths:
-            if path.exists():
-                source_dir = path
-                break
-
-        # Fallback: search recursively for skill in skills directory
-        if not source_dir and skills_base.exists():
-            for cat_dir in skills_base.iterdir():
-                if not cat_dir.is_dir():
-                    continue
-                potential = cat_dir / skill_name
-                if potential.exists():
-                    source_dir = potential
+        # Fallback: search using old logic (for backward compatibility)
+        if not source_dir:
+            skills_base = collection_dir / "skills"
+            category = skill.get("category", "")
+
+            # Try multiple possible locations
+            search_paths = []
+
+            # Try category-based path
+            if category and skills_base.exists():
+                search_paths.append(skills_base / category / skill_name)
+
+            # Try universal/toolchains structure
+            if (collection_dir / "universal").exists():
+                search_paths.append(collection_dir / "universal" / skill_name)
+
+            if (collection_dir / "toolchains").exists():
+                toolchain_dir = collection_dir / "toolchains"
+                for tc in toolchain_dir.iterdir():
+                    if tc.is_dir():
+                        search_paths.append(tc / skill_name)
+
+            # Search in all possible locations
+            for path in search_paths:
+                if path.exists():
+                    source_dir = path
                     break
 
+            # Final fallback: search recursively for skill in skills directory
+            if not source_dir and skills_base.exists():
+                for cat_dir in skills_base.iterdir():
+                    if not cat_dir.is_dir():
+                        continue
+                    potential = cat_dir / skill_name
+                    if potential.exists():
+                        source_dir = potential
+                        break
+
         if not source_dir or not source_dir.exists():
             return {
                 "deployed": False,
@@ -887,12 +981,14 @@ class SkillsDeployerService(LoggerMixin):
             # NOTE: We use copy instead of symlink to maintain Claude Code compatibility
             shutil.copytree(source_dir, target_dir)
 
-            # Track deployment in index
+            # Track deployment in index using normalized name
             from claude_mpm.services.skills.selective_skill_deployer import (
                 track_deployed_skill,
             )
 
-            track_deployed_skill(self.CLAUDE_SKILLS_DIR, skill_name, collection_name)
+            # Use normalized name for tracking (matches configuration.yaml format)
+            track_name = normalized_name if source_path else skill_name
+            track_deployed_skill(self.CLAUDE_SKILLS_DIR, track_name, collection_name)
 
             self.logger.debug(
                 f"Deployed {skill_name} from {source_dir} to {target_dir}"

claude_mpm/skills/bundled/security-scanning.md

@@ -81,6 +81,36 @@ API_KEY = "sk-1234567890abcdef"  # In code!  # pragma: allowlist secret
 
 # ✅ Safe: Use environment variables
 API_KEY = os.getenv("API_KEY")
+
+# ❌ CRITICAL: MCP config files with API keys
+# NEVER commit these files:
+# - .mcp-vector-search/config.json (OpenRouter API keys)
+# - .mcp/config.json (MCP server credentials)
+# - openrouter.json, anthropic-config.json
+# - credentials.json, secrets.json, api-keys.json
+
+# ✅ Safe: Verify .gitignore before committing
+# Check file is ignored: git check-ignore <file_path>
+# Check file not tracked: git ls-files <file_path>
+```
+
+**MCP Secret File Patterns (High Risk):**
+```bash
+# Files that commonly contain API keys:
+.mcp-vector-search/config.json     # OpenRouter, OpenAI keys
+.mcp/config.json                    # MCP server credentials
+**/mcp-config.json
+openrouter.json
+anthropic-config.json
+openai-config.json
+credentials.json
+secrets.json
+api-keys.json
+
+# ALWAYS add to .gitignore:
+echo ".mcp-vector-search/" >> .gitignore
+echo "credentials.json" >> .gitignore
+echo "secrets.json" >> .gitignore
 ```
 
 ### 4. XML External Entities (XXE)
@@ -178,6 +208,88 @@ if failed_login_count > 5:
     alert_security_team()
 ```
 
+## Secret Detection and Prevention
+
+### Pre-commit Hooks with detect-secrets
+```bash
+# Install detect-secrets
+pip install detect-secrets
+
+# Create baseline of existing secrets
+detect-secrets scan > .secrets.baseline
+
+# Install pre-commit hooks
+pip install pre-commit
+pre-commit install
+
+# Add to .pre-commit-config.yaml:
+# - repo: https://github.com/Yelp/detect-secrets
+#   rev: v1.5.0
+#   hooks:
+#     - id: detect-secrets
+#       args: ['--baseline', '.secrets.baseline']
+
+# Scan for new secrets
+detect-secrets scan --baseline .secrets.baseline
+
+# Audit baseline (mark false positives)
+detect-secrets audit .secrets.baseline
+```
+
+### Manual Secret Scanning
+```bash
+# Check if file is ignored by git
+git check-ignore .mcp-vector-search/config.json
+# Exit code 0 = ignored (safe)
+# Exit code 1 = NOT ignored (DANGER!)
+
+# Check if file is tracked by git
+git ls-files .mcp-vector-search/config.json
+# Output present = tracked (CRITICAL - remove immediately!)
+# No output = not tracked (safe if also in .gitignore)
+
+# Search git history for committed secrets
+git log --all --full-history -- .mcp-vector-search/config.json
+
+# Remove file from git history (if accidentally committed)
+git filter-branch --force --index-filter \
+  'git rm --cached --ignore-unmatch .mcp-vector-search/config.json' \
+  --prune-empty --tag-name-filter cat -- --all
+```
+
+### Incident Response: Exposed API Key
+If you've committed an API key to git:
+
+1. **IMMEDIATELY rotate the exposed credential**
+   - OpenRouter: https://openrouter.ai/settings/keys
+   - Anthropic: https://console.anthropic.com/settings/keys
+   - OpenAI: https://platform.openai.com/api-keys
+
+2. **Remove from git history**
+   ```bash
+   # Using git-filter-repo (recommended)
+   git filter-repo --path .mcp-vector-search/config.json --invert-paths
+
+   # Force push to remote (WARNING: destructive)
+   git push origin --force --all
+   git push origin --force --tags
+   ```
+
+3. **Add to .gitignore** (if not already there)
+   ```bash
+   echo ".mcp-vector-search/" >> .gitignore
+   git add .gitignore
+   git commit -m "chore: add MCP config to gitignore"
+   ```
+
+4. **Verify cleanup**
+   ```bash
+   git log --all --full-history -- .mcp-vector-search/config.json
+   # Should show no results
+   ```
+
+5. **Notify stakeholders** if the key had production access
+
 ## Security Scanning Tools
 
 ### Python