claude-mpm 5.4.22__py3-none-any.whl → 5.6.34__py3-none-any.whl

claude_mpm/services/skills/git_skill_source_manager.py

@@ -16,6 +16,7 @@ Trade-offs:
 - Flexibility: Easy to extend with skills-specific features
 """
 
+import os
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from datetime import datetime, timezone
 from pathlib import Path
@@ -32,6 +33,46 @@ from claude_mpm.services.skills.skill_discovery_service import SkillDiscoverySer
 logger = get_logger(__name__)
 
 
+def _get_github_token(source: Optional[SkillSource] = None) -> Optional[str]:
+    """Get GitHub token with source-specific override support.
+
+    Priority: source.token > GITHUB_TOKEN > GH_TOKEN
+
+    Args:
+        source: Optional SkillSource to check for per-source token
+
+    Returns:
+        GitHub token if found, None otherwise
+
+    Token Resolution:
+        1. If source has token starting with "$", resolve as env var
+        2. If source has direct token, use it (not recommended for security)
+        3. Fall back to GITHUB_TOKEN env var
+        4. Fall back to GH_TOKEN env var
+        5. Return None if no token found
+
+    Security Note:
+        Token is never logged or printed to avoid exposure.
+        Direct tokens in config are discouraged - use env var refs ($VAR_NAME).
+
+    Example:
+        >>> source = SkillSource(..., token="$PRIVATE_TOKEN")
+        >>> token = _get_github_token(source)  # Resolves $PRIVATE_TOKEN from env
+        >>> token = _get_github_token()  # Falls back to GITHUB_TOKEN
+    """
+    # Priority 1: Per-source token (env var reference or direct)
+    if source and source.token:
+        if source.token.startswith("$"):
+            # Env var reference: $VAR_NAME -> os.environ.get("VAR_NAME")
+            env_var_name = source.token[1:]
+            return os.environ.get(env_var_name)
+        # Direct token (not recommended but supported)
+        return source.token
+
+    # Priority 2-3: Global environment variables
+    return os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN")
+
+
 class GitSkillSourceManager:
     """Manages multiple Git-based skill sources with priority resolution.
 
@@ -217,9 +258,21 @@ class GitSkillSourceManager:
             )
 
             # Discover skills in cache
+            self.logger.debug(f"Scanning cache path for skills: {cache_path}")
             discovery_service = SkillDiscoveryService(cache_path)
             discovered_skills = discovery_service.discover_skills()
 
+            # Log discovery results
+            if len(discovered_skills) == 0:
+                self.logger.info(
+                    f"No SKILL.md files found in {cache_path}. "
+                    "Ensure your skill source has SKILL.md files with valid frontmatter."
+                )
+            else:
+                self.logger.debug(
+                    f"Successfully parsed {len(discovered_skills)} skills from {cache_path}"
+                )
+
             # Build result
             result = {
                 "synced": True,
@@ -469,7 +522,7 @@ class GitSkillSourceManager:
         # Step 1: Discover all files via GitHub Tree API (single request)
         # This discovers the COMPLETE repository structure (272 files for skills)
         all_files = self._discover_repository_files_via_tree_api(
-            owner_repo, source.branch
+            owner_repo, source.branch, source
         )
 
         if not all_files:
@@ -504,7 +557,7 @@ class GitSkillSourceManager:
                 raw_url = f"https://raw.githubusercontent.com/{owner_repo}/{source.branch}/{file_path}"
                 cache_file = cache_path / file_path
                 future = executor.submit(
-                    self._download_file_with_etag, raw_url, cache_file, force
+                    self._download_file_with_etag, raw_url, cache_file, force, source
                 )
                 future_to_file[future] = file_path
 
@@ -533,7 +586,7 @@ class GitSkillSourceManager:
         return files_updated, files_cached
 
     def _discover_repository_files_via_tree_api(
-        self, owner_repo: str, branch: str
+        self, owner_repo: str, branch: str, source: Optional[SkillSource] = None
     ) -> List[str]:
         """Discover all files in repository using GitHub Git Tree API.
 
@@ -596,9 +649,17 @@ class GitSkillSourceManager:
             )
             self.logger.debug(f"Fetching commit SHA from {refs_url}")
 
-            refs_response = requests.get(
-                refs_url, headers={"Accept": "application/vnd.github+json"}, timeout=30
-            )
+            # Build headers with authentication if token available
+            headers = {"Accept": "application/vnd.github+json"}
+            token = _get_github_token(source)
+            if token:
+                headers["Authorization"] = f"token {token}"
+                if source and source.token:
+                    self.logger.debug(f"Using source-specific token for {source.id}")
+                else:
+                    self.logger.debug("Using GitHub token for authentication")
+
+            refs_response = requests.get(refs_url, headers=headers, timeout=30)
 
             # Check for rate limiting
             if refs_response.status_code == 403:
@@ -621,7 +682,7 @@ class GitSkillSourceManager:
             self.logger.debug(f"Fetching recursive tree from {tree_url}")
             tree_response = requests.get(
                 tree_url,
-                headers={"Accept": "application/vnd.github+json"},
+                headers=headers,  # Reuse headers with auth from Step 1
                 params=params,
                 timeout=30,
             )
@@ -652,7 +713,11 @@ class GitSkillSourceManager:
         return all_files
 
     def _download_file_with_etag(
-        self, url: str, local_path: Path, force: bool = False
+        self,
+        url: str,
+        local_path: Path,
+        force: bool = False,
+        source: Optional[SkillSource] = None,
     ) -> bool:
         """Download file from URL with ETag caching (thread-safe).
 
@@ -660,6 +725,7 @@ class GitSkillSourceManager:
             url: Raw GitHub URL
             local_path: Local file path to save to
             force: Force download even if cached
+            source: Optional SkillSource for token resolution
 
         Returns:
             True if file was updated, False if cached
@@ -682,7 +748,7 @@ class GitSkillSourceManager:
                 try:
                     with open(etag_cache_file, encoding="utf-8") as f:
                         etag_cache = json.load(f)
-                except Exception:
+                except Exception:  # nosec B110 - intentional: proceed without cache on read failure
                     pass
 
             cached_etag = etag_cache.get(str(local_path))
@@ -692,6 +758,11 @@ class GitSkillSourceManager:
         if cached_etag and not force:
             headers["If-None-Match"] = cached_etag
 
+        # Add GitHub authentication if token available
+        token = _get_github_token(source)
+        if token:
+            headers["Authorization"] = f"token {token}"
+
         try:
             response = requests.get(url, headers=headers, timeout=30)
 
@@ -991,12 +1062,17 @@ class GitSkillSourceManager:
         progress_callback=None,
         skill_filter: Optional[Set[str]] = None,
     ) -> Dict[str, Any]:
-        """Deploy skills from cache to target directory with flat structure.
+        """Deploy skills from cache to target directory with flat structure and automatic cleanup.
 
         Flattens nested Git repository structure into Claude Code compatible
         flat directory structure. Each skill directory is copied with a
         hyphen-separated name derived from its path.
 
+        CRITICAL: When skill_filter is provided (agent-referenced skills), this function:
+        1. Deploys ONLY the filtered skills
+        2. REMOVES orphaned skills (deployed but not in filter)
+        3. Returns removed_count and removed_skills in result
+
         Transformation Example:
             Cache: collaboration/dispatching-parallel-agents/SKILL.md
             Deploy: collaboration-dispatching-parallel-agents/SKILL.md
@@ -1006,8 +1082,8 @@ class GitSkillSourceManager:
             force: Overwrite existing skills
             progress_callback: Optional callback(increment: int) called for each skill deployed
             skill_filter: Optional set of skill names to deploy (selective deployment).
-                         If None, deploys all skills. If provided, only deploys skills
-                         whose name matches an entry in the filter set.
+                         If None, deploys ALL skills WITHOUT cleanup.
+                         If provided, deploys ONLY filtered skills AND removes orphans.
 
         Returns:
             Dict with deployment results:
@@ -1018,7 +1094,9 @@ class GitSkillSourceManager:
                 "deployed_skills": List[str],
                 "skipped_skills": List[str],
                 "errors": List[str],
-                "filtered_count": int  # Number of skills filtered out
+                "filtered_count": int,  # Number of skills filtered out
+                "removed_count": int,   # Number of orphaned skills removed
+                "removed_skills": List[str]  # Names of removed orphaned skills
             }
 
         Example:
@@ -1026,10 +1104,10 @@ class GitSkillSourceManager:
             >>> result = manager.deploy_skills()
             >>> print(f"Deployed {result['deployed_count']} skills")
 
-            # Selective deployment based on agent requirements:
+            # Selective deployment based on agent requirements (with cleanup):
             >>> required = {"typescript-core", "react-patterns"}
             >>> result = manager.deploy_skills(skill_filter=required)
-            >>> print(f"Deployed {result['deployed_count']} of {len(required)} required skills")
+            >>> print(f"Deployed {result['deployed_count']}, removed {result['removed_count']} orphans")
         """
         if target_dir is None:
             target_dir = Path.home() / ".claude" / "skills"
@@ -1040,6 +1118,7 @@ class GitSkillSourceManager:
         skipped = []
         errors = []
         filtered_count = 0
+        removed_skills = []  # Track removed orphaned skills
 
         # Get all skills from all sources
         all_skills = self.get_all_skills()
@@ -1049,12 +1128,31 @@ class GitSkillSourceManager:
             original_count = len(all_skills)
             # Normalize filter to lowercase for case-insensitive matching
             normalized_filter = {s.lower() for s in skill_filter}
-            # Match against deployment_name (not display name) since skill_filter contains
-            # deployment-style names like "toolchains-python-frameworks-django"
+
+            def matches_filter(deployment_name: str) -> bool:
+                """Match using same fuzzy logic as ProfileManager.is_skill_enabled()"""
+                deployment_lower = deployment_name.lower()
+
+                # Exact match
+                if deployment_lower in normalized_filter:
+                    return True
+
+                # Fuzzy match: check if deployment name ends with or contains short name
+                # Example: "toolchains-python-frameworks-flask" matches "flask"
+                for short_name in normalized_filter:
+                    if deployment_lower.endswith(f"-{short_name}"):
+                        return True
+                    # Check if short name is contained as a segment
+                    if f"-{short_name}-" in deployment_lower:
+                        return True
+                    if deployment_lower.startswith(f"{short_name}-"):
+                        return True
+
+                return False
+
+            # Match against deployment_name using fuzzy matching
             all_skills = [
-                s
-                for s in all_skills
-                if s.get("deployment_name", "").lower() in normalized_filter
+                s for s in all_skills if matches_filter(s.get("deployment_name", ""))
             ]
             filtered_count = original_count - len(all_skills)
             self.logger.info(
@@ -1062,6 +1160,19 @@ class GitSkillSourceManager:
                 f"match agent requirements ({filtered_count} filtered out)"
             )
 
+            # Cleanup: Remove skills from target directory that aren't in the filtered set
+            # This ensures only agent-referenced skills remain deployed
+            removed_skills = self._cleanup_unfiltered_skills(target_dir, all_skills)
+            if removed_skills:
+                self.logger.info(
+                    f"Removed {len(removed_skills)} orphaned skills not referenced by agents: {removed_skills[:10]}"
+                    + (
+                        f" (and {len(removed_skills) - 10} more)"
+                        if len(removed_skills) > 10
+                        else ""
+                    )
+                )
+
         self.logger.info(
             f"Deploying {len(all_skills)} skills to {target_dir} (force={force})"
         )
@@ -1103,6 +1214,7 @@ class GitSkillSourceManager:
         self.logger.info(
             f"Deployment complete: {len(deployed)} deployed, "
             f"{len(skipped)} skipped, {len(errors)} errors"
+            + (f", {len(removed_skills)} removed" if removed_skills else "")
         )
 
         return {
@@ -1113,8 +1225,157 @@ class GitSkillSourceManager:
             "skipped_skills": skipped,
             "errors": errors,
             "filtered_count": filtered_count,
+            "removed_count": len(removed_skills),
+            "removed_skills": removed_skills,
+        }
+
+    def _cleanup_unfiltered_skills(
+        self, target_dir: Path, filtered_skills: List[Dict[str, Any]]
+    ) -> List[str]:
+        """Remove skills from target directory that aren't in the filtered skill list.
+
+        CRITICAL: Only removes MPM-managed skills (those in our cache). Custom user skills
+        are preserved. This prevents accidental deletion of user-created skills that were
+        never part of MPM's skill repository.
+
+        Uses fuzzy matching to handle both exact deployment names and short skill names.
+        For example:
+        - "toolchains-python-frameworks-flask" (deployed dir) matches "flask" (filter)
+        - "toolchains-elixir-frameworks-phoenix-liveview" matches "phoenix-liveview"
+
+        Args:
+            target_dir: Target deployment directory
+            filtered_skills: List of skills that should remain deployed
+
+        Returns:
+            List of skill names that were removed
+        """
+        import shutil
+
+        removed_skills = []
+
+        # Build set of deployment names (exact matches)
+        expected_deployments = {
+            skill.get("deployment_name").lower()
+            for skill in filtered_skills
+            if skill.get("deployment_name")
         }
 
+        # Build helper function for fuzzy matching (matches logic from deploy_skills)
+        def should_keep_skill(deployed_dir_name: str) -> bool:
+            """Check if deployed skill matches any expected deployment using fuzzy matching.
+
+            Matches the same logic as matches_filter() in deploy_skills() at line 1053.
+            """
+            deployed_lower = deployed_dir_name.lower()
+
+            # Exact match
+            if deployed_lower in expected_deployments:
+                return True
+
+            # Fuzzy match: check if deployment name matches any short name pattern
+            # Example: "toolchains-elixir-frameworks-phoenix-liveview" matches "phoenix-liveview"
+            for expected_name in expected_deployments:
+                # Suffix match: deployment ends with "-shortname"
+                if deployed_lower.endswith(f"-{expected_name}"):
+                    return True
+                # Segment match: "-shortname-" appears in deployment
+                if f"-{expected_name}-" in deployed_lower:
+                    return True
+                # Prefix match: deployment starts with "shortname-"
+                if deployed_lower.startswith(f"{expected_name}-"):
+                    return True
+
+            return False
+
+        def is_mpm_managed_skill(skill_dir_name: str) -> bool:
+            """Check if skill is managed by MPM (exists in our cache).
+
+            Custom user skills (not in cache) are NEVER deleted, even if not in filter.
+            Only MPM-managed skills (in cache but not in filter) are candidates for removal.
+
+            Args:
+                skill_dir_name: Name of deployed skill directory
+
+            Returns:
+                True if skill exists in MPM cache (MPM-managed), False if custom user skill
+            """
+            # Check all configured skill sources for this skill
+            for source in self.config.get_enabled_sources():
+                cache_path = self._get_source_cache_path(source)
+                if not cache_path.exists():
+                    continue
+
+                # Check if this skill directory exists anywhere in the cache
+                # Use glob to find matching directories recursively
+                matches = list(cache_path.rglob(f"*{skill_dir_name}*"))
+                if matches:
+                    # Found in cache - this is MPM-managed
+                    self.logger.debug(
+                        f"Skill '{skill_dir_name}' found in cache at {matches[0]} - MPM-managed"
+                    )
+                    return True
+
+            # Not found in any cache - this is a custom user skill
+            self.logger.debug(
+                f"Skill '{skill_dir_name}' not found in cache - custom user skill, preserving"
+            )
+            return False
+
+        # Check each directory in target_dir
+        if not target_dir.exists():
+            return removed_skills
+
+        try:
+            for item in target_dir.iterdir():
+                # Skip files, only process directories
+                if not item.is_dir():
+                    continue
+
+                # Skip hidden directories
+                if item.name.startswith("."):
+                    continue
+
+                # Check if this skill directory should be kept (fuzzy matching)
+                if not should_keep_skill(item.name):
+                    # CRITICAL: Check if this is an MPM-managed skill before deletion
+                    if not is_mpm_managed_skill(item.name):
+                        # This is a custom user skill - NEVER delete
+                        self.logger.debug(
+                            f"Preserving custom user skill (not in MPM cache): {item.name}"
+                        )
+                        continue
+
+                    # It's MPM-managed but not in filter - safe to remove
+                    try:
+                        # Security: Validate path is within target_dir
+                        if not self._validate_safe_path(target_dir, item):
+                            self.logger.error(
+                                f"Refusing to remove path outside target directory: {item}"
+                            )
+                            continue
+
+                        # Remove the skill directory
+                        if item.is_symlink():
+                            item.unlink()
+                        else:
+                            shutil.rmtree(item)
+
+                        removed_skills.append(item.name)
+                        self.logger.info(
+                            f"Removed orphaned MPM-managed skill: {item.name}"
+                        )
+
+                    except Exception as e:
+                        self.logger.warning(
+                            f"Failed to remove skill directory {item.name}: {e}"
+                        )
+
+        except Exception as e:
+            self.logger.error(f"Error during skill cleanup: {e}")
+
+        return removed_skills
+
     def _deploy_single_skill(
         self, skill: Dict[str, Any], target_dir: Path, deployment_name: str, force: bool
     ) -> Dict[str, Any]: