skill-seekers 2.7.3__py3-none-any.whl

skill_seekers/mcp/source_manager.py

@@ -0,0 +1,285 @@
+#!/usr/bin/env python3
+"""
+Config Source Manager
+Manages registry of custom config sources (git repositories)
+"""
+
+import json
+from datetime import datetime, timezone
+from pathlib import Path
+
+
+class SourceManager:
+    """Manages config source registry at ~/.skill-seekers/sources.json"""
+
+    def __init__(self, config_dir: str | None = None):
+        """
+        Initialize source manager.
+
+        Args:
+            config_dir: Base config directory. Defaults to ~/.skill-seekers/
+        """
+        if config_dir:
+            self.config_dir = Path(config_dir)
+        else:
+            self.config_dir = Path.home() / ".skill-seekers"
+
+        # Ensure config directory exists
+        self.config_dir.mkdir(parents=True, exist_ok=True)
+
+        # Registry file path
+        self.registry_file = self.config_dir / "sources.json"
+
+        # Initialize registry if it doesn't exist
+        if not self.registry_file.exists():
+            self._write_registry({"version": "1.0", "sources": []})
+
+    def add_source(
+        self,
+        name: str,
+        git_url: str,
+        source_type: str = "github",
+        token_env: str | None = None,
+        branch: str = "main",
+        priority: int = 100,
+        enabled: bool = True,
+    ) -> dict:
+        """
+        Add or update a config source.
+
+        Args:
+            name: Source identifier (lowercase, alphanumeric + hyphens/underscores)
+            git_url: Git repository URL
+            source_type: Source type (github, gitlab, bitbucket, custom)
+            token_env: Environment variable name for auth token
+            branch: Git branch to use (default: main)
+            priority: Source priority (lower = higher priority, default: 100)
+            enabled: Whether source is enabled (default: True)
+
+        Returns:
+            Source dictionary
+
+        Raises:
+            ValueError: If name is invalid or git_url is empty
+        """
+        # Validate name
+        if not name or not name.replace("-", "").replace("_", "").isalnum():
+            raise ValueError(
+                f"Invalid source name '{name}'. Must be alphanumeric with optional hyphens/underscores."
+            )
+
+        # Validate git_url
+        if not git_url or not git_url.strip():
+            raise ValueError("git_url cannot be empty")
+
+        # Auto-detect token_env if not provided
+        if token_env is None:
+            token_env = self._default_token_env(source_type)
+
+        # Create source entry
+        source = {
+            "name": name.lower(),
+            "git_url": git_url.strip(),
+            "type": source_type.lower(),
+            "token_env": token_env,
+            "branch": branch,
+            "enabled": enabled,
+            "priority": priority,
+            "added_at": datetime.now(timezone.utc).isoformat(),
+            "updated_at": datetime.now(timezone.utc).isoformat(),
+        }
+
+        # Load registry
+        registry = self._read_registry()
+
+        # Check if source exists
+        existing_index = None
+        for i, existing_source in enumerate(registry["sources"]):
+            if existing_source["name"] == source["name"]:
+                existing_index = i
+                # Preserve added_at timestamp
+                source["added_at"] = existing_source.get("added_at", source["added_at"])
+                break
+
+        # Add or update
+        if existing_index is not None:
+            registry["sources"][existing_index] = source
+        else:
+            registry["sources"].append(source)
+
+        # Sort by priority (lower first)
+        registry["sources"].sort(key=lambda s: s["priority"])
+
+        # Save registry
+        self._write_registry(registry)
+
+        return source
+
+    def get_source(self, name: str) -> dict:
+        """
+        Get source by name.
+
+        Args:
+            name: Source identifier
+
+        Returns:
+            Source dictionary
+
+        Raises:
+            KeyError: If source not found
+        """
+        registry = self._read_registry()
+
+        # Search for source (case-insensitive)
+        name_lower = name.lower()
+        for source in registry["sources"]:
+            if source["name"] == name_lower:
+                return source
+
+        # Not found - provide helpful error
+        available = [s["name"] for s in registry["sources"]]
+        raise KeyError(
+            f"Source '{name}' not found. Available sources: {', '.join(available) if available else 'none'}"
+        )
+
+    def list_sources(self, enabled_only: bool = False) -> list[dict]:
+        """
+        List all config sources.
+
+        Args:
+            enabled_only: If True, only return enabled sources
+
+        Returns:
+            List of source dictionaries (sorted by priority)
+        """
+        registry = self._read_registry()
+
+        if enabled_only:
+            return [s for s in registry["sources"] if s.get("enabled", True)]
+
+        return registry["sources"]
+
+    def remove_source(self, name: str) -> bool:
+        """
+        Remove source by name.
+
+        Args:
+            name: Source identifier
+
+        Returns:
+            True if removed, False if not found
+        """
+        registry = self._read_registry()
+
+        # Find source index
+        name_lower = name.lower()
+        for i, source in enumerate(registry["sources"]):
+            if source["name"] == name_lower:
+                # Remove source
+                del registry["sources"][i]
+                # Save registry
+                self._write_registry(registry)
+                return True
+
+        return False
+
+    def update_source(self, name: str, **kwargs) -> dict:
+        """
+        Update specific fields of an existing source.
+
+        Args:
+            name: Source identifier
+            **kwargs: Fields to update (git_url, branch, enabled, priority, etc.)
+
+        Returns:
+            Updated source dictionary
+
+        Raises:
+            KeyError: If source not found
+        """
+        # Get existing source
+        source = self.get_source(name)
+
+        # Update allowed fields
+        allowed_fields = {"git_url", "type", "token_env", "branch", "enabled", "priority"}
+        for field, value in kwargs.items():
+            if field in allowed_fields:
+                source[field] = value
+
+        # Update timestamp
+        source["updated_at"] = datetime.now(timezone.utc).isoformat()
+
+        # Save changes
+        registry = self._read_registry()
+        for i, s in enumerate(registry["sources"]):
+            if s["name"] == source["name"]:
+                registry["sources"][i] = source
+                break
+
+        # Re-sort by priority
+        registry["sources"].sort(key=lambda s: s["priority"])
+
+        self._write_registry(registry)
+
+        return source
+
+    def _read_registry(self) -> dict:
+        """
+        Read registry from file.
+
+        Returns:
+            Registry dictionary
+        """
+        try:
+            with open(self.registry_file, encoding="utf-8") as f:
+                return json.load(f)
+        except json.JSONDecodeError as e:
+            raise ValueError(f"Corrupted registry file: {e}") from e
+
+    def _write_registry(self, registry: dict) -> None:
+        """
+        Write registry to file atomically.
+
+        Args:
+            registry: Registry dictionary
+        """
+        # Validate schema
+        if "version" not in registry or "sources" not in registry:
+            raise ValueError("Invalid registry schema")
+
+        # Atomic write: write to temp file, then rename
+        temp_file = self.registry_file.with_suffix(".tmp")
+
+        try:
+            with open(temp_file, "w", encoding="utf-8") as f:
+                json.dump(registry, f, indent=2, ensure_ascii=False)
+
+            # Atomic rename
+            temp_file.replace(self.registry_file)
+
+        except Exception as e:
+            # Clean up temp file on error
+            if temp_file.exists():
+                temp_file.unlink()
+            raise e
+
+    @staticmethod
+    def _default_token_env(source_type: str) -> str:
+        """
+        Get default token environment variable name for source type.
+
+        Args:
+            source_type: Source type (github, gitlab, bitbucket, custom)
+
+        Returns:
+            Environment variable name (e.g., GITHUB_TOKEN)
+        """
+        type_map = {
+            "github": "GITHUB_TOKEN",
+            "gitlab": "GITLAB_TOKEN",
+            "gitea": "GITEA_TOKEN",
+            "bitbucket": "BITBUCKET_TOKEN",
+            "custom": "GIT_TOKEN",
+        }
+
+        return type_map.get(source_type.lower(), "GIT_TOKEN")

skill_seekers/mcp/tools/__init__.py

@@ -0,0 +1,115 @@
+"""
+MCP Tool Implementations
+
+This package contains modular tool implementations for the Skill Seekers MCP server.
+Tools are organized by functionality:
+
+- config_tools: Configuration management (generate, list, validate)
+- scraping_tools: Scraping operations (docs, GitHub, PDF, estimation)
+- packaging_tools: Skill packaging and upload
+- splitting_tools: Config splitting and router generation
+- source_tools: Config source management (fetch, submit, add/remove sources)
+"""
+
+__version__ = "2.7.2"
+
+from .config_tools import (
+    generate_config as generate_config_impl,
+)
+from .config_tools import (
+    list_configs as list_configs_impl,
+)
+from .config_tools import (
+    validate_config as validate_config_impl,
+)
+from .packaging_tools import (
+    enhance_skill_tool as enhance_skill_impl,
+)
+from .packaging_tools import (
+    install_skill_tool as install_skill_impl,
+)
+from .packaging_tools import (
+    package_skill_tool as package_skill_impl,
+)
+from .packaging_tools import (
+    upload_skill_tool as upload_skill_impl,
+)
+from .scraping_tools import (
+    build_how_to_guides_tool as build_how_to_guides_impl,
+)
+from .scraping_tools import (
+    detect_patterns_tool as detect_patterns_impl,
+)
+from .scraping_tools import (
+    estimate_pages_tool as estimate_pages_impl,
+)
+from .scraping_tools import (
+    extract_config_patterns_tool as extract_config_patterns_impl,
+)
+from .scraping_tools import (
+    extract_test_examples_tool as extract_test_examples_impl,
+)
+from .scraping_tools import (
+    scrape_codebase_tool as scrape_codebase_impl,
+)
+from .scraping_tools import (
+    scrape_docs_tool as scrape_docs_impl,
+)
+from .scraping_tools import (
+    scrape_github_tool as scrape_github_impl,
+)
+from .scraping_tools import (
+    scrape_pdf_tool as scrape_pdf_impl,
+)
+from .source_tools import (
+    add_config_source_tool as add_config_source_impl,
+)
+from .source_tools import (
+    fetch_config_tool as fetch_config_impl,
+)
+from .source_tools import (
+    list_config_sources_tool as list_config_sources_impl,
+)
+from .source_tools import (
+    remove_config_source_tool as remove_config_source_impl,
+)
+from .source_tools import (
+    submit_config_tool as submit_config_impl,
+)
+from .splitting_tools import (
+    generate_router as generate_router_impl,
+)
+from .splitting_tools import (
+    split_config as split_config_impl,
+)
+
+__all__ = [
+    # Config tools
+    "generate_config_impl",
+    "list_configs_impl",
+    "validate_config_impl",
+    # Scraping tools
+    "estimate_pages_impl",
+    "scrape_docs_impl",
+    "scrape_github_impl",
+    "scrape_pdf_impl",
+    "scrape_codebase_impl",
+    "detect_patterns_impl",
+    "extract_test_examples_impl",
+    "build_how_to_guides_impl",
+    "extract_config_patterns_impl",
+    # Packaging tools
+    "package_skill_impl",
+    "upload_skill_impl",
+    "enhance_skill_impl",
+    "install_skill_impl",
+    # Splitting tools
+    "split_config_impl",
+    "generate_router_impl",
+    # Source tools
+    "fetch_config_impl",
+    "submit_config_impl",
+    "add_config_source_impl",
+    "list_config_sources_impl",
+    "remove_config_source_impl",
+]

skill_seekers/mcp/tools/config_tools.py

@@ -0,0 +1,251 @@
+"""
+Config management tools for Skill Seeker MCP Server.
+
+This module provides tools for generating, listing, and validating configuration files
+for documentation scraping.
+"""
+
+import json
+import sys
+from pathlib import Path
+
+try:
+    from mcp.types import TextContent
+except ImportError:
+    # Graceful degradation: Create a simple fallback class for testing
+    class TextContent:
+        """Fallback TextContent for when MCP is not installed"""
+
+        def __init__(self, type: str, text: str):
+            self.type = type
+            self.text = text
+
+
+# Path to CLI tools
+CLI_DIR = Path(__file__).parent.parent.parent / "cli"
+
+# Import config validator for validation
+sys.path.insert(0, str(CLI_DIR))
+try:
+    from config_validator import ConfigValidator
+except ImportError:
+    ConfigValidator = None  # Graceful degradation if not available
+
+
+async def generate_config(args: dict) -> list[TextContent]:
+    """
+    Generate a config file for documentation scraping.
+
+    Interactively creates a JSON config for any documentation website with default
+    selectors and sensible defaults. The config can be further customized after creation.
+
+    Args:
+        args: Dictionary containing:
+            - name (str): Skill name (lowercase, alphanumeric, hyphens, underscores)
+            - url (str): Base documentation URL (must include http:// or https://)
+            - description (str): Description of when to use this skill
+            - max_pages (int, optional): Maximum pages to scrape (default: 100, use -1 for unlimited)
+            - unlimited (bool, optional): Remove all limits - scrape all pages (default: False). Overrides max_pages.
+            - rate_limit (float, optional): Delay between requests in seconds (default: 0.5)
+
+    Returns:
+        List[TextContent]: Success message with config path and next steps, or error message.
+    """
+    name = args["name"]
+    url = args["url"]
+    description = args["description"]
+    max_pages = args.get("max_pages", 100)
+    unlimited = args.get("unlimited", False)
+    rate_limit = args.get("rate_limit", 0.5)
+
+    # Handle unlimited mode
+    if unlimited or max_pages == -1:
+        max_pages = None
+        limit_msg = "unlimited (no page limit)"
+    else:
+        limit_msg = str(max_pages)
+
+    # Create config
+    config = {
+        "name": name,
+        "description": description,
+        "base_url": url,
+        "selectors": {"main_content": "article", "title": "h1", "code_blocks": "pre code"},
+        "url_patterns": {"include": [], "exclude": []},
+        "categories": {},
+        "rate_limit": rate_limit,
+        "max_pages": max_pages,
+    }
+
+    # Save to configs directory
+    config_path = Path("configs") / f"{name}.json"
+    config_path.parent.mkdir(exist_ok=True)
+
+    with open(config_path, "w") as f:
+        json.dump(config, f, indent=2)
+
+    result = f"""✅ Config created: {config_path}
+
+Configuration:
+  Name: {name}
+  URL: {url}
+  Max pages: {limit_msg}
+  Rate limit: {rate_limit}s
+
+Next steps:
+  1. Review/edit config: cat {config_path}
+  2. Estimate pages: Use estimate_pages tool
+  3. Scrape docs: Use scrape_docs tool
+
+Note: Default selectors may need adjustment for your documentation site.
+"""
+
+    return [TextContent(type="text", text=result)]
+
+
+async def list_configs(_args: dict) -> list[TextContent]:
+    """
+    List all available preset configurations.
+
+    Scans the configs directory and lists all available config files with their
+    basic information (name, URL, description).
+
+    Args:
+        args: Dictionary (empty, no parameters required)
+
+    Returns:
+        List[TextContent]: Formatted list of available configs with details, or error if no configs found.
+    """
+    configs_dir = Path("configs")
+
+    if not configs_dir.exists():
+        return [TextContent(type="text", text="No configs directory found")]
+
+    configs = list(configs_dir.glob("*.json"))
+
+    if not configs:
+        return [TextContent(type="text", text="No config files found")]
+
+    result = "📋 Available Configs:\n\n"
+
+    for config_file in sorted(configs):
+        try:
+            with open(config_file) as f:
+                config = json.load(f)
+                name = config.get("name", config_file.stem)
+                desc = config.get("description", "No description")
+                url = config.get("base_url", "")
+
+                result += f"  • {config_file.name}\n"
+                result += f"    Name: {name}\n"
+                result += f"    URL: {url}\n"
+                result += f"    Description: {desc}\n\n"
+        except Exception as e:
+            result += f"  • {config_file.name} - Error reading: {e}\n\n"
+
+    return [TextContent(type="text", text=result)]
+
+
+async def validate_config(args: dict) -> list[TextContent]:
+    """
+    Validate a config file for errors.
+
+    Validates both legacy (single-source) and unified (multi-source) config formats.
+    Checks for required fields, valid URLs, proper structure, and provides detailed
+    feedback on any issues found.
+
+    Args:
+        args: Dictionary containing:
+            - config_path (str): Path to config JSON file to validate
+
+    Returns:
+        List[TextContent]: Validation results with format details and any errors/warnings, or error message.
+    """
+    config_path = args["config_path"]
+
+    # Import validation classes
+    sys.path.insert(0, str(CLI_DIR))
+
+    try:
+        # Check if file exists
+        if not Path(config_path).exists():
+            return [
+                TextContent(type="text", text=f"❌ Error: Config file not found: {config_path}")
+            ]
+
+        # Try unified config validator first
+        try:
+            from config_validator import validate_config
+
+            validator = validate_config(config_path)
+
+            result = "✅ Config is valid!\n\n"
+
+            # Show format
+            if validator.is_unified:
+                result += "📦 Format: Unified (multi-source)\n"
+                result += f"  Name: {validator.config['name']}\n"
+                result += f"  Sources: {len(validator.config.get('sources', []))}\n"
+
+                # Show sources
+                for i, source in enumerate(validator.config.get("sources", []), 1):
+                    result += f"\n  Source {i}: {source['type']}\n"
+                    if source["type"] == "documentation":
+                        result += f"    URL: {source.get('base_url', 'N/A')}\n"
+                        result += f"    Max pages: {source.get('max_pages', 'Not set')}\n"
+                    elif source["type"] == "github":
+                        result += f"    Repo: {source.get('repo', 'N/A')}\n"
+                        result += (
+                            f"    Code depth: {source.get('code_analysis_depth', 'surface')}\n"
+                        )
+                    elif source["type"] == "pdf":
+                        result += f"    Path: {source.get('path', 'N/A')}\n"
+
+                # Show merge settings if applicable
+                if validator.needs_api_merge():
+                    merge_mode = validator.config.get("merge_mode", "rule-based")
+                    result += f"\n  Merge mode: {merge_mode}\n"
+                    result += "  API merging: Required (docs + code sources)\n"
+
+            else:
+                result += "📦 Format: Legacy (single source)\n"
+                result += f"  Name: {validator.config['name']}\n"
+                result += f"  Base URL: {validator.config.get('base_url', 'N/A')}\n"
+                result += f"  Max pages: {validator.config.get('max_pages', 'Not set')}\n"
+                result += f"  Rate limit: {validator.config.get('rate_limit', 'Not set')}s\n"
+
+            return [TextContent(type="text", text=result)]
+
+        except ImportError:
+            # Fall back to legacy validation
+            import json
+
+            from doc_scraper import validate_config
+
+            with open(config_path) as f:
+                config = json.load(f)
+
+            # Validate config - returns (errors, warnings) tuple
+            errors, warnings = validate_config(config)
+
+            if errors:
+                result = "❌ Config validation failed:\n\n"
+                for error in errors:
+                    result += f"  • {error}\n"
+            else:
+                result = "✅ Config is valid!\n\n"
+                result += "📦 Format: Legacy (single source)\n"
+                result += f"  Name: {config['name']}\n"
+                result += f"  Base URL: {config['base_url']}\n"
+                result += f"  Max pages: {config.get('max_pages', 'Not set')}\n"
+                result += f"  Rate limit: {config.get('rate_limit', 'Not set')}s\n"
+
+                if warnings:
+                    result += "\n⚠️  Warnings:\n"
+                    for warning in warnings:
+                        result += f"  • {warning}\n"
+
+            return [TextContent(type="text", text=result)]
+
+    except Exception as e:
+        return [TextContent(type="text", text=f"❌ Error: {str(e)}")]