napistu 0.1.0__py3-none-any.whl → 0.2.4.dev3__py3-none-any.whl

napistu/mcp/__main__.py

@@ -0,0 +1,180 @@
+"""
+MCP (Model Context Protocol) Server for Napistu.
+"""
+
+import asyncio
+import logging
+import click
+import click_logging
+
+import napistu
+from napistu.mcp.profiles import get_profile, ServerProfile
+from napistu.mcp.server import create_server
+
+logger = logging.getLogger(napistu.__name__)
+click_logging.basic_config(logger)
+
+
+@click.group()
+def cli():
+    """The Napistu MCP (Model Context Protocol) Server CLI"""
+    pass
+
+
+@click.group()
+def server():
+    """Start and manage MCP servers."""
+    pass
+
+
+@server.command(name="start")
+@click.option(
+    "--profile",
+    type=click.Choice(["local", "remote", "full"]),
+    default="remote",
+    help="Predefined configuration profile",
+)
+@click.option("--server-name", type=str, help="Name of the MCP server")
+@click_logging.simple_verbosity_option(logger)
+def start_server(profile, server_name):
+    """Start an MCP server with the specified profile."""
+    # Collect configuration
+    config = {}
+    if server_name:
+        config["server_name"] = server_name
+
+    # Get profile with overrides
+    server_profile = get_profile(profile, **config)
+
+    # Create and start the server
+    logger.info(f"Starting Napistu MCP Server with {profile} profile...")
+    server = create_server(server_profile)
+    asyncio.run(server.start())
+
+
+@server.command(name="local")
+@click.option(
+    "--server-name", type=str, default="napistu-local", help="Name of the MCP server"
+)
+@click_logging.simple_verbosity_option(logger)
+def start_local(server_name):
+    """Start a local MCP server optimized for function execution."""
+    # Get profile with overrides
+    server_profile = get_profile("local", server_name=server_name)
+
+    # Create and start the server
+    logger.info("Starting Napistu local MCP Server...")
+    server = create_server(server_profile)
+    asyncio.run(server.start())
+
+
+@server.command(name="remote")
+@click.option(
+    "--server-name", type=str, default="napistu-docs", help="Name of the MCP server"
+)
+@click.option("--codebase-path", type=str, help="Path to the Napistu codebase")
+@click.option(
+    "--docs-paths",
+    type=str,
+    help="Comma-separated list of paths to documentation files",
+)
+@click.option("--tutorials-path", type=str, help="Path to the tutorials directory")
+@click_logging.simple_verbosity_option(logger)
+def start_remote(server_name, tutorials_path):
+    """Start a remote MCP server for documentation and codebase exploration."""
+    # Collect configuration
+    config = {"server_name": server_name}
+    if tutorials_path:
+        config["tutorials_path"] = tutorials_path
+
+    # Get profile with overrides
+    server_profile = get_profile("remote", **config)
+
+    # Create and start the server
+    logger.info("Starting Napistu remote MCP Server...")
+    server = create_server(server_profile)
+    asyncio.run(server.start())
+
+
+@click.group()
+def component():
+    """Enable or disable specific MCP server components."""
+    pass
+
+
+@component.command(name="list")
+def list_components():
+    """List available MCP server components."""
+    click.echo("Available MCP server components:")
+    click.echo("  - documentation: Documentation components")
+    click.echo("  - codebase: Codebase exploration components")
+    click.echo("  - execution: Function execution components")
+    click.echo("  - tutorials: Tutorial components")
+
+
+@component.command(name="custom")
+@click.option(
+    "--enable-documentation/--disable-documentation",
+    default=None,
+    help="Enable/disable documentation components",
+)
+@click.option(
+    "--enable-codebase/--disable-codebase",
+    default=None,
+    help="Enable/disable codebase exploration components",
+)
+@click.option(
+    "--enable-execution/--disable-execution",
+    default=None,
+    help="Enable/disable function execution components",
+)
+@click.option(
+    "--enable-tutorials/--disable-tutorials",
+    default=None,
+    help="Enable/disable tutorial components",
+)
+@click.option(
+    "--server-name", type=str, default="napistu-custom", help="Name of the MCP server"
+)
+@click.option("--codebase-path", type=str, help="Path to the Napistu codebase")
+@click.option(
+    "--docs-paths",
+    type=str,
+    help="Comma-separated list of paths to documentation files",
+)
+@click.option("--tutorials-path", type=str, help="Path to the tutorials directory")
+@click_logging.simple_verbosity_option(logger)
+def custom_server(
+    enable_documentation,
+    enable_codebase,
+    enable_execution,
+    enable_tutorials,
+    server_name,
+):
+    """Start an MCP server with custom component configuration."""
+    # Collect configuration
+    config = {"server_name": server_name}
+    if enable_documentation is not None:
+        config["enable_documentation"] = enable_documentation
+    if enable_codebase is not None:
+        config["enable_codebase"] = enable_codebase
+    if enable_execution is not None:
+        config["enable_execution"] = enable_execution
+    if enable_tutorials is not None:
+        config["enable_tutorials"] = enable_tutorials
+
+    # Create a custom profile
+    server_profile = ServerProfile(**config)
+
+    # Create and start the server
+    logger.info("Starting Napistu custom MCP Server...")
+    server = create_server(server_profile)
+    asyncio.run(server.start())
+
+
+# Add command groups to the CLI
+cli.add_command(server)
+cli.add_command(component)
+
+if __name__ == "__main__":
+    cli()

napistu/mcp/codebase.py

@@ -0,0 +1,182 @@
+"""
+Codebase exploration components for the Napistu MCP server.
+"""
+
+from napistu.mcp.constants import NAPISTU_PY_READTHEDOCS_API
+
+from fastmcp import FastMCP
+
+from typing import Dict, Any
+import json
+
+from napistu.mcp import codebase_utils
+from napistu.mcp import utils as mcp_utils
+
+# Global cache for codebase information
+_codebase_cache = {
+    "modules": {},
+    "classes": {},
+    "functions": {},
+}
+
+
+async def initialize_components() -> bool:
+    """
+    Initialize codebase components.
+
+    Returns
+    -------
+    bool
+        True if initialization is successful.
+    """
+    global _codebase_cache
+    # Load documentation from the ReadTheDocs API
+    _codebase_cache["modules"] = await codebase_utils.read_read_the_docs(
+        NAPISTU_PY_READTHEDOCS_API
+    )
+    # Extract functions and classes from the modules
+    _codebase_cache["functions"], _codebase_cache["classes"] = (
+        codebase_utils.extract_functions_and_classes_from_modules(
+            _codebase_cache["modules"]
+        )
+    )
+    return True
+
+
+def register_components(mcp: FastMCP):
+    """
+    Register codebase exploration components with the MCP server.
+
+    Args:
+        mcp: FastMCP server instance
+    """
+    global _codebase_cache
+
+    # Register resources
+    @mcp.resource("napistu://codebase/summary")
+    async def get_codebase_summary() -> Dict[str, Any]:
+        """
+        Get a summary of the Napistu codebase structure.
+        """
+        return {
+            "modules": list(_codebase_cache["modules"].keys()),
+            "top_level_classes": [
+                class_name
+                for class_name, info in _codebase_cache["classes"].items()
+                if "." not in class_name  # Only include top-level classes
+            ],
+            "top_level_functions": [
+                func_name
+                for func_name, info in _codebase_cache["functions"].items()
+                if "." not in func_name  # Only include top-level functions
+            ],
+        }
+
+    @mcp.resource("napistu://codebase/modules/{module_name}")
+    async def get_module_details(module_name: str) -> Dict[str, Any]:
+        """
+        Get detailed information about a specific module.
+
+        Args:
+            module_name: Name of the module
+        """
+        if module_name not in _codebase_cache["modules"]:
+            return {"error": f"Module {module_name} not found"}
+
+        return _codebase_cache["modules"][module_name]
+
+    # Register tools
+    @mcp.tool()
+    async def search_codebase(query: str) -> Dict[str, Any]:
+        """
+        Search the codebase for a specific query.
+
+        Args:
+            query: Search term
+
+        Returns:
+            Dictionary with search results organized by code element type, including snippets for context.
+        """
+        results = {
+            "modules": [],
+            "classes": [],
+            "functions": [],
+        }
+
+        # Search modules
+        for module_name, info in _codebase_cache["modules"].items():
+            # Use docstring or description for snippet
+            doc = info.get("doc") or info.get("description") or ""
+            module_text = json.dumps(info)
+            if query.lower() in module_text.lower():
+                snippet = mcp_utils.get_snippet(doc, query)
+                results["modules"].append(
+                    {
+                        "name": module_name,
+                        "description": doc,
+                        "snippet": snippet,
+                    }
+                )
+
+        # Search classes
+        for class_name, info in _codebase_cache["classes"].items():
+            doc = info.get("doc") or info.get("description") or ""
+            class_text = json.dumps(info)
+            if query.lower() in class_text.lower():
+                snippet = mcp_utils.get_snippet(doc, query)
+                results["classes"].append(
+                    {
+                        "name": class_name,
+                        "description": doc,
+                        "snippet": snippet,
+                    }
+                )
+
+        # Search functions
+        for func_name, info in _codebase_cache["functions"].items():
+            doc = info.get("doc") or info.get("description") or ""
+            func_text = json.dumps(info)
+            if query.lower() in func_text.lower():
+                snippet = mcp_utils.get_snippet(doc, query)
+                results["functions"].append(
+                    {
+                        "name": func_name,
+                        "description": doc,
+                        "signature": info.get("signature", ""),
+                        "snippet": snippet,
+                    }
+                )
+
+        return results
+
+    @mcp.tool()
+    async def get_function_documentation(function_name: str) -> Dict[str, Any]:
+        """
+        Get detailed documentation for a specific function.
+
+        Args:
+            function_name: Name of the function
+
+        Returns:
+            Dictionary with function documentation
+        """
+        if function_name not in _codebase_cache["functions"]:
+            return {"error": f"Function {function_name} not found"}
+
+        return _codebase_cache["functions"][function_name]
+
+    @mcp.tool()
+    async def get_class_documentation(class_name: str) -> Dict[str, Any]:
+        """
+        Get detailed documentation for a specific class.
+
+        Args:
+            class_name: Name of the class
+
+        Returns:
+            Dictionary with class documentation
+        """
+        if class_name not in _codebase_cache["classes"]:
+            return {"error": f"Class {class_name} not found"}
+
+        return _codebase_cache["classes"][class_name]

napistu/mcp/codebase_utils.py

@@ -0,0 +1,298 @@
+"""
+Utilities for scanning and analyzing the Napistu codebase.
+"""
+
+from typing import Dict, Optional, Any, Set
+
+from napistu.mcp import utils as mcp_utils
+from napistu.mcp.constants import READTHEDOCS_TOC_CSS_SELECTOR
+
+# Import optional dependencies with error handling
+try:
+    from bs4 import BeautifulSoup
+except ImportError:
+    raise ImportError(
+        "Documentation utilities require additional dependencies. Install with 'pip install napistu[mcp]'"
+    )
+
+
+async def read_read_the_docs(package_toc_url: str) -> dict:
+    """
+    Recursively parse all modules and submodules starting from the package TOC.
+    """
+    # Step 1: Get all module URLs from the TOC
+    packages_dict = await _process_rtd_package_toc(package_toc_url)
+    docs_dict = {}
+    visited = set()
+
+    # Step 2: Recursively parse each module page
+    for package_name, module_url in packages_dict.items():
+        if not module_url.startswith("http"):
+            # Make absolute if needed
+            base = package_toc_url.rsplit("/", 1)[0]
+            module_url = base + "/" + module_url.lstrip("/")
+        await _parse_rtd_module_recursive(module_url, visited, docs_dict)
+
+    return docs_dict
+
+
+def extract_functions_and_classes_from_modules(modules: dict) -> tuple[dict, dict]:
+    """
+    Process the modules cache and return a tuple (functions_dict, classes_dict),
+    where each is a dict keyed by fully qualified name (e.g., 'module.func', 'module.Class').
+    Recursively processes submodules.
+
+    Args:
+        modules (dict): The modules cache as returned by read_read_the_docs.
+
+    Returns:
+        tuple: (functions_dict, classes_dict)
+    """
+    functions = {}
+    classes = {}
+
+    def _process_module(module_name: str, module_info: dict):
+        # Functions
+        for func_name, func_info in module_info.get("functions", {}).items():
+            fq_name = f"{module_name}.{func_name}"
+            functions[fq_name] = func_info
+        # Classes
+        for class_name, class_info in module_info.get("classes", {}).items():
+            fq_name = f"{module_name}.{class_name}"
+            classes[fq_name] = class_info
+        # Submodules (if present in the cache)
+        for submod_name in module_info.get("submodules", {}):
+            fq_submod_name = f"{module_name}.{submod_name}"
+            if fq_submod_name in modules:
+                _process_module(fq_submod_name, modules[fq_submod_name])
+
+    for module_name, module_info in modules.items():
+        _process_module(module_name, module_info)
+
+    return functions, classes
+
+
+def _parse_rtd_module_page(html: str, url: Optional[str] = None) -> dict:
+    """
+    Parse a ReadTheDocs module HTML page and extract functions, classes, methods, attributes, and submodules.
+    Returns a dict suitable for MCP server use, with functions, classes, and methods keyed by name.
+
+    Args:
+        html (str): The HTML content of the module page.
+        url (Optional[str]): The URL of the page (for reference).
+
+    Returns:
+        dict: {
+            'module': str,
+            'url': str,
+            'functions': Dict[str, dict],
+            'classes': Dict[str, dict],
+            'submodules': Dict[str, dict]
+        }
+    """
+    soup = BeautifulSoup(html, "html.parser")
+    result = {
+        "module": None,
+        "url": url,
+        "functions": {},
+        "classes": {},
+        "submodules": _format_submodules(soup),
+    }
+    # Get module name from <h1>
+    h1 = soup.find("h1")
+    if h1:
+        module_name = h1.get_text(strip=True).replace("\uf0c1", "").strip()
+        result["module"] = module_name
+    # Functions
+    for func_dl in soup.find_all("dl", class_="py function"):
+        func = _format_function(func_dl.find("dt"), func_dl.find("dd"))
+        if func["name"]:
+            result["functions"][func["name"]] = func
+    # Classes
+    for class_dl in soup.find_all("dl", class_="py class"):
+        cls = _format_class(class_dl)
+        if cls["name"]:
+            result["classes"][cls["name"]] = cls
+    return result
+
+
+async def _process_rtd_package_toc(
+    url: str, css_selector: str = READTHEDOCS_TOC_CSS_SELECTOR
+) -> dict:
+    """
+    Parse the ReadTheDocs package TOC and return a dict of {name: url}.
+    """
+    page_html = await mcp_utils.load_html_page(url)
+    soup = BeautifulSoup(page_html, "html.parser")
+    selected = soup.select(css_selector)
+    return _parse_module_tags(selected)
+
+
+def _parse_module_tags(td_list: list, base_url: str = "") -> dict:
+    """
+    Parse a list of <td> elements containing module links and return a dict of {name: url}.
+    Optionally prepends base_url to relative hrefs.
+    """
+    result = {}
+    for td in td_list:
+        a = td.find("a", class_="reference internal")
+        if a:
+            # Get the module name from the <span class="pre"> tag
+            span = a.find("span", class_="pre")
+            if span:
+                name = span.text.strip()
+                href = a.get("href")
+                # Prepend base_url if href is relative
+                if href and not href.startswith("http"):
+                    href = base_url.rstrip("/") + "/" + href.lstrip("/")
+                result[name] = href
+    return result
+
+
+def _format_function(sig_dt, doc_dd) -> Dict[str, Any]:
+    """
+    Format a function or method signature and its documentation into a dictionary.
+
+    Args:
+        sig_dt: The <dt> tag containing the function/method signature.
+        doc_dd: The <dd> tag containing the function/method docstring.
+
+    Returns:
+        dict: A dictionary with keys 'name', 'signature', 'id', and 'doc'.
+    """
+    name = (
+        sig_dt.find("span", class_="sig-name").get_text(strip=True) if sig_dt else None
+    )
+    signature = sig_dt.get_text(strip=True) if sig_dt else None
+    return {
+        "name": mcp_utils._clean_signature_text(name),
+        "signature": mcp_utils._clean_signature_text(signature),
+        "id": sig_dt.get("id") if sig_dt else None,
+        "doc": doc_dd.get_text(" ", strip=True) if doc_dd else None,
+    }
+
+
+def _format_attribute(attr_dl) -> Dict[str, Any]:
+    """
+    Format a class attribute's signature and documentation into a dictionary.
+
+    Args:
+        attr_dl: The <dl> tag for the attribute, containing <dt> and <dd>.
+
+    Returns:
+        dict: A dictionary with keys 'name', 'signature', 'id', and 'doc'.
+    """
+    sig = attr_dl.find("dt")
+    doc = attr_dl.find("dd")
+    name = sig.find("span", class_="sig-name").get_text(strip=True) if sig else None
+    signature = sig.get_text(strip=True) if sig else None
+    return {
+        "name": mcp_utils._clean_signature_text(name),
+        "signature": mcp_utils._clean_signature_text(signature),
+        "id": sig.get("id") if sig else None,
+        "doc": doc.get_text(" ", strip=True) if doc else None,
+    }
+
+
+def _format_class(class_dl) -> Dict[str, Any]:
+    """
+    Format a class definition, including its methods and attributes, into a dictionary.
+
+    Args:
+        class_dl: The <dl> tag for the class, containing <dt> and <dd>.
+
+    Returns:
+        dict: A dictionary with keys 'name', 'signature', 'id', 'doc', 'methods', and 'attributes'.
+              'methods' and 'attributes' are themselves dicts keyed by name.
+    """
+    sig = class_dl.find("dt")
+    doc = class_dl.find("dd")
+    class_name = (
+        sig.find("span", class_="sig-name").get_text(strip=True) if sig else None
+    )
+    methods = {}
+    attributes = {}
+    if doc:
+        for meth_dl in doc.find_all("dl", class_="py method"):
+            meth = _format_function(meth_dl.find("dt"), meth_dl.find("dd"))
+            if meth["name"]:
+                methods[meth["name"]] = meth
+        for attr_dl in doc.find_all("dl", class_="py attribute"):
+            attr = _format_attribute(attr_dl)
+            if attr["name"]:
+                attributes[attr["name"]] = attr
+    return {
+        "name": mcp_utils._clean_signature_text(class_name),
+        "signature": mcp_utils._clean_signature_text(
+            sig.get_text(strip=True) if sig else None
+        ),
+        "id": sig.get("id") if sig else None,
+        "doc": doc.get_text(" ", strip=True) if doc else None,
+        "methods": methods,
+        "attributes": attributes,
+    }
+
+
+def _format_submodules(soup) -> dict:
+    """
+    Extract submodules from a ReadTheDocs module page soup object.
+    Looks for a 'Modules' rubric and parses the following table or list for submodule names, URLs, and descriptions.
+
+    Args:
+        soup (BeautifulSoup): Parsed HTML soup of the module page.
+
+    Returns:
+        dict: {submodule_name: {"url": str, "description": str}}
+    """
+    submodules = {}
+    for rubric in soup.find_all("p", class_="rubric"):
+        if rubric.get_text(strip=True).lower() == "modules":
+            sib = rubric.find_next_sibling()
+            if sib and sib.name in ("table", "ul"):
+                for a in sib.find_all("a", href=True):
+                    submod_name = a.get_text(strip=True)
+                    submod_url = a["href"]
+                    desc = ""
+                    td = a.find_parent("td")
+                    if td and td.find_next_sibling("td"):
+                        desc = td.find_next_sibling("td").get_text(strip=True)
+                    elif a.parent.name == "li":
+                        next_p = a.find_next_sibling("p")
+                        if next_p:
+                            desc = next_p.get_text(strip=True)
+                    submodules[submod_name] = {"url": submod_url, "description": desc}
+    return submodules
+
+
+async def _parse_rtd_module_recursive(
+    module_url: str,
+    visited: Optional[Set[str]] = None,
+    docs_dict: Optional[Dict[str, Any]] = None,
+) -> Dict[str, Any]:
+    """
+    Recursively parse a module page and all its submodules.
+    """
+
+    if visited is None:
+        visited = set()
+    if docs_dict is None:
+        docs_dict = {}
+
+    if module_url in visited:
+        return docs_dict
+    visited.add(module_url)
+
+    page_html = await mcp_utils.load_html_page(module_url)
+    module_doc = _parse_rtd_module_page(page_html, module_url)
+    module_name = module_doc.get("module") or module_url
+    docs_dict[module_name] = module_doc
+
+    # Recursively parse submodules
+    for submod_name, submod_info in module_doc.get("submodules", {}).items():
+        submod_url = submod_info["url"]
+        if not submod_url.startswith("http"):
+            base = module_url.rsplit("/", 1)[0]
+            submod_url = base + "/" + submod_url.lstrip("/")
+        await _parse_rtd_module_recursive(submod_url, visited, docs_dict)
+
+    return docs_dict