python-docs-mcp-server 0.1.2__py3-none-any.whl

mcp_server_python_docs/retrieval/ranker.py

@@ -0,0 +1,369 @@
+"""BM25 ranking with column weights and FTS5 snippets.
+
+Search functions that execute FTS5 queries with BM25 column weighting
+and snippet() excerpts. All functions return list[SymbolHit] with the
+locked hit shape (RETR-09).
+
+Receives sqlite3.Connection as parameter -- does not import storage.
+"""
+from __future__ import annotations
+
+import logging
+import sqlite3
+
+from mcp_server_python_docs.models import SymbolHit
+
+logger = logging.getLogger(__name__)
+
+
+def _page_uri(uri: str) -> str:
+    """Return the page part of an objects.inv URI."""
+    return uri.split("#", 1)[0]
+
+
+def _document_candidates(uri: str) -> tuple[str, ...]:
+    """Return possible document slugs for a symbol URI.
+
+    ``objects.inv`` entries use HTML paths such as ``library/json.html``,
+    while Sphinx JSON content is ingested with extensionless slugs such as
+    ``library/json``. Prefer the exact URI first, then the extensionless form.
+    """
+    page_uri = _page_uri(uri)
+    if page_uri.endswith(".html"):
+        return (page_uri, page_uri[:-5])
+    return (page_uri,)
+
+
+def _resolve_symbol_location(
+    conn: sqlite3.Connection,
+    row: sqlite3.Row,
+) -> tuple[str, str | None]:
+    """Resolve a symbol row to a get_docs-compatible slug and anchor.
+
+    Returns ``(slug, None)`` when the hit is page-level. ``get_docs()`` treats
+    any non-None anchor as a section lookup, so blank anchors must never leak.
+    """
+    uri = str(row["uri"])
+    fallback_slug = _page_uri(uri)
+    fallback_anchor = str(row["anchor"] or "")
+
+    section_id = row["section_id"]
+    if section_id is not None:
+        section_row = conn.execute(
+            """
+            SELECT d.slug, s.anchor
+            FROM sections s
+            JOIN documents d ON s.document_id = d.id
+            WHERE s.id = ?
+            LIMIT 1
+            """,
+            (section_id,),
+        ).fetchone()
+        if section_row is not None:
+            return section_row["slug"], (section_row["anchor"] or None)
+
+    doc_row = None
+    document_id = row["document_id"]
+    if document_id is not None:
+        doc_row = conn.execute(
+            "SELECT id, slug FROM documents WHERE id = ? LIMIT 1",
+            (document_id,),
+        ).fetchone()
+
+    if doc_row is None:
+        for candidate in _document_candidates(uri):
+            doc_row = conn.execute(
+                """
+                SELECT id, slug
+                FROM documents
+                WHERE doc_set_id = ? AND (slug = ? OR uri = ?)
+                LIMIT 1
+                """,
+                (row["doc_set_id"], candidate, candidate),
+            ).fetchone()
+            if doc_row is not None:
+                break
+
+    if doc_row is None:
+        return fallback_slug, (fallback_anchor or None)
+
+    if fallback_anchor:
+        section_row = conn.execute(
+            """
+            SELECT 1
+            FROM sections
+            WHERE document_id = ? AND anchor = ?
+            LIMIT 1
+            """,
+            (doc_row["id"], fallback_anchor),
+        ).fetchone()
+        if section_row is not None:
+            return doc_row["slug"], fallback_anchor
+
+    return doc_row["slug"], None
+
+
+def _normalize_scores(hits: list[SymbolHit]) -> list[SymbolHit]:
+    """Normalize BM25 scores to [0.1, 1.0] range.
+
+    BM25 returns negative scores (lower = better). This normalizes
+    to a positive range where 1.0 is best match and 0.1 is worst
+    in the batch.
+    """
+    if not hits:
+        return hits
+    if len(hits) == 1:
+        # Single hit always gets score 1.0
+        return [hits[0].model_copy(update={"score": 1.0})]
+
+    raw_scores = [h.score for h in hits]
+    min_score = min(raw_scores)
+    max_score = max(raw_scores)
+
+    if max_score == min_score:
+        return [h.model_copy(update={"score": 1.0}) for h in hits]
+
+    normalized = []
+    for hit in hits:
+        # BM25 scores are negative; lower (more negative) = better
+        # Normalize so the best (most negative) gets 1.0
+        norm = 0.1 + 0.9 * (max_score - hit.score) / (max_score - min_score)
+        normalized.append(hit.model_copy(update={"score": round(norm, 4)}))
+
+    return normalized
+
+
+def search_sections(
+    conn: sqlite3.Connection,
+    match_expr: str,
+    version: str | None,
+    max_results: int,
+) -> list[SymbolHit]:
+    """Search sections via FTS5 with BM25 column weights (RETR-06, RETR-07).
+
+    Column weights: heading (10.0) > content_text (1.0).
+    Returns ~200-char FTS5 snippet() excerpts.
+
+    Args:
+        conn: Read-only SQLite connection.
+        match_expr: FTS5-safe MATCH expression (from fts5_escape or
+            build_match_expression).
+        version: Python version filter, or None for all versions.
+        max_results: Maximum number of hits.
+
+    Returns:
+        List of SymbolHit with kind="section" and FTS5 snippets.
+    """
+    try:
+        cursor = conn.execute(
+            """
+            SELECT s.id, s.heading, s.uri, s.anchor,
+                   d.version, doc.slug,
+                   bm25(sections_fts, 10.0, 1.0) as score,
+                   snippet(sections_fts, 1, '**', '**', '...', 32) as snippet_text
+            FROM sections_fts
+            JOIN sections s ON sections_fts.rowid = s.id
+            JOIN documents doc ON s.document_id = doc.id
+            JOIN doc_sets d ON doc.doc_set_id = d.id
+            WHERE sections_fts MATCH ?
+              AND (? IS NULL OR d.version = ?)
+            ORDER BY bm25(sections_fts, 10.0, 1.0)
+            LIMIT ?
+            """,
+            (match_expr, version, version, max_results),
+        )
+        rows = cursor.fetchall()
+    except sqlite3.OperationalError:
+        logger.warning("FTS5 query failed for sections: %r", match_expr)
+        return []
+
+    hits = [
+        SymbolHit(
+            uri=row["uri"],
+            title=row["heading"],
+            kind="section",
+            snippet=row["snippet_text"] or "",
+            score=row["score"],
+            version=row["version"],
+            slug=row["slug"],
+            anchor=row["anchor"] or None,
+        )
+        for row in rows
+    ]
+
+    return _normalize_scores(hits)
+
+
+def search_symbols(
+    conn: sqlite3.Connection,
+    match_expr: str,
+    version: str | None,
+    max_results: int,
+) -> list[SymbolHit]:
+    """Search symbols via FTS5 with BM25 column weights (RETR-06).
+
+    Column weights: qualified_name (10.0) > module (1.0).
+
+    Args:
+        conn: Read-only SQLite connection.
+        match_expr: FTS5-safe MATCH expression.
+        version: Python version filter, or None for all versions.
+        max_results: Maximum number of hits.
+
+    Returns:
+        List of SymbolHit with kind from symbol_type.
+    """
+    try:
+        cursor = conn.execute(
+            """
+            SELECT sym.id, sym.doc_set_id, sym.document_id, sym.section_id,
+                   sym.qualified_name, sym.symbol_type, sym.uri,
+                   sym.anchor, sym.module, d.version,
+                   bm25(symbols_fts, 10.0, 1.0) as score,
+                   snippet(symbols_fts, 0, '**', '**', '...', 32) as snippet_text
+            FROM symbols_fts
+            JOIN symbols sym ON symbols_fts.rowid = sym.id
+            JOIN doc_sets d ON sym.doc_set_id = d.id
+            WHERE symbols_fts MATCH ?
+              AND (? IS NULL OR d.version = ?)
+            ORDER BY bm25(symbols_fts, 10.0, 1.0)
+            LIMIT ?
+            """,
+            (match_expr, version, version, max_results),
+        )
+        rows = cursor.fetchall()
+    except sqlite3.OperationalError:
+        logger.warning("FTS5 query failed for symbols: %r", match_expr)
+        return []
+
+    hits = []
+    for row in rows:
+        slug, anchor = _resolve_symbol_location(conn, row)
+        hits.append(SymbolHit(
+            uri=row["uri"],
+            title=row["qualified_name"],
+            kind=row["symbol_type"] or "symbol",
+            snippet=row["snippet_text"] or "",
+            score=row["score"],
+            version=row["version"],
+            slug=slug,
+            anchor=anchor,
+        ))
+
+    return _normalize_scores(hits)
+
+
+def search_examples(
+    conn: sqlite3.Connection,
+    match_expr: str,
+    version: str | None,
+    max_results: int,
+) -> list[SymbolHit]:
+    """Search examples via FTS5 (RETR-06).
+
+    Args:
+        conn: Read-only SQLite connection.
+        match_expr: FTS5-safe MATCH expression.
+        version: Python version filter, or None for all versions.
+        max_results: Maximum number of hits.
+
+    Returns:
+        List of SymbolHit with kind="example" or "doctest".
+    """
+    try:
+        cursor = conn.execute(
+            """
+            SELECT e.id, e.code, e.is_doctest,
+                   s.heading, s.uri as section_uri, s.anchor,
+                   d.version, doc.slug,
+                   bm25(examples_fts) as score,
+                   snippet(examples_fts, 0, '**', '**', '...', 32) as snippet_text
+            FROM examples_fts
+            JOIN examples e ON examples_fts.rowid = e.id
+            JOIN sections s ON e.section_id = s.id
+            JOIN documents doc ON s.document_id = doc.id
+            JOIN doc_sets d ON doc.doc_set_id = d.id
+            WHERE examples_fts MATCH ?
+              AND (? IS NULL OR d.version = ?)
+            ORDER BY bm25(examples_fts)
+            LIMIT ?
+            """,
+            (match_expr, version, version, max_results),
+        )
+        rows = cursor.fetchall()
+    except sqlite3.OperationalError:
+        logger.warning("FTS5 query failed for examples: %r", match_expr)
+        return []
+
+    hits = [
+        SymbolHit(
+            uri=row["section_uri"],
+            title=row["heading"],
+            kind="doctest" if row["is_doctest"] else "example",
+            snippet=row["snippet_text"] or "",
+            score=row["score"],
+            version=row["version"],
+            slug=row["slug"],
+            anchor=row["anchor"] or None,
+        )
+        for row in rows
+    ]
+
+    return _normalize_scores(hits)
+
+
+def lookup_symbols_exact(
+    conn: sqlite3.Connection,
+    query: str,
+    version: str | None,
+    max_results: int,
+) -> list[SymbolHit]:
+    """Direct symbol table lookup for the symbol fast-path (RETR-04).
+
+    Bypasses FTS5 entirely. Uses exact match first, then LIKE prefix
+    match. Scores: exact match = 1.0, prefix match = 0.8.
+
+    Args:
+        conn: Read-only SQLite connection.
+        query: Symbol name to look up.
+        version: Python version filter, or None for all versions.
+        max_results: Maximum number of hits.
+
+    Returns:
+        List of SymbolHit with kind from symbol_type.
+    """
+    # Escape LIKE wildcards in user input
+    escaped_query = (
+        query.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
+    )
+
+    cursor = conn.execute(
+        """
+        SELECT s.doc_set_id, s.document_id, s.section_id,
+               s.qualified_name, s.symbol_type, s.uri, s.anchor,
+               s.module, d.version
+        FROM symbols s
+        JOIN doc_sets d ON s.doc_set_id = d.id
+        WHERE (s.qualified_name = ? OR s.qualified_name LIKE ? ESCAPE '\\')
+          AND (? IS NULL OR d.version = ?)
+        ORDER BY CASE WHEN s.qualified_name = ? THEN 0 ELSE 1 END
+        LIMIT ?
+        """,
+        (query, f"%{escaped_query}%", version, version, query, max_results),
+    )
+    rows = cursor.fetchall()
+
+    hits = []
+    for row in rows:
+        slug, anchor = _resolve_symbol_location(conn, row)
+        hits.append(SymbolHit(
+            uri=row["uri"],
+            title=row["qualified_name"],
+            kind=row["symbol_type"] or "symbol",
+            snippet="",
+            score=1.0 if row["qualified_name"] == query else 0.8,
+            version=row["version"],
+            slug=slug,
+            anchor=anchor,
+        ))
+    return hits

mcp_server_python_docs/server.py

@@ -0,0 +1,329 @@
+"""FastMCP server with lifespan DI and tool registration.
+
+Thin server layer — delegates all tool logic to services.
+Dependency rule: server -> services -> retrieval/storage.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import importlib.resources
+import logging
+import sqlite3
+import sys
+import traceback
+from collections.abc import AsyncIterator
+from contextlib import asynccontextmanager
+from typing import Annotated, Literal
+
+import yaml
+from mcp.server.fastmcp import Context, FastMCP
+from mcp.server.fastmcp.exceptions import ToolError
+from mcp.types import ToolAnnotations
+from pydantic import Field
+
+from mcp_server_python_docs.app_context import AppContext
+from mcp_server_python_docs.detection import detect_python_version, match_to_indexed
+from mcp_server_python_docs.errors import DocsServerError
+from mcp_server_python_docs.models import (
+    DetectPythonVersionResult,
+    GetDocsResult,
+    ListVersionsResult,
+    PackageDocsResult,
+    SearchDocsResult,
+)
+from mcp_server_python_docs.services.content import ContentService
+from mcp_server_python_docs.services.package_docs import PackageDocsService
+from mcp_server_python_docs.services.persistent_cache import PersistentDocsCache
+from mcp_server_python_docs.services.search import SearchService
+from mcp_server_python_docs.services.version import VersionService
+from mcp_server_python_docs.storage.db import (
+    get_cache_dir,
+    get_index_path,
+    get_readonly_connection,
+)
+
+logger = logging.getLogger(__name__)
+
+
+def _load_synonyms() -> dict[str, list[str]]:
+    """Load synonyms.yaml from package data via importlib.resources (SRVR-12)."""
+    ref = importlib.resources.files("mcp_server_python_docs") / "data" / "synonyms.yaml"
+    with importlib.resources.as_file(ref) as path:
+        data = yaml.safe_load(path.read_text())
+    return {k: v for k, v in data.items() if isinstance(v, list)}
+
+
+def _assert_fts5(conn: sqlite3.Connection) -> None:
+    """Check FTS5 availability with platform-aware error (STOR-08)."""
+    from mcp_server_python_docs.storage.db import assert_fts5_available
+
+    assert_fts5_available(conn)
+
+
+@asynccontextmanager
+async def app_lifespan(server: FastMCP) -> AsyncIterator[AppContext]:
+    """Manage application lifecycle with typed context (SRVR-01).
+
+    Loads synonyms eagerly (SRVR-11), opens read-only DB handle (STOR-06),
+    constructs service instances, and fails fast on missing index or
+    unavailable FTS5.
+    """
+    cache_dir = get_cache_dir()
+    index_path = get_index_path()
+
+    # Fail fast on missing index (SRVR-10)
+    if not index_path.exists():
+        from mcp_server_python_docs.ingestion.cpython_versions import (
+            SUPPORTED_DOC_VERSIONS_CSV,
+        )
+
+        msg = (
+            f"No index found at {index_path}\n"
+            f"Run: python-docs-mcp-server build-index --versions "
+            f"{SUPPORTED_DOC_VERSIONS_CSV}"
+        )
+        logger.error(msg)
+        print(msg, file=sys.stderr)
+        raise SystemExit(1)
+
+    # Load synonyms from package data (SRVR-11, SRVR-12)
+    synonyms = _load_synonyms()
+    logger.info("Loaded %d synonym entries", len(synonyms))
+
+    # Open read-only connection (STOR-06, STOR-07)
+    db = get_readonly_connection(index_path)
+    persistent_docs_cache: PersistentDocsCache | None = None
+
+    try:
+        # Check FTS5 (STOR-08)
+        _assert_fts5(db)
+
+        # Construct service instances (Phase 5 — service layer wiring)
+        persistent_docs_cache = PersistentDocsCache(
+            cache_path=cache_dir / "retrieved-docs-cache.sqlite3",
+            index_path=index_path,
+        )
+        search_svc = SearchService(db, synonyms)
+        content_svc = ContentService(db, persistent_cache=persistent_docs_cache)
+        version_svc = VersionService(db)
+        package_docs_svc = PackageDocsService()
+
+        # Detect user's Python version and match to indexed versions
+        detected_ver, detected_src = detect_python_version()
+        indexed_versions = [
+            r[0] for r in db.execute("SELECT version FROM doc_sets ORDER BY version").fetchall()
+        ]
+        matched = match_to_indexed(detected_ver, indexed_versions)
+        if matched:
+            logger.info("User Python %s matches indexed version — using as default", matched)
+        else:
+            logger.info(
+                "User Python %s not in index %s — using normal default",
+                detected_ver,
+                indexed_versions,
+            )
+
+        try:
+            yield AppContext(
+                db=db,
+                index_path=index_path,
+                synonyms=synonyms,
+                search_service=search_svc,
+                content_service=content_svc,
+                version_service=version_svc,
+                package_docs_service=package_docs_svc,
+                persistent_docs_cache=persistent_docs_cache,
+                detected_python_version=matched,
+                detected_python_source=detected_src,
+            )
+        except Exception:
+            # HYGN-05: log lifespan errors, write last-error.log, re-raise original
+            error_msg = traceback.format_exc()
+            logger.error("Lifespan error: %s", error_msg)
+            try:
+                error_log = cache_dir / "last-error.log"
+                error_log.write_text(error_msg)
+            except Exception:
+                pass
+            raise
+    finally:
+        if persistent_docs_cache is not None:
+            try:
+                persistent_docs_cache.close()
+            except Exception:
+                pass
+        db.close()
+
+
+# Shared tool annotations — all tools are read-only (SRVR-02, SRVR-03, SRVR-04)
+_TOOL_ANNOTATIONS = ToolAnnotations(
+    readOnlyHint=True,
+    destructiveHint=False,
+    idempotentHint=True,
+    openWorldHint=False,
+)
+_PYPI_TOOL_ANNOTATIONS = ToolAnnotations(
+    readOnlyHint=True,
+    destructiveHint=False,
+    idempotentHint=True,
+    openWorldHint=True,
+)
+
+SearchQueryParam = Annotated[
+    str,
+    Field(
+        max_length=500,
+        description="Search query - Python symbol (asyncio.TaskGroup) or concept (parse json)",
+    ),
+]
+VersionParam = Annotated[
+    str | None,
+    Field(description="Python version (e.g. '3.13'). Defaults to latest."),
+]
+SearchKindParam = Annotated[
+    Literal["auto", "page", "symbol", "section", "example"],
+    Field(
+        description=(
+            "Search type. Use 'symbol' for API lookups, "
+            "'example' for code samples, 'auto' otherwise."
+        )
+    ),
+]
+MaxResultsParam = Annotated[
+    int,
+    Field(ge=1, le=20, description="Maximum number of results to return."),
+]
+SlugParam = Annotated[
+    str,
+    Field(max_length=500, description="Page slug (e.g. 'library/asyncio-task.html')"),
+]
+AnchorParam = Annotated[
+    str | None,
+    Field(description="Section anchor for section-only retrieval"),
+]
+MaxCharsParam = Annotated[
+    int,
+    Field(ge=100, le=50000, description="Maximum characters to return"),
+]
+StartIndexParam = Annotated[
+    int,
+    Field(ge=0, description="Start position for pagination"),
+]
+PackageParam = Annotated[
+    str,
+    Field(min_length=1, max_length=214, description="PyPI package/project name"),
+]
+
+
+def create_server() -> FastMCP:
+    """Create and configure the FastMCP server."""
+    mcp = FastMCP(
+        "python-docs-mcp-server",
+        lifespan=app_lifespan,
+    )
+
+    @mcp.tool(annotations=_TOOL_ANNOTATIONS)
+    def search_docs(
+        query: SearchQueryParam,
+        version: VersionParam = None,
+        kind: SearchKindParam = "auto",
+        max_results: MaxResultsParam = 5,
+        ctx: Context = None,  # type: ignore[assignment]
+    ) -> SearchDocsResult:
+        """Search Python documentation. Use kind='symbol' for API lookups
+        (asyncio.TaskGroup), kind='example' for code samples, kind='auto' otherwise.
+        When version is omitted, searches across all versions. Pass the version
+        from each hit's version field to get_docs for consistent results."""
+        app_ctx: AppContext = ctx.request_context.lifespan_context
+        try:
+            return app_ctx.search_service.search(query, version, kind, max_results)
+        except DocsServerError as e:
+            raise ToolError(str(e))
+        except Exception as e:
+            logger.exception("Unexpected error in search_docs")
+            raise ToolError(f"Internal error: {type(e).__name__}")
+
+    @mcp.tool(annotations=_TOOL_ANNOTATIONS)
+    def get_docs(
+        slug: SlugParam,
+        version: VersionParam = None,
+        anchor: AnchorParam = None,
+        max_chars: MaxCharsParam = 8000,
+        start_index: StartIndexParam = 0,
+        ctx: Context = None,  # type: ignore[assignment]
+    ) -> GetDocsResult:
+        """Retrieve a documentation page or specific section. Provide anchor for
+        section-only retrieval (much cheaper). Pagination via start_index."""
+        app_ctx: AppContext = ctx.request_context.lifespan_context
+        # Auto-default to detected Python version when no version specified
+        if version is None and app_ctx.detected_python_version:
+            version = app_ctx.detected_python_version
+        try:
+            return app_ctx.content_service.get_docs(slug, version, anchor, max_chars, start_index)
+        except DocsServerError as e:
+            raise ToolError(str(e))
+        except Exception as e:
+            logger.exception("Unexpected error in get_docs")
+            raise ToolError(f"Internal error: {type(e).__name__}")
+
+    @mcp.tool(annotations=_PYPI_TOOL_ANNOTATIONS)
+    async def lookup_package_docs(
+        package: PackageParam,
+        ctx: Context = None,  # type: ignore[assignment]
+    ) -> PackageDocsResult:
+        """Look up package-declared docs/homepage/source URLs via official PyPI metadata.
+
+        This is not generic web search: it only queries PyPI's JSON API and
+        returns official PyPI metadata plus package-declared project URLs.
+        """
+        app_ctx: AppContext = ctx.request_context.lifespan_context
+        try:
+            return await asyncio.to_thread(app_ctx.package_docs_service.lookup, package)
+        except Exception as e:
+            logger.exception("Unexpected error in lookup_package_docs")
+            raise ToolError(f"Internal error: {type(e).__name__}")
+
+    @mcp.tool(annotations=_TOOL_ANNOTATIONS)
+    def list_versions(
+        ctx: Context = None,  # type: ignore[assignment]
+    ) -> ListVersionsResult:
+        """List Python documentation versions available in this index."""
+        app_ctx: AppContext = ctx.request_context.lifespan_context
+        try:
+            return app_ctx.version_service.list_versions()
+        except DocsServerError as e:
+            raise ToolError(str(e))
+        except Exception as e:
+            logger.exception("Unexpected error in list_versions")
+            raise ToolError(f"Internal error: {type(e).__name__}")
+
+    @mcp.tool(annotations=_TOOL_ANNOTATIONS)
+    def detect_python_version(
+        ctx: Context = None,  # type: ignore[assignment]
+    ) -> DetectPythonVersionResult:
+        """Detect the Python version in the user's environment.
+        Returns the detected version, how it was found, and whether it
+        matches an indexed documentation set."""
+        app_ctx: AppContext = ctx.request_context.lifespan_context
+        detected_ver = app_ctx.detected_python_version
+
+        # Re-run detection to get the raw version even if it didn't match
+        from mcp_server_python_docs.detection import detect_python_version as _detect
+
+        raw_ver, raw_src = _detect()
+
+        return DetectPythonVersionResult(
+            detected_version=raw_ver,
+            source=raw_src,
+            matched_index_version=detected_ver,
+            is_default=detected_ver is not None,
+        )
+
+    # SRVR-07: _meta hint for get_docs tool.
+    # FastMCP 1.27 does not expose a public API for setting _meta on tool
+    # definitions. Deferred until the mcp SDK adds _meta support to the
+    # decorator API or tool manager. The hint is advisory — clients work
+    # correctly without it.
+
+    return mcp

mcp_server_python_docs/services/__init__.py

@@ -0,0 +1,7 @@
+"""Service layer — SearchService, ContentService, VersionService.
+
+Services sit between FastMCP tool handlers and the retrieval/storage layers.
+Dependency rule: server -> services -> retrieval/storage.
+Services receive sqlite3.Connection via constructor and execute queries directly.
+No service imports MCP types.
+"""