lean-explore 1.1.1__py3-none-any.whl → 1.2.0__py3-none-any.whl

lean_explore/extract/doc_gen4.py

@@ -8,6 +8,7 @@ import logging
 import os
 import shutil
 import subprocess
+import time
 from pathlib import Path
 
 from lean_explore.extract.github import extract_lean_version
@@ -80,6 +81,56 @@ def _setup_workspace(package_config: PackageConfig) -> tuple[str, str]:
     return lean_toolchain, git_ref
 
 
+def _run_lake_update_with_retry(
+    workspace_path: Path,
+    package_name: str,
+    env: dict[str, str],
+    verbose: bool = False,
+    max_retries: int = 3,
+    base_delay: float = 30.0,
+) -> None:
+    """Run ``lake update`` with retries for transient network failures.
+
+    Large repositories like mathlib4 require cloning several gigabytes of git
+    data. Transient network issues (DNS blips, connection resets, GitHub
+    throttling) can cause the clone to fail with git exit code 128. Retrying
+    with exponential backoff handles these cases.
+
+    Args:
+        workspace_path: Path to the Lake workspace directory.
+        package_name: Name of the package (for log messages).
+        env: Environment variables to pass to the subprocess.
+        verbose: Log stdout from ``lake update``.
+        max_retries: Maximum number of retry attempts after the initial try.
+        base_delay: Seconds to wait before the first retry. Doubles each retry.
+    """
+    for attempt in range(1, max_retries + 2):
+        logger.info(f"[{package_name}] Running lake update (attempt {attempt})...")
+        result = subprocess.run(
+            ["lake", "update"],
+            cwd=workspace_path,
+            capture_output=True,
+            text=True,
+            env=env,
+        )
+        if verbose and result.stdout:
+            logger.info(result.stdout)
+        if result.returncode == 0:
+            return
+
+        if attempt <= max_retries:
+            delay = base_delay * (2 ** (attempt - 1))
+            logger.warning(
+                f"[{package_name}] lake update failed (attempt {attempt}), "
+                f"retrying in {delay:.0f}s..."
+            )
+            logger.warning(f"[{package_name}] stderr: {result.stderr.strip()}")
+            time.sleep(delay)
+        else:
+            logger.error(result.stderr)
+            raise RuntimeError(f"lake update failed for {package_name}")
+
+
 def _run_lake_for_package(package_name: str, verbose: bool = False) -> None:
     """Run lake update, cache get, and doc-gen4 for a package."""
     workspace_path = Path("lean") / package_name
@@ -87,19 +138,7 @@ def _run_lake_for_package(package_name: str, verbose: bool = False) -> None:
     env = os.environ.copy()
     env["MATHLIB_NO_CACHE_ON_UPDATE"] = "1"
 
-    logger.info(f"[{package_name}] Running lake update...")
-    result = subprocess.run(
-        ["lake", "update"],
-        cwd=workspace_path,
-        capture_output=True,
-        text=True,
-        env=env,
-    )
-    if verbose and result.stdout:
-        logger.info(result.stdout)
-    if result.returncode != 0:
-        logger.error(result.stderr)
-        raise RuntimeError(f"lake update failed for {package_name}")
+    _run_lake_update_with_retry(workspace_path, package_name, env, verbose)
 
     # Fetch mathlib cache for packages that depend on mathlib
     if "mathlib" in package_config.depends_on or package_name == "mathlib":

lean_explore/mcp/app.py

@@ -69,7 +69,24 @@ async def app_lifespan(server: FastMCP) -> AsyncIterator[AppContext]:
 mcp_app = FastMCP(
     name="LeanExploreMCPServer",
     instructions=(
-        "MCP Server for Lean Explore, providing tools to search Lean declarations."
+        "MCP Server for searching Lean 4 mathematical declarations (theorems, "
+        "definitions, lemmas, instances, etc.) from Mathlib and other Lean "
+        "packages.\n\n"
+        "The search engine is hybrid: it matches by declaration name (e.g., "
+        "'List.map', 'Nat.add') AND by informal natural language meaning (e.g., "
+        "'a continuous function on a compact set', 'prime number divisibility'). "
+        "You can use either style of query.\n\n"
+        "Recommended workflow:\n"
+        "1. Use search_summary to browse results (low token cost).\n"
+        "2. Use per-field tools to fetch only what you need:\n"
+        "   - get_source_code: Lean source code\n"
+        "   - get_source_link: GitHub link to source\n"
+        "   - get_docstring: documentation string\n"
+        "   - get_description: natural language description\n"
+        "   - get_module: module path in the package\n"
+        "   - get_dependencies: declarations this depends on\n"
+        "3. Use search only when you need full details for all results "
+        "at once."
     ),
     lifespan=app_lifespan,
 )

lean_explore/mcp/tools.py

@@ -54,6 +54,54 @@ class SearchResponseDict(TypedDict, total=False):
     processing_time_ms: int | None
 
 
+class SourceCodeResultDict(TypedDict):
+    """Result containing declaration id, name, and source code."""
+
+    id: int
+    name: str
+    source_text: str
+
+
+class SourceLinkResultDict(TypedDict):
+    """Result containing declaration id, name, and GitHub source link."""
+
+    id: int
+    name: str
+    source_link: str
+
+
+class DocstringResultDict(TypedDict):
+    """Result containing declaration id, name, and docstring."""
+
+    id: int
+    name: str
+    docstring: str | None
+
+
+class DescriptionResultDict(TypedDict):
+    """Result containing declaration id, name, and informalization."""
+
+    id: int
+    name: str
+    informalization: str | None
+
+
+class ModuleResultDict(TypedDict):
+    """Result containing declaration id, name, and module path."""
+
+    id: int
+    name: str
+    module: str
+
+
+class DependenciesResultDict(TypedDict):
+    """Result containing declaration id, name, and dependencies."""
+
+    id: int
+    name: str
+    dependencies: str | None
+
+
 logger = logging.getLogger(__name__)
 
 
@@ -112,6 +160,33 @@ async def _execute_backend_search(
     )
 
 
+async def _execute_backend_get_by_id(
+    backend: BackendServiceType,
+    declaration_id: int,
+) -> SearchResult | None:
+    """Execute get_by_id on the backend, handling both async and sync backends.
+
+    Args:
+        backend: The backend service (ApiClient or Service).
+        declaration_id: The numeric id of the declaration to retrieve.
+
+    Returns:
+        The SearchResult from the backend, or None if not found.
+
+    Raises:
+        RuntimeError: If the backend does not support get_by_id.
+    """
+    if not hasattr(backend, "get_by_id"):
+        logger.error("Backend service does not have a 'get_by_id' method.")
+        raise RuntimeError(
+            "Get by ID functionality not available on configured backend."
+        )
+
+    if asyncio.iscoroutinefunction(backend.get_by_id):
+        return await backend.get_by_id(declaration_id=declaration_id)
+    return backend.get_by_id(declaration_id=declaration_id)
+
+
 @mcp_app.tool()
 async def search(
     ctx: MCPContext,
@@ -120,11 +195,29 @@ async def search(
     rerank_top: int | None = 50,
     packages: list[str] | None = None,
 ) -> SearchResponseDict:
-    """Searches Lean declarations by a query string.
+    """Search Lean 4 declarations and return full results including source code.
+
+    Accepts two kinds of queries:
+      - By name: a full or partial Lean declaration name, e.g., "List.map",
+        "Nat.Prime", "CategoryTheory.Functor.map".
+      - By meaning: an informal natural language description, e.g.,
+        "continuous function on a compact set", "sum of a geometric series",
+        "a group homomorphism preserving multiplication".
+
+    The search engine handles both styles simultaneously via hybrid retrieval
+    (lexical name matching + semantic similarity), so you do not need to
+    specify which kind of query you are making.
+
+    Returns full results including source code, module, dependencies, and
+    informalization for every hit. If you only need names and short
+    descriptions, prefer search_summary to save tokens, then use the
+    per-field tools (get_source_code, get_docstring, get_description,
+    get_module, get_dependencies) for the entries you care about.
 
     Args:
         ctx: The MCP context, providing access to the backend service.
-        query: A search query string, e.g., "continuous function".
+        query: A Lean declaration name (e.g., "List.filter") or an informal
+            natural language description (e.g., "prime number divisibility").
         limit: The maximum number of search results to return. Defaults to 10.
         rerank_top: Number of candidates to rerank with cross-encoder. Set to 0 or
             None to skip reranking. Defaults to 50. Only used with local backend.
@@ -155,15 +248,29 @@ async def search_summary(
     rerank_top: int | None = 50,
     packages: list[str] | None = None,
 ) -> SearchSummaryResponseDict:
-    """Searches Lean declarations and returns concise results.
+    """Search Lean 4 declarations and return concise results (recommended first step).
 
-    Returns slim results (id, name, short description) to minimize token usage.
-    Use get_by_id to retrieve full details for specific declarations, or
-    search to get all fields upfront.
+    This is the preferred starting point for search. Returns only id, name,
+    and a short natural language description for each hit, keeping token
+    usage low. After reviewing these summaries, use the per-field tools
+    (get_source_code, get_docstring, get_description, get_module,
+    get_dependencies) for the entries you need details on.
+
+    Accepts two kinds of queries:
+      - By name: a full or partial Lean declaration name, e.g., "List.map",
+        "Nat.Prime", "CategoryTheory.Functor.map".
+      - By meaning: an informal natural language description, e.g.,
+        "continuous function on a compact set", "sum of a geometric series",
+        "a group homomorphism preserving multiplication".
+
+    The search engine handles both styles simultaneously via hybrid retrieval
+    (lexical name matching + semantic similarity), so you do not need to
+    specify which kind of query you are making.
 
     Args:
         ctx: The MCP context, providing access to the backend service.
-        query: A search query string, e.g., "continuous function".
+        query: A Lean declaration name (e.g., "List.filter") or an informal
+            natural language description (e.g., "prime number divisibility").
         limit: The maximum number of search results to return. Defaults to 10.
         rerank_top: Number of candidates to rerank with cross-encoder. Set to 0 or
             None to skip reranking. Defaults to 50. Only used with local backend.
@@ -203,38 +310,219 @@ async def search_summary(
 
 
 @mcp_app.tool()
-async def get_by_id(
+async def get_source_code(
     ctx: MCPContext,
     declaration_id: int,
-) -> SearchResultDict | None:
-    """Retrieves a specific declaration by its unique identifier.
+) -> SourceCodeResultDict | None:
+    """Retrieve the Lean source code for a declaration by id.
+
+    Returns the declaration name and its Lean 4 source code. Use this after
+    calling search_summary to inspect the actual implementation.
 
-    Returns the full declaration including source code, dependencies, module
-    info, and informalization. Use this to expand results from the search tool.
+    The id values come from the search or search_summary result lists.
 
     Args:
         ctx: The MCP context, providing access to the backend service.
-        declaration_id: The unique integer identifier of the declaration.
+        declaration_id: The numeric id from a search or search_summary result.
 
     Returns:
-        A dictionary representing the SearchResult, or None if not found.
+        A dictionary with id, name, and source_text, or None if the id
+        does not exist.
     """
     backend = await _get_backend_from_context(ctx)
-    logger.info(f"MCP Tool 'get_by_id' called for declaration_id: {declaration_id}")
+    logger.info(
+        f"MCP Tool 'get_source_code' called for declaration_id: {declaration_id}"
+    )
 
-    if not hasattr(backend, "get_by_id"):
-        logger.error("Backend service does not have a 'get_by_id' method.")
-        raise RuntimeError(
-            "Get by ID functionality not available on configured backend."
-        )
+    result = await _execute_backend_get_by_id(backend, declaration_id)
+    if result is None:
+        return None
 
-    # Call backend get_by_id (handle both async and sync)
-    if asyncio.iscoroutinefunction(backend.get_by_id):
-        result: SearchResult | None = await backend.get_by_id(
-            declaration_id=declaration_id
-        )
-    else:
-        result: SearchResult | None = backend.get_by_id(declaration_id=declaration_id)
+    return SourceCodeResultDict(
+        id=result.id,
+        name=result.name,
+        source_text=result.source_text,
+    )
+
+
+@mcp_app.tool()
+async def get_source_link(
+    ctx: MCPContext,
+    declaration_id: int,
+) -> SourceLinkResultDict | None:
+    """Retrieve the GitHub source link for a declaration by id.
+
+    Returns the declaration name and a URL to the source code on GitHub.
+    Use this when you need to reference or link to the original source.
+
+    The id values come from the search or search_summary result lists.
 
-    # Return as dict for MCP, or None
-    return result.model_dump(exclude_none=True) if result else None
+    Args:
+        ctx: The MCP context, providing access to the backend service.
+        declaration_id: The numeric id from a search or search_summary result.
+
+    Returns:
+        A dictionary with id, name, and source_link, or None if the id
+        does not exist.
+    """
+    backend = await _get_backend_from_context(ctx)
+    logger.info(
+        f"MCP Tool 'get_source_link' called for declaration_id: {declaration_id}"
+    )
+
+    result = await _execute_backend_get_by_id(backend, declaration_id)
+    if result is None:
+        return None
+
+    return SourceLinkResultDict(
+        id=result.id,
+        name=result.name,
+        source_link=result.source_link,
+    )
+
+
+@mcp_app.tool()
+async def get_docstring(
+    ctx: MCPContext,
+    declaration_id: int,
+) -> DocstringResultDict | None:
+    """Retrieve the docstring for a declaration by id.
+
+    Returns the declaration name and its documentation string from the Lean
+    source code. Use this to check what documentation exists without
+    fetching the full source code.
+
+    The id values come from the search or search_summary result lists.
+
+    Args:
+        ctx: The MCP context, providing access to the backend service.
+        declaration_id: The numeric id from a search or search_summary result.
+
+    Returns:
+        A dictionary with id, name, and docstring, or None if the id
+        does not exist.
+    """
+    backend = await _get_backend_from_context(ctx)
+    logger.info(
+        f"MCP Tool 'get_docstring' called for declaration_id: {declaration_id}"
+    )
+
+    result = await _execute_backend_get_by_id(backend, declaration_id)
+    if result is None:
+        return None
+
+    return DocstringResultDict(
+        id=result.id,
+        name=result.name,
+        docstring=result.docstring,
+    )
+
+
+@mcp_app.tool()
+async def get_description(
+    ctx: MCPContext,
+    declaration_id: int,
+) -> DescriptionResultDict | None:
+    """Retrieve the natural language description for a declaration by id.
+
+    Returns the declaration name and its informalization, an AI-generated
+    plain-English explanation of what the declaration states or does.
+
+    The id values come from the search or search_summary result lists.
+
+    Args:
+        ctx: The MCP context, providing access to the backend service.
+        declaration_id: The numeric id from a search or search_summary result.
+
+    Returns:
+        A dictionary with id, name, and informalization, or None if the id
+        does not exist.
+    """
+    backend = await _get_backend_from_context(ctx)
+    logger.info(
+        f"MCP Tool 'get_description' called for declaration_id: {declaration_id}"
+    )
+
+    result = await _execute_backend_get_by_id(backend, declaration_id)
+    if result is None:
+        return None
+
+    return DescriptionResultDict(
+        id=result.id,
+        name=result.name,
+        informalization=result.informalization,
+    )
+
+
+@mcp_app.tool()
+async def get_module(
+    ctx: MCPContext,
+    declaration_id: int,
+) -> ModuleResultDict | None:
+    """Retrieve the module path for a declaration by id.
+
+    Returns the declaration name and the Lean module it belongs to
+    (e.g., 'Mathlib.Data.List.Basic'). Use this to find where a
+    declaration lives in the package structure.
+
+    The id values come from the search or search_summary result lists.
+
+    Args:
+        ctx: The MCP context, providing access to the backend service.
+        declaration_id: The numeric id from a search or search_summary result.
+
+    Returns:
+        A dictionary with id, name, and module, or None if the id does
+        not exist.
+    """
+    backend = await _get_backend_from_context(ctx)
+    logger.info(
+        f"MCP Tool 'get_module' called for declaration_id: {declaration_id}"
+    )
+
+    result = await _execute_backend_get_by_id(backend, declaration_id)
+    if result is None:
+        return None
+
+    return ModuleResultDict(
+        id=result.id,
+        name=result.name,
+        module=result.module,
+    )
+
+
+@mcp_app.tool()
+async def get_dependencies(
+    ctx: MCPContext,
+    declaration_id: int,
+) -> DependenciesResultDict | None:
+    """Retrieve the dependencies for a declaration by id.
+
+    Returns the declaration name and a JSON array of other declaration
+    names that this declaration depends on. Use this to understand what
+    a declaration builds upon.
+
+    The id values come from the search or search_summary result lists.
+
+    Args:
+        ctx: The MCP context, providing access to the backend service.
+        declaration_id: The numeric id from a search or search_summary result.
+
+    Returns:
+        A dictionary with id, name, and dependencies, or None if the id
+        does not exist.
+    """
+    backend = await _get_backend_from_context(ctx)
+    logger.info(
+        f"MCP Tool 'get_dependencies' called for declaration_id: {declaration_id}"
+    )
+
+    result = await _execute_backend_get_by_id(backend, declaration_id)
+    if result is None:
+        return None
+
+    return DependenciesResultDict(
+        id=result.id,
+        name=result.name,
+        dependencies=result.dependencies,
+    )

lean_explore/models/search_types.py

@@ -28,7 +28,8 @@ class SearchResultSummary(BaseModel):
     """A slim search result containing only identification and description.
 
     Used by the MCP search tool to return concise results that minimize
-    token usage. Consumers can use the id to fetch full details via get_by_id.
+    token usage. Consumers can use the id to fetch specific fields via the
+    per-field tools (get_source_code, get_docstring, etc.).
     """
 
     id: int

{lean_explore-1.1.1.dist-info → lean_explore-1.2.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lean-explore
-Version: 1.1.1
+Version: 1.2.0
 Summary: A search engine for Lean 4 declarations.
 Author-email: Justin Asher <justinchadwickasher@gmail.com>
 License:                                  Apache License

{lean_explore-1.1.1.dist-info → lean_explore-1.2.0.dist-info}/RECORD

@@ -8,7 +8,7 @@ lean_explore/cli/display.py,sha256=t7qfP8Cdw05KM-kfYmFEeWSHNzyYuBOcvx4aCb5lFv0,5
 lean_explore/cli/main.py,sha256=feVxTmzH0FMZGP7Urb6X43atlESJIYXgC6DJylgiKp0,3937
 lean_explore/extract/__init__.py,sha256=ZpCPPHjFCRxyHwYxQNhGpbpd_XuZOkgB9qkxa9zSUD0,184
 lean_explore/extract/__main__.py,sha256=Aquy2g-WIU7YO39UZRq0EoHgJWW-uM5G0FSK2OwDuR8,12010
-lean_explore/extract/doc_gen4.py,sha256=5pvw-EdfDxg1NPxLpa6kQuoVUe1_TtKL1_hO13I0t78,6734
+lean_explore/extract/doc_gen4.py,sha256=FG-P4S6dGU55IRhJyYtEp3XUIMhKMad6_IZpAA5hWwM,8305
 lean_explore/extract/doc_parser.py,sha256=tb3y7409mn5DhPjR2prPh2Cm7QMf3VJy8qfir_-Y4z8,17511
 lean_explore/extract/embeddings.py,sha256=atZzaM2wZBSZflL2sTbStGUSd1WKjyex218fMxG_CWc,11875
 lean_explore/extract/github.py,sha256=1Zyyl2u1esULJ3KFHHL8F80zcxLDx1Fh8u_gxCmBTi8,3499
@@ -19,12 +19,12 @@ lean_explore/extract/package_registry.py,sha256=N-M3WYry77P80pjt12lpFcCuPW1M9Sgx
 lean_explore/extract/package_utils.py,sha256=SBD4gbCA5enldm2m3AoL4Dti1FsSs-ShQYeYMjyOADg,3400
 lean_explore/extract/types.py,sha256=Sp6sYuioTE_Gs0Z0lbq4h7OMyAnaZafdLV8UGtKn-zs,583
 lean_explore/mcp/__init__.py,sha256=YO0RM466ik2jQk8YMDITzkm3AHPtjEhn7Wm7rOusUXo,462
-lean_explore/mcp/app.py,sha256=PJ2HwX6VyTqKejuI1G8Ld4aO9XWp9hT5H8loaA5g0Lc,2173
+lean_explore/mcp/app.py,sha256=7Se0sqvyzigxwkqti19leRWNEe4CAK1VrCW2Br6TZbw,3165
 lean_explore/mcp/server.py,sha256=Lf3SCn8ghPNkZ3BybHh3VCXn91F-yX6RSRke1rvC7Pk,8234
-lean_explore/mcp/tools.py,sha256=3MLeePzHS9mMkr97Lx5JM2YxF4QiHBWYZvAh6yBh-9g,7837
+lean_explore/mcp/tools.py,sha256=WlH73HOt81a6nYGOK_wkuQ_d2x739TWik9pZon2ow9w,16757
 lean_explore/models/__init__.py,sha256=k4tDULTDIp61iLuP-CBXLJzjCaEBgNVgO_x9JpdrjgY,523
 lean_explore/models/search_db.py,sha256=_a6B6FpqUevyHvW4KmNLeziiznIuxftpMUy0AtSDBJE,2673
-lean_explore/models/search_types.py,sha256=3r0eql--zoTGRtwosbvZpTMK7tdBdPSfWPBFTU_pupE,3001
+lean_explore/models/search_types.py,sha256=UgwB-J5nN30PpVdKmlSe1ewjJ3DsmeYtDR4Er8AAmUc,3057
 lean_explore/search/__init__.py,sha256=0k_iHe5xrurepznk7NzMYz10QFbK10ydMlpFlsuyFSc,1216
 lean_explore/search/engine.py,sha256=oAsqiltBEXsbur0t77-zG8ATgcZ-8vKX8vvdoPNZsv0,23660
 lean_explore/search/scoring.py,sha256=VkH-kpGheX14_tf8uJYBOp0nrG05_JJLmv7_0QdfAQk,4168
@@ -35,9 +35,9 @@ lean_explore/util/embedding_client.py,sha256=6XJGJrGTXAiefDr-E1j_SPHTTZMIJYi62Pw
 lean_explore/util/logging.py,sha256=hF8YPi-1I6DdC1B_yROXA6u5GG14IIhD0Nym2FfqgRA,649
 lean_explore/util/openrouter_client.py,sha256=C_0HLO5o1seYjGl2zn6897i2onK7CdI6XxtE3cWb3Os,1926
 lean_explore/util/reranker_client.py,sha256=kLCTGPMQuphjwAj0PPi9KXpSzDP7o9JRQJpTbmWGiMs,6074
-lean_explore-1.1.1.dist-info/licenses/LICENSE,sha256=l4QLw1kIvEOjUktmmKm4dycK1E249Qs2s2AQTYbMXpY,11354
-lean_explore-1.1.1.dist-info/METADATA,sha256=q0RKNHMMwkLvPzToLOuUbNoumHaCILKaecT6ypKGmnQ,17084
-lean_explore-1.1.1.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
-lean_explore-1.1.1.dist-info/entry_points.txt,sha256=FuKSRE7GmI9B_kM-xoiWEJj2dQ4upqhHnw8qH1vcjW8,59
-lean_explore-1.1.1.dist-info/top_level.txt,sha256=h51BKWrFvB7iym-IlaNAAHX5MZfA8Gmg-aDuXGo0fQ8,13
-lean_explore-1.1.1.dist-info/RECORD,,
+lean_explore-1.2.0.dist-info/licenses/LICENSE,sha256=l4QLw1kIvEOjUktmmKm4dycK1E249Qs2s2AQTYbMXpY,11354
+lean_explore-1.2.0.dist-info/METADATA,sha256=yKIU62Habl2ntFl_oloPMLJ2bNJwtFA6nfotyO6Leyk,17084
+lean_explore-1.2.0.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+lean_explore-1.2.0.dist-info/entry_points.txt,sha256=FuKSRE7GmI9B_kM-xoiWEJj2dQ4upqhHnw8qH1vcjW8,59
+lean_explore-1.2.0.dist-info/top_level.txt,sha256=h51BKWrFvB7iym-IlaNAAHX5MZfA8Gmg-aDuXGo0fQ8,13
+lean_explore-1.2.0.dist-info/RECORD,,