lean-explore 0.3.0__tar.gz → 1.0.0__tar.gz

{lean_explore-0.3.0 → lean_explore-1.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lean-explore
-Version: 0.3.0
+Version: 1.0.0
 Summary: A search engine for Lean 4 declarations.
 Author-email: Justin Asher <justinchadwickasher@gmail.com>
 License:                                  Apache License
@@ -208,7 +208,7 @@ License:                                  Apache License
 Project-URL: Homepage, https://www.leanexplore.com/
 Project-URL: Repository, https://github.com/justincasher/lean-explore
 Keywords: lean,lean4,search,formal methods,theorem prover,math,AI
-Classifier: Development Status :: 3 - Alpha
+Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: Science/Research
 Classifier: License :: OSI Approved :: Apache Software License
@@ -223,12 +223,13 @@ Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: sqlalchemy>=2.0
+Requires-Dist: aiosqlite>=0.19.0
+Requires-Dist: greenlet>=3.0.0
 Requires-Dist: numpy>=1.20
 Requires-Dist: faiss-cpu>=1.7
-Requires-Dist: sentence-transformers>=2.2.0
 Requires-Dist: filelock>=3.0.0
 Requires-Dist: nltk>=3.6
-Requires-Dist: rank-bm25>=0.2.2
+Requires-Dist: bm25s>=0.2.0
 Requires-Dist: httpx>=0.23.0
 Requires-Dist: pydantic>=2.0
 Requires-Dist: typer[all]>=0.9.0
@@ -236,7 +237,23 @@ Requires-Dist: toml>=0.10.0
 Requires-Dist: openai-agents>=0.0.16
 Requires-Dist: mcp>=1.9.0
 Requires-Dist: tqdm>=4.60
+Requires-Dist: rich>=13.0.0
 Requires-Dist: requests>=2.25.0
+Requires-Dist: tenacity>=8.0.0
+Requires-Dist: pooch>=1.8.0
+Provides-Extra: extract
+Requires-Dist: sentence-transformers>=2.2.0; extra == "extract"
+Requires-Dist: networkx>=3.0; extra == "extract"
+Requires-Dist: torch>=2.0.0; extra == "extract"
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == "dev"
+Requires-Dist: pytest-cov>=4.0; extra == "dev"
+Requires-Dist: pytest-asyncio>=0.21.0; extra == "dev"
+Requires-Dist: ruff>=0.1.0; extra == "dev"
+Requires-Dist: pre-commit>=3.0.0; extra == "dev"
+Requires-Dist: networkx>=3.0; extra == "dev"
+Requires-Dist: torch>=2.0.0; extra == "dev"
+Requires-Dist: sentence-transformers>=2.2.0; extra == "dev"
 Dynamic: license-file
 
 <h1 align="center">
@@ -269,21 +286,28 @@ A search engine for Lean 4 declarations. This project provides tools and resourc
 The current indexed projects include:
 
 * Batteries
-* Lean
+* CSLib
+* FLT (Fermat's Last Theorem)
+* FormalConjectures
 * Init
+* Lean
 * Mathlib
 * PhysLean
 * Std
 
 This code is distributed under an Apache License (see [LICENSE](LICENSE)).
 
-### Cite
+## Contributing
+
+Contributions are welcome! Please see [CONTRIBUTING.md](CONTRIBUTING.md) for guidelines on code style, testing, and development setup.
+
+## Cite
 
 If you use LeanExplore in your research or work, please cite it as follows:
 
 **General Citation:**
 
-Justin Asher. (2025). *LeanExplore: A search engine for Lean 4 declarations*. LeanExplore.com. (GitHub: [https://github.com/justincasher/lean-explore](https://github.com/justincasher/lean-explore)).
+Justin Asher. (2025). *LeanExplore: A search engine for Lean 4 declarations*. [https://arxiv.org/abs/2506.11085](https://arxiv.org/abs/2506.11085)
 
 **BibTeX Entry:**
 
@@ -292,7 +316,6 @@ Justin Asher. (2025). *LeanExplore: A search engine for Lean 4 declarations*. Le
   author = {Asher, Justin},
   title = {{LeanExplore: A search engine for Lean 4 declarations}},
   year = {2025},
-  url = {http://www.leanexplore.com},
-  note = {GitHub repository: https://github.com/justincasher/lean-explore}
+  url = {https://arxiv.org/abs/2506.11085}
 }
 ```

{lean_explore-0.3.0 → lean_explore-1.0.0}/README.md

@@ -28,21 +28,28 @@ A search engine for Lean 4 declarations. This project provides tools and resourc
 The current indexed projects include:
 
 * Batteries
-* Lean
+* CSLib
+* FLT (Fermat's Last Theorem)
+* FormalConjectures
 * Init
+* Lean
 * Mathlib
 * PhysLean
 * Std
 
 This code is distributed under an Apache License (see [LICENSE](LICENSE)).
 
-### Cite
+## Contributing
+
+Contributions are welcome! Please see [CONTRIBUTING.md](CONTRIBUTING.md) for guidelines on code style, testing, and development setup.
+
+## Cite
 
 If you use LeanExplore in your research or work, please cite it as follows:
 
 **General Citation:**
 
-Justin Asher. (2025). *LeanExplore: A search engine for Lean 4 declarations*. LeanExplore.com. (GitHub: [https://github.com/justincasher/lean-explore](https://github.com/justincasher/lean-explore)).
+Justin Asher. (2025). *LeanExplore: A search engine for Lean 4 declarations*. [https://arxiv.org/abs/2506.11085](https://arxiv.org/abs/2506.11085)
 
 **BibTeX Entry:**
 
@@ -51,7 +58,6 @@ Justin Asher. (2025). *LeanExplore: A search engine for Lean 4 declarations*. Le
   author = {Asher, Justin},
   title = {{LeanExplore: A search engine for Lean 4 declarations}},
   year = {2025},
-  url = {http://www.leanexplore.com},
-  note = {GitHub repository: https://github.com/justincasher/lean-explore}
+  url = {https://arxiv.org/abs/2506.11085}
 }
 ```

{lean_explore-0.3.0 → lean_explore-1.0.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "lean-explore"
-version = "0.3.0"
+version = "1.0.0"
 authors = [
     { name = "Justin Asher", email = "justinchadwickasher@gmail.com" },
 ]
@@ -14,7 +14,7 @@ requires-python = ">=3.10"
 license = { file = "LICENSE" }
 keywords = ["lean", "lean4", "search", "formal methods", "theorem prover", "math", "AI"]
 classifiers = [
-    "Development Status :: 3 - Alpha",
+    "Development Status :: 4 - Beta",
     "Intended Audience :: Developers",
     "Intended Audience :: Science/Research",
     "License :: OSI Approved :: Apache Software License",
@@ -28,14 +28,15 @@ classifiers = [
 ]
 
 dependencies = [
-    # Core data and search (primarily for local backend)
+    # Core data and search
     "sqlalchemy>=2.0",
+    "aiosqlite>=0.19.0",
+    "greenlet>=3.0.0",
     "numpy>=1.20",
     "faiss-cpu>=1.7",
-    "sentence-transformers>=2.2.0",
     "filelock>=3.0.0",
     "nltk>=3.6",
-    "rank-bm25>=0.2.2",
+    "bm25s>=0.2.0",
 
     # API Client / Shared Data Models
     "httpx>=0.23.0",
@@ -51,7 +52,10 @@ dependencies = [
 
     # Utilities
     "tqdm>=4.60",
+    "rich>=13.0.0",
     "requests>=2.25.0",
+    "tenacity>=8.0.0",
+    "pooch>=1.8.0",
 ]
 
 [project.urls]
@@ -59,14 +63,37 @@ Homepage = "https://www.leanexplore.com/"
 Repository = "https://github.com/justincasher/lean-explore"
 
 [project.scripts]
-leanexplore = "lean_explore.cli.main:app"
+lean-explore = "lean_explore.cli.main:app"
+
+[project.optional-dependencies]
+extract = [
+    "sentence-transformers>=2.2.0",
+    "networkx>=3.0",
+    "torch>=2.0.0",
+]
+
+dev = [
+    "pytest>=7.0",
+    "pytest-cov>=4.0",
+    "pytest-asyncio>=0.21.0",
+    "ruff>=0.1.0",
+    "pre-commit>=3.0.0",
+    "networkx>=3.0",
+    "torch>=2.0.0",
+    "sentence-transformers>=2.2.0",
+]
 
 [tool.setuptools.packages.find]
 where = ["src"]
 
 [tool.pytest.ini_options]
-asyncio_mode = "strict"
+asyncio_mode = "auto"
 asyncio_default_fixture_loop_scope = "function"
+markers = [
+    "slow: marks tests as slow (deselect with '-m \"not slow\"')",
+    "integration: marks tests as integration tests",
+    "external: marks tests that require external services",
+]
 
 # -- Ruff Configuration --
 
@@ -74,8 +101,8 @@ asyncio_default_fixture_loop_scope = "function"
 # Set the maximum line length.
 line-length = 88
 
-# Set based on your `requires-python = ">=3.8"`.
-target-version = "py38"
+# Set based on requires-python = ">=3.10".
+target-version = "py310"
 
 # Define the patterns for files Ruff should lint.
 include = [

lean_explore-1.0.0/src/lean_explore/__init__.py

@@ -0,0 +1,14 @@
+"""Lean Explore - Search and explore Lean mathematical libraries.
+
+This package provides tools for searching Lean declarations using hybrid
+semantic and lexical search, with support for both local and remote backends.
+
+Subpackages:
+    api: Remote API client for the Lean Explore cloud service.
+    cli: Command-line interface for search and data management.
+    extract: Data extraction pipeline from doc-gen4 output.
+    mcp: Model Context Protocol server for AI assistant integration.
+    models: Data models for declarations and search results.
+    search: Local search engine with BM25 and semantic search.
+    util: Shared utilities for embeddings, reranking, and logging.
+"""

lean_explore-1.0.0/src/lean_explore/api/__init__.py

@@ -0,0 +1,12 @@
+"""Remote API client package for Lean Explore.
+
+This package provides an async HTTP client for connecting to the remote
+Lean Explore API service as an alternative to local search.
+
+Modules:
+    client: ApiClient class for search and declaration retrieval via HTTP.
+"""
+
+from lean_explore.api.client import ApiClient
+
+__all__ = ["ApiClient"]

lean_explore-1.0.0/src/lean_explore/api/client.py

@@ -0,0 +1,104 @@
+"""Client for interacting with the remote Lean Explore API."""
+
+import os
+
+import httpx
+
+from lean_explore.config import Config
+from lean_explore.models import SearchResponse, SearchResult
+
+
+class ApiClient:
+    """Async client for the remote Lean Explore API.
+
+    This client handles making HTTP requests to the API, authenticating
+    with an API key, and parsing responses into SearchResult objects.
+    """
+
+    def __init__(self, api_key: str | None = None, timeout: float = 10.0):
+        """Initialize the API client.
+
+        Args:
+            api_key: The API key for authentication. If None, reads from
+                LEANEXPLORE_API_KEY environment variable.
+            timeout: Default timeout for HTTP requests in seconds.
+
+        Raises:
+            ValueError: If no API key is provided and LEANEXPLORE_API_KEY is not set.
+        """
+        self.base_url: str = Config.API_BASE_URL
+        self.api_key: str = api_key or os.getenv("LEANEXPLORE_API_KEY", "")
+        if not self.api_key:
+            raise ValueError(
+                "API key required. Pass api_key parameter or set LEANEXPLORE_API_KEY "
+                "environment variable."
+            )
+        self.timeout: float = timeout
+        self._headers: dict[str, str] = {"Authorization": f"Bearer {self.api_key}"}
+
+    async def search(
+        self,
+        query: str,
+        limit: int = 20,
+        rerank_top: int | None = None,  # Ignored for API (server handles reranking)
+        packages: list[str] | None = None,
+    ) -> SearchResponse:
+        """Search for Lean declarations via the API.
+
+        Args:
+            query: The search query string.
+            limit: Maximum number of results to return.
+            rerank_top: Ignored for API backend (included for interface consistency).
+            packages: Filter results to specific packages (e.g., ["Mathlib"]).
+
+        Returns:
+            SearchResponse containing results and metadata.
+
+        Raises:
+            httpx.HTTPStatusError: If the API returns an HTTP error status.
+            httpx.RequestError: For network-related issues.
+        """
+        del rerank_top  # Unused - server handles reranking
+        endpoint = f"{self.base_url}/search"
+        params: dict[str, str | int] = {"q": query, "limit": limit}
+        if packages:
+            params["packages"] = ",".join(packages)
+
+        async with httpx.AsyncClient(timeout=self.timeout) as client:
+            response = await client.get(endpoint, params=params, headers=self._headers)
+            response.raise_for_status()
+            data = response.json()
+
+            # Parse API response into our types
+            results = [SearchResult(**item) for item in data.get("results", [])]
+
+            return SearchResponse(
+                query=query,
+                results=results,
+                count=len(results),
+                processing_time_ms=data.get("processing_time_ms"),
+            )
+
+    async def get_by_id(self, declaration_id: int) -> SearchResult | None:
+        """Retrieve a declaration by ID via the API.
+
+        Args:
+            declaration_id: The declaration ID.
+
+        Returns:
+            SearchResult if found, None otherwise.
+
+        Raises:
+            httpx.HTTPStatusError: If the API returns an error (except 404).
+            httpx.RequestError: For network-related issues.
+        """
+        endpoint = f"{self.base_url}/declarations/{declaration_id}"
+
+        async with httpx.AsyncClient(timeout=self.timeout) as client:
+            response = await client.get(endpoint, headers=self._headers)
+
+            if response.status_code == 404:
+                return None
+
+            response.raise_for_status()
+            return SearchResult(**response.json())

lean_explore-1.0.0/src/lean_explore/cli/__init__.py

@@ -0,0 +1,10 @@
+"""Command-line interface package for Lean Explore.
+
+This package provides CLI commands to search for Lean declarations via the
+remote API, manage MCP servers, and download/manage local data toolchains.
+
+Modules:
+    main: Core CLI application and top-level commands.
+    data_commands: Subcommands for managing local data toolchains.
+    display: Formatting and display utilities for search results.
+"""

lean_explore-1.0.0/src/lean_explore/cli/data_commands.py

@@ -0,0 +1,242 @@
+# src/lean_explore/cli/data_commands.py
+
+"""Manages local Lean Explore data toolchains.
+
+Provides CLI commands to download, install, and clean data files (database,
+FAISS index, etc.) from remote storage using Pooch for checksums and caching.
+"""
+
+import logging
+import shutil
+from typing import TypedDict
+
+import pooch
+import requests
+import typer
+from rich.console import Console
+
+from lean_explore.config import Config
+
+
+class ManifestFileEntry(TypedDict):
+    """A file entry in the manifest's toolchain version."""
+
+    remote_name: str
+    local_name: str
+    sha256: str
+
+
+class ToolchainVersionInfo(TypedDict):
+    """Version information for a specific toolchain in the manifest."""
+
+    assets_base_path_r2: str
+    files: list[ManifestFileEntry]
+
+
+class Manifest(TypedDict):
+    """Remote data manifest structure."""
+
+    default_toolchain: str
+    toolchains: dict[str, ToolchainVersionInfo]
+
+logger = logging.getLogger(__name__)
+
+app = typer.Typer(
+    name="data",
+    help="Manage local data toolchains for Lean Explore (e.g., download, list, "
+    "select, clean).",
+    no_args_is_help=True,
+)
+
+
+def _get_console() -> Console:
+    """Create a Rich console instance for output."""
+    return Console()
+
+
+def _fetch_manifest() -> Manifest | None:
+    """Fetches the remote data manifest.
+
+    Returns:
+        The manifest dictionary, or None if fetch fails.
+    """
+    console = _get_console()
+    try:
+        response = requests.get(Config.MANIFEST_URL, timeout=10)
+        response.raise_for_status()
+        return response.json()
+    except requests.exceptions.RequestException as error:
+        logger.error("Failed to fetch manifest: %s", error)
+        console.print(f"[bold red]Error fetching manifest: {error}[/bold red]")
+        return None
+
+
+def _resolve_version(manifest: Manifest, version: str | None) -> str:
+    """Resolves the version string to an actual toolchain version.
+
+    Args:
+        manifest: The manifest dictionary containing toolchain information.
+        version: The requested version, or None/"stable" for default.
+
+    Returns:
+        The resolved version string.
+
+    Raises:
+        ValueError: If the version cannot be resolved.
+    """
+    if not version or version.lower() == "stable":
+        resolved = manifest.get("default_toolchain")
+        if not resolved:
+            raise ValueError("No default_toolchain specified in manifest")
+        return resolved
+    return version
+
+
+def _build_file_registry(version_info: ToolchainVersionInfo) -> dict[str, str]:
+    """Builds a Pooch registry from version info.
+
+    Args:
+        version_info: The version information from the manifest.
+
+    Returns:
+        A dictionary mapping remote filenames to SHA256 checksums.
+    """
+    return {
+        file_entry["remote_name"]: f"sha256:{file_entry['sha256']}"
+        for file_entry in version_info.get("files", [])
+        if file_entry.get("remote_name") and file_entry.get("sha256")
+    }
+
+
+def _write_active_version(version: str) -> None:
+    """Write the active version to the version file.
+
+    Args:
+        version: The version string to write.
+    """
+    version_file = Config.CACHE_DIRECTORY.parent / "active_version"
+    version_file.parent.mkdir(parents=True, exist_ok=True)
+    version_file.write_text(version)
+    logger.info("Set active version to: %s", version)
+
+
+def _cleanup_old_versions(current_version: str) -> None:
+    """Remove all cached versions except the current one.
+
+    Args:
+        current_version: The version to keep.
+    """
+    if not Config.CACHE_DIRECTORY.exists():
+        return
+
+    for item in Config.CACHE_DIRECTORY.iterdir():
+        if item.is_dir() and item.name != current_version:
+            logger.info("Removing old version: %s", item.name)
+            try:
+                shutil.rmtree(item)
+            except OSError as error:
+                logger.warning("Failed to remove %s: %s", item.name, error)
+
+
+def _install_toolchain(version: str | None = None) -> None:
+    """Installs the data toolchain for the specified version.
+
+    Downloads and verifies all required data files (database, FAISS index, etc.)
+    using Pooch. Files are automatically decompressed and cached locally.
+    After successful installation, sets this version as the active version.
+
+    Args:
+        version: The version to install. If None, uses the default version.
+
+    Raises:
+        ValueError: If manifest fetch fails or version is not found.
+    """
+    console = _get_console()
+
+    manifest = _fetch_manifest()
+    if not manifest:
+        raise ValueError("Failed to fetch manifest")
+
+    resolved_version = _resolve_version(manifest, version)
+    version_info = manifest.get("toolchains", {}).get(resolved_version)
+    if not version_info:
+        available = list(manifest.get("toolchains", {}).keys())
+        raise ValueError(
+            f"Version '{resolved_version}' not found. Available: {available}"
+        )
+
+    registry = _build_file_registry(version_info)
+    base_path = version_info.get("assets_base_path_r2", "")
+    base_url = f"{Config.R2_ASSETS_BASE_URL}/{base_path}/"
+
+    file_downloader = pooch.create(
+        path=Config.CACHE_DIRECTORY / resolved_version,
+        base_url=base_url,
+        registry=registry,
+    )
+
+    # Download and decompress each file
+    for file_entry in version_info.get("files", []):
+        remote_name = file_entry.get("remote_name")
+        local_name = file_entry.get("local_name")
+        if remote_name and local_name:
+            logger.info("Downloading %s -> %s", remote_name, local_name)
+            file_downloader.fetch(
+                remote_name, processor=pooch.Decompress(name=local_name)
+            )
+
+    # Set this version as the active version and clean up old versions
+    _write_active_version(resolved_version)
+    _cleanup_old_versions(resolved_version)
+
+    console.print(f"[green]Installed data for version {resolved_version}[/green]")
+
+
+@app.callback()
+def main() -> None:
+    """Lean-Explore data CLI.
+
+    This callback exists only to prevent Typer from treating the first
+    sub-command as a *default* command when there is otherwise just one.
+    """
+    pass
+
+
+@app.command()
+def fetch(
+    version: str = typer.Option(
+        None,
+        "--version",
+        "-v",
+        help="Version to install (e.g., '0.1.0'). Defaults to stable/latest.",
+    ),
+) -> None:
+    """Fetches and installs the data toolchain from the remote repository.
+
+    Downloads the database, FAISS index, and other required data files.
+    Files are verified with SHA256 checksums and automatically decompressed.
+    """
+    _install_toolchain(version)
+
+
+@app.command("clean")
+def clean_data_toolchains() -> None:
+    """Removes all downloaded local data toolchains."""
+    console = _get_console()
+
+    if not Config.CACHE_DIRECTORY.exists():
+        console.print("[yellow]No local data found to clean.[/yellow]")
+        return
+
+    if typer.confirm("Delete all cached data?", default=False, abort=True):
+        try:
+            shutil.rmtree(Config.CACHE_DIRECTORY)
+            console.print("[green]Data cache cleared.[/green]")
+        except OSError as error:
+            logger.error("Failed to clean cache directory: %s", error)
+            console.print(f"[bold red]Error cleaning data: {error}[/bold red]")
+            raise typer.Exit(code=1)
+
+
+if __name__ == "__main__":
+    app()