lean-explore 0.3.0__tar.gz → 1.0.1__tar.gz

{lean_explore-0.3.0 → lean_explore-1.0.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lean-explore
-Version: 0.3.0
+Version: 1.0.1
 Summary: A search engine for Lean 4 declarations.
 Author-email: Justin Asher <justinchadwickasher@gmail.com>
 License:                                  Apache License
@@ -208,7 +208,7 @@ License:                                  Apache License
 Project-URL: Homepage, https://www.leanexplore.com/
 Project-URL: Repository, https://github.com/justincasher/lean-explore
 Keywords: lean,lean4,search,formal methods,theorem prover,math,AI
-Classifier: Development Status :: 3 - Alpha
+Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: Science/Research
 Classifier: License :: OSI Approved :: Apache Software License
@@ -223,12 +223,13 @@ Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: sqlalchemy>=2.0
+Requires-Dist: aiosqlite>=0.19.0
+Requires-Dist: greenlet>=3.0.0
 Requires-Dist: numpy>=1.20
 Requires-Dist: faiss-cpu>=1.7
-Requires-Dist: sentence-transformers>=2.2.0
 Requires-Dist: filelock>=3.0.0
 Requires-Dist: nltk>=3.6
-Requires-Dist: rank-bm25>=0.2.2
+Requires-Dist: bm25s>=0.2.0
 Requires-Dist: httpx>=0.23.0
 Requires-Dist: pydantic>=2.0
 Requires-Dist: typer[all]>=0.9.0
@@ -236,7 +237,23 @@ Requires-Dist: toml>=0.10.0
 Requires-Dist: openai-agents>=0.0.16
 Requires-Dist: mcp>=1.9.0
 Requires-Dist: tqdm>=4.60
+Requires-Dist: rich>=13.0.0
 Requires-Dist: requests>=2.25.0
+Requires-Dist: tenacity>=8.0.0
+Requires-Dist: pooch>=1.8.0
+Provides-Extra: extract
+Requires-Dist: sentence-transformers>=2.2.0; extra == "extract"
+Requires-Dist: networkx>=3.0; extra == "extract"
+Requires-Dist: torch>=2.0.0; extra == "extract"
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == "dev"
+Requires-Dist: pytest-cov>=4.0; extra == "dev"
+Requires-Dist: pytest-asyncio>=0.21.0; extra == "dev"
+Requires-Dist: ruff>=0.1.0; extra == "dev"
+Requires-Dist: pre-commit>=3.0.0; extra == "dev"
+Requires-Dist: networkx>=3.0; extra == "dev"
+Requires-Dist: torch>=2.0.0; extra == "dev"
+Requires-Dist: sentence-transformers>=2.2.0; extra == "dev"
 Dynamic: license-file
 
 <h1 align="center">
@@ -269,21 +286,28 @@ A search engine for Lean 4 declarations. This project provides tools and resourc
 The current indexed projects include:
 
 * Batteries
-* Lean
+* CSLib
+* FLT (Fermat's Last Theorem)
+* FormalConjectures
 * Init
+* Lean
 * Mathlib
 * PhysLean
 * Std
 
 This code is distributed under an Apache License (see [LICENSE](LICENSE)).
 
-### Cite
+## Contributing
+
+Contributions are welcome! Please see [CONTRIBUTING.md](CONTRIBUTING.md) for guidelines on code style, testing, and development setup.
+
+## Cite
 
 If you use LeanExplore in your research or work, please cite it as follows:
 
 **General Citation:**
 
-Justin Asher. (2025). *LeanExplore: A search engine for Lean 4 declarations*. LeanExplore.com. (GitHub: [https://github.com/justincasher/lean-explore](https://github.com/justincasher/lean-explore)).
+Justin Asher. (2025). *LeanExplore: A search engine for Lean 4 declarations*. [https://arxiv.org/abs/2506.11085](https://arxiv.org/abs/2506.11085)
 
 **BibTeX Entry:**
 
@@ -292,7 +316,6 @@ Justin Asher. (2025). *LeanExplore: A search engine for Lean 4 declarations*. Le
   author = {Asher, Justin},
   title = {{LeanExplore: A search engine for Lean 4 declarations}},
   year = {2025},
-  url = {http://www.leanexplore.com},
-  note = {GitHub repository: https://github.com/justincasher/lean-explore}
+  url = {https://arxiv.org/abs/2506.11085}
 }
 ```

{lean_explore-0.3.0 → lean_explore-1.0.1}/README.md

@@ -28,21 +28,28 @@ A search engine for Lean 4 declarations. This project provides tools and resourc
 The current indexed projects include:
 
 * Batteries
-* Lean
+* CSLib
+* FLT (Fermat's Last Theorem)
+* FormalConjectures
 * Init
+* Lean
 * Mathlib
 * PhysLean
 * Std
 
 This code is distributed under an Apache License (see [LICENSE](LICENSE)).
 
-### Cite
+## Contributing
+
+Contributions are welcome! Please see [CONTRIBUTING.md](CONTRIBUTING.md) for guidelines on code style, testing, and development setup.
+
+## Cite
 
 If you use LeanExplore in your research or work, please cite it as follows:
 
 **General Citation:**
 
-Justin Asher. (2025). *LeanExplore: A search engine for Lean 4 declarations*. LeanExplore.com. (GitHub: [https://github.com/justincasher/lean-explore](https://github.com/justincasher/lean-explore)).
+Justin Asher. (2025). *LeanExplore: A search engine for Lean 4 declarations*. [https://arxiv.org/abs/2506.11085](https://arxiv.org/abs/2506.11085)
 
 **BibTeX Entry:**
 
@@ -51,7 +58,6 @@ Justin Asher. (2025). *LeanExplore: A search engine for Lean 4 declarations*. Le
   author = {Asher, Justin},
   title = {{LeanExplore: A search engine for Lean 4 declarations}},
   year = {2025},
-  url = {http://www.leanexplore.com},
-  note = {GitHub repository: https://github.com/justincasher/lean-explore}
+  url = {https://arxiv.org/abs/2506.11085}
 }
 ```

{lean_explore-0.3.0 → lean_explore-1.0.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "lean-explore"
-version = "0.3.0"
+version = "1.0.1"
 authors = [
     { name = "Justin Asher", email = "justinchadwickasher@gmail.com" },
 ]
@@ -14,7 +14,7 @@ requires-python = ">=3.10"
 license = { file = "LICENSE" }
 keywords = ["lean", "lean4", "search", "formal methods", "theorem prover", "math", "AI"]
 classifiers = [
-    "Development Status :: 3 - Alpha",
+    "Development Status :: 4 - Beta",
     "Intended Audience :: Developers",
     "Intended Audience :: Science/Research",
     "License :: OSI Approved :: Apache Software License",
@@ -28,14 +28,15 @@ classifiers = [
 ]
 
 dependencies = [
-    # Core data and search (primarily for local backend)
+    # Core data and search
     "sqlalchemy>=2.0",
+    "aiosqlite>=0.19.0",
+    "greenlet>=3.0.0",
     "numpy>=1.20",
     "faiss-cpu>=1.7",
-    "sentence-transformers>=2.2.0",
     "filelock>=3.0.0",
     "nltk>=3.6",
-    "rank-bm25>=0.2.2",
+    "bm25s>=0.2.0",
 
     # API Client / Shared Data Models
     "httpx>=0.23.0",
@@ -51,7 +52,10 @@ dependencies = [
 
     # Utilities
     "tqdm>=4.60",
+    "rich>=13.0.0",
     "requests>=2.25.0",
+    "tenacity>=8.0.0",
+    "pooch>=1.8.0",
 ]
 
 [project.urls]
@@ -59,14 +63,37 @@ Homepage = "https://www.leanexplore.com/"
 Repository = "https://github.com/justincasher/lean-explore"
 
 [project.scripts]
-leanexplore = "lean_explore.cli.main:app"
+lean-explore = "lean_explore.cli.main:app"
+
+[project.optional-dependencies]
+extract = [
+    "sentence-transformers>=2.2.0",
+    "networkx>=3.0",
+    "torch>=2.0.0",
+]
+
+dev = [
+    "pytest>=7.0",
+    "pytest-cov>=4.0",
+    "pytest-asyncio>=0.21.0",
+    "ruff>=0.1.0",
+    "pre-commit>=3.0.0",
+    "networkx>=3.0",
+    "torch>=2.0.0",
+    "sentence-transformers>=2.2.0",
+]
 
 [tool.setuptools.packages.find]
 where = ["src"]
 
 [tool.pytest.ini_options]
-asyncio_mode = "strict"
+asyncio_mode = "auto"
 asyncio_default_fixture_loop_scope = "function"
+markers = [
+    "slow: marks tests as slow (deselect with '-m \"not slow\"')",
+    "integration: marks tests as integration tests",
+    "external: marks tests that require external services",
+]
 
 # -- Ruff Configuration --
 
@@ -74,8 +101,8 @@ asyncio_default_fixture_loop_scope = "function"
 # Set the maximum line length.
 line-length = 88
 
-# Set based on your `requires-python = ">=3.8"`.
-target-version = "py38"
+# Set based on requires-python = ">=3.10".
+target-version = "py310"
 
 # Define the patterns for files Ruff should lint.
 include = [

lean_explore-1.0.1/src/lean_explore/__init__.py

@@ -0,0 +1,14 @@
+"""Lean Explore - Search and explore Lean mathematical libraries.
+
+This package provides tools for searching Lean declarations using hybrid
+semantic and lexical search, with support for both local and remote backends.
+
+Subpackages:
+    api: Remote API client for the Lean Explore cloud service.
+    cli: Command-line interface for search and data management.
+    extract: Data extraction pipeline from doc-gen4 output.
+    mcp: Model Context Protocol server for AI assistant integration.
+    models: Data models for declarations and search results.
+    search: Local search engine with BM25 and semantic search.
+    util: Shared utilities for embeddings, reranking, and logging.
+"""

lean_explore-1.0.1/src/lean_explore/api/__init__.py

@@ -0,0 +1,12 @@
+"""Remote API client package for Lean Explore.
+
+This package provides an async HTTP client for connecting to the remote
+Lean Explore API service as an alternative to local search.
+
+Modules:
+    client: ApiClient class for search and declaration retrieval via HTTP.
+"""
+
+from lean_explore.api.client import ApiClient
+
+__all__ = ["ApiClient"]

lean_explore-1.0.1/src/lean_explore/api/client.py

@@ -0,0 +1,104 @@
+"""Client for interacting with the remote Lean Explore API."""
+
+import os
+
+import httpx
+
+from lean_explore.config import Config
+from lean_explore.models import SearchResponse, SearchResult
+
+
+class ApiClient:
+    """Async client for the remote Lean Explore API.
+
+    This client handles making HTTP requests to the API, authenticating
+    with an API key, and parsing responses into SearchResult objects.
+    """
+
+    def __init__(self, api_key: str | None = None, timeout: float = 10.0):
+        """Initialize the API client.
+
+        Args:
+            api_key: The API key for authentication. If None, reads from
+                LEANEXPLORE_API_KEY environment variable.
+            timeout: Default timeout for HTTP requests in seconds.
+
+        Raises:
+            ValueError: If no API key is provided and LEANEXPLORE_API_KEY is not set.
+        """
+        self.base_url: str = Config.API_BASE_URL
+        self.api_key: str = api_key or os.getenv("LEANEXPLORE_API_KEY", "")
+        if not self.api_key:
+            raise ValueError(
+                "API key required. Pass api_key parameter or set LEANEXPLORE_API_KEY "
+                "environment variable."
+            )
+        self.timeout: float = timeout
+        self._headers: dict[str, str] = {"Authorization": f"Bearer {self.api_key}"}
+
+    async def search(
+        self,
+        query: str,
+        limit: int = 20,
+        rerank_top: int | None = None,  # Ignored for API (server handles reranking)
+        packages: list[str] | None = None,
+    ) -> SearchResponse:
+        """Search for Lean declarations via the API.
+
+        Args:
+            query: The search query string.
+            limit: Maximum number of results to return.
+            rerank_top: Ignored for API backend (included for interface consistency).
+            packages: Filter results to specific packages (e.g., ["Mathlib"]).
+
+        Returns:
+            SearchResponse containing results and metadata.
+
+        Raises:
+            httpx.HTTPStatusError: If the API returns an HTTP error status.
+            httpx.RequestError: For network-related issues.
+        """
+        del rerank_top  # Unused - server handles reranking
+        endpoint = f"{self.base_url}/search"
+        params: dict[str, str | int] = {"q": query, "limit": limit}
+        if packages:
+            params["packages"] = ",".join(packages)
+
+        async with httpx.AsyncClient(timeout=self.timeout) as client:
+            response = await client.get(endpoint, params=params, headers=self._headers)
+            response.raise_for_status()
+            data = response.json()
+
+            # Parse API response into our types
+            results = [SearchResult(**item) for item in data.get("results", [])]
+
+            return SearchResponse(
+                query=query,
+                results=results,
+                count=len(results),
+                processing_time_ms=data.get("processing_time_ms"),
+            )
+
+    async def get_by_id(self, declaration_id: int) -> SearchResult | None:
+        """Retrieve a declaration by ID via the API.
+
+        Args:
+            declaration_id: The declaration ID.
+
+        Returns:
+            SearchResult if found, None otherwise.
+
+        Raises:
+            httpx.HTTPStatusError: If the API returns an error (except 404).
+            httpx.RequestError: For network-related issues.
+        """
+        endpoint = f"{self.base_url}/declarations/{declaration_id}"
+
+        async with httpx.AsyncClient(timeout=self.timeout) as client:
+            response = await client.get(endpoint, headers=self._headers)
+
+            if response.status_code == 404:
+                return None
+
+            response.raise_for_status()
+            return SearchResult(**response.json())

lean_explore-1.0.1/src/lean_explore/cli/__init__.py

@@ -0,0 +1,10 @@
+"""Command-line interface package for Lean Explore.
+
+This package provides CLI commands to search for Lean declarations via the
+remote API, manage MCP servers, and download/manage local data toolchains.
+
+Modules:
+    main: Core CLI application and top-level commands.
+    data_commands: Subcommands for managing local data toolchains.
+    display: Formatting and display utilities for search results.
+"""

lean_explore-1.0.1/src/lean_explore/cli/data_commands.py

@@ -0,0 +1,259 @@
+# src/lean_explore/cli/data_commands.py
+
+"""Manages local Lean Explore data toolchains.
+
+Provides CLI commands to download, install, and clean data files (database,
+FAISS index, BM25 indexes, etc.) from remote storage.
+"""
+
+import logging
+import shutil
+from pathlib import Path
+
+import requests
+import typer
+from rich.console import Console
+from rich.progress import (
+    BarColumn,
+    DownloadColumn,
+    Progress,
+    TextColumn,
+    TransferSpeedColumn,
+)
+
+from lean_explore.config import Config
+
+logger = logging.getLogger(__name__)
+
+app = typer.Typer(
+    name="data",
+    help="Manage local data toolchains for Lean Explore (e.g., download, list, "
+    "select, clean).",
+    no_args_is_help=True,
+)
+
+# Files required for the search engine (relative to version directory)
+REQUIRED_FILES: list[str] = [
+    "lean_explore.db",
+    "informalization_faiss.index",
+    "informalization_faiss_ids_map.json",
+    "bm25_ids_map.json",
+]
+
+# BM25 index directories and their contents
+BM25_DIRECTORIES: dict[str, list[str]] = {
+    "bm25_name_raw": [
+        "data.csc.index.npy",
+        "indices.csc.index.npy",
+        "indptr.csc.index.npy",
+        "nonoccurrence_array.index.npy",
+        "params.index.json",
+        "vocab.index.json",
+    ],
+    "bm25_name_spaced": [
+        "data.csc.index.npy",
+        "indices.csc.index.npy",
+        "indptr.csc.index.npy",
+        "nonoccurrence_array.index.npy",
+        "params.index.json",
+        "vocab.index.json",
+    ],
+}
+
+
+def _get_console() -> Console:
+    """Create a Rich console instance for output."""
+    return Console()
+
+
+def _fetch_latest_version() -> str:
+    """Fetch the latest version identifier from remote storage.
+
+    Returns:
+        The version string (e.g., "20260127_103630").
+
+    Raises:
+        ValueError: If the latest version cannot be fetched.
+    """
+    latest_url = f"{Config.R2_ASSETS_BASE_URL}/assets/latest.txt"
+    try:
+        response = requests.get(latest_url, timeout=10)
+        response.raise_for_status()
+        return response.text.strip()
+    except requests.exceptions.RequestException as error:
+        logger.error("Failed to fetch latest version: %s", error)
+        raise ValueError(f"Failed to fetch latest version: {error}") from error
+
+
+def _download_file(url: str, destination: Path, progress: Progress) -> None:
+    """Download a file with progress tracking.
+
+    Args:
+        url: The URL to download from.
+        destination: The local path to save the file.
+        progress: Rich progress instance for tracking.
+    """
+    destination.parent.mkdir(parents=True, exist_ok=True)
+
+    response = requests.get(url, stream=True, timeout=300)
+    response.raise_for_status()
+
+    total_size = int(response.headers.get("content-length", 0))
+    task_id = progress.add_task(destination.name, total=total_size)
+
+    with open(destination, "wb") as file:
+        for chunk in response.iter_content(chunk_size=8192):
+            file.write(chunk)
+            progress.update(task_id, advance=len(chunk))
+
+
+def _write_active_version(version: str) -> None:
+    """Write the active version to the version file.
+
+    Args:
+        version: The version string to write.
+    """
+    version_file = Config.CACHE_DIRECTORY.parent / "active_version"
+    version_file.parent.mkdir(parents=True, exist_ok=True)
+    version_file.write_text(version)
+    logger.info("Set active version to: %s", version)
+
+
+def _cleanup_old_versions(current_version: str) -> None:
+    """Remove all cached versions except the current one.
+
+    Args:
+        current_version: The version to keep.
+    """
+    if not Config.CACHE_DIRECTORY.exists():
+        return
+
+    for item in Config.CACHE_DIRECTORY.iterdir():
+        if item.is_dir() and item.name != current_version:
+            logger.info("Removing old version: %s", item.name)
+            try:
+                shutil.rmtree(item)
+            except OSError as error:
+                logger.warning("Failed to remove %s: %s", item.name, error)
+
+
+def _install_toolchain(version: str | None = None) -> None:
+    """Install the data toolchain for the specified version.
+
+    Downloads all required data files (database, FAISS index, BM25 indexes)
+    from remote storage. After successful installation, sets this version
+    as the active version and cleans up old versions.
+
+    Args:
+        version: The version to install. If None, fetches the latest version.
+
+    Raises:
+        ValueError: If version fetch fails or download errors occur.
+    """
+    console = _get_console()
+
+    if version:
+        resolved_version = version
+    else:
+        console.print("Fetching latest version...")
+        resolved_version = _fetch_latest_version()
+
+    console.print(f"Installing version: [bold]{resolved_version}[/bold]")
+
+    base_url = f"{Config.R2_ASSETS_BASE_URL}/assets/{resolved_version}"
+    cache_path = Config.CACHE_DIRECTORY / resolved_version
+
+    # Build list of all files to download
+    files_to_download: list[tuple[str, Path]] = []
+
+    for filename in REQUIRED_FILES:
+        url = f"{base_url}/{filename}"
+        destination = cache_path / filename
+        files_to_download.append((url, destination))
+
+    for directory_name, directory_files in BM25_DIRECTORIES.items():
+        for filename in directory_files:
+            url = f"{base_url}/{directory_name}/{filename}"
+            destination = cache_path / directory_name / filename
+            files_to_download.append((url, destination))
+
+    # Download all files with progress
+    with Progress(
+        TextColumn("[bold blue]{task.description}"),
+        BarColumn(),
+        DownloadColumn(),
+        TransferSpeedColumn(),
+        console=console,
+    ) as progress:
+        for url, destination in files_to_download:
+            if destination.exists():
+                logger.info("Skipping existing file: %s", destination.name)
+                continue
+            try:
+                _download_file(url, destination, progress)
+            except requests.exceptions.RequestException as error:
+                logger.error("Failed to download %s: %s", url, error)
+                raise ValueError(f"Failed to download {url}: {error}") from error
+
+    # Set this version as active and clean up old versions
+    _write_active_version(resolved_version)
+    _cleanup_old_versions(resolved_version)
+
+    console.print(f"[green]Installed data for version {resolved_version}[/green]")
+
+
+@app.callback()
+def main() -> None:
+    """Lean-Explore data CLI.
+
+    This callback exists only to prevent Typer from treating the first
+    sub-command as a *default* command when there is otherwise just one.
+    """
+    pass
+
+
+@app.command()
+def fetch(
+    version: str = typer.Option(
+        None,
+        "--version",
+        "-v",
+        help="Version to install (e.g., '20260127_103630'). Defaults to latest.",
+    ),
+) -> None:
+    """Fetch and install the data toolchain from remote storage.
+
+    Downloads the database, FAISS index, and BM25 indexes required for
+    local search. Automatically cleans up old cached versions.
+    """
+    _install_toolchain(version)
+
+
+@app.command("clean")
+def clean_data_toolchains() -> None:
+    """Remove all downloaded local data toolchains."""
+    console = _get_console()
+
+    cache_exists = Config.CACHE_DIRECTORY.exists()
+    version_file = Config.CACHE_DIRECTORY.parent / "active_version"
+    version_exists = version_file.exists()
+
+    if not cache_exists and not version_exists:
+        console.print("[yellow]No local data found to clean.[/yellow]")
+        return
+
+    if typer.confirm("Delete all cached data?", default=False, abort=True):
+        try:
+            if cache_exists:
+                shutil.rmtree(Config.CACHE_DIRECTORY)
+            if version_exists:
+                version_file.unlink()
+            console.print("[green]Data cache cleared.[/green]")
+        except OSError as error:
+            logger.error("Failed to clean cache directory: %s", error)
+            console.print(f"[bold red]Error cleaning data: {error}[/bold red]")
+            raise typer.Exit(code=1)
+
+
+if __name__ == "__main__":
+    app()