lean-explore 1.0.0__py3-none-any.whl → 1.0.2__py3-none-any.whl

lean_explore/cli/data_commands.py

@@ -3,42 +3,26 @@
 """Manages local Lean Explore data toolchains.
 
 Provides CLI commands to download, install, and clean data files (database,
-FAISS index, etc.) from remote storage using Pooch for checksums and caching.
+FAISS index, BM25 indexes, etc.) from remote storage.
 """
 
 import logging
 import shutil
-from typing import TypedDict
+from pathlib import Path
 
-import pooch
 import requests
 import typer
 from rich.console import Console
+from rich.progress import (
+    BarColumn,
+    DownloadColumn,
+    Progress,
+    TextColumn,
+    TransferSpeedColumn,
+)
 
 from lean_explore.config import Config
 
-
-class ManifestFileEntry(TypedDict):
-    """A file entry in the manifest's toolchain version."""
-
-    remote_name: str
-    local_name: str
-    sha256: str
-
-
-class ToolchainVersionInfo(TypedDict):
-    """Version information for a specific toolchain in the manifest."""
-
-    assets_base_path_r2: str
-    files: list[ManifestFileEntry]
-
-
-class Manifest(TypedDict):
-    """Remote data manifest structure."""
-
-    default_toolchain: str
-    toolchains: dict[str, ToolchainVersionInfo]
-
 logger = logging.getLogger(__name__)
 
 app = typer.Typer(
@@ -48,64 +32,79 @@ app = typer.Typer(
     no_args_is_help=True,
 )
 
+# Files required for the search engine (relative to version directory)
+REQUIRED_FILES: list[str] = [
+    "lean_explore.db",
+    "informalization_faiss.index",
+    "informalization_faiss_ids_map.json",
+    "bm25_ids_map.json",
+]
+
+# BM25 index directories and their contents
+BM25_DIRECTORIES: dict[str, list[str]] = {
+    "bm25_name_raw": [
+        "data.csc.index.npy",
+        "indices.csc.index.npy",
+        "indptr.csc.index.npy",
+        "nonoccurrence_array.index.npy",
+        "params.index.json",
+        "vocab.index.json",
+    ],
+    "bm25_name_spaced": [
+        "data.csc.index.npy",
+        "indices.csc.index.npy",
+        "indptr.csc.index.npy",
+        "nonoccurrence_array.index.npy",
+        "params.index.json",
+        "vocab.index.json",
+    ],
+}
+
 
 def _get_console() -> Console:
     """Create a Rich console instance for output."""
     return Console()
 
 
-def _fetch_manifest() -> Manifest | None:
-    """Fetches the remote data manifest.
+def _fetch_latest_version() -> str:
+    """Fetch the latest version identifier from remote storage.
 
     Returns:
-        The manifest dictionary, or None if fetch fails.
+        The version string (e.g., "20260127_103630").
+
+    Raises:
+        ValueError: If the latest version cannot be fetched.
     """
-    console = _get_console()
+    latest_url = f"{Config.R2_ASSETS_BASE_URL}/assets/latest.txt"
     try:
-        response = requests.get(Config.MANIFEST_URL, timeout=10)
+        response = requests.get(latest_url, timeout=10)
         response.raise_for_status()
-        return response.json()
+        return response.text.strip()
     except requests.exceptions.RequestException as error:
-        logger.error("Failed to fetch manifest: %s", error)
-        console.print(f"[bold red]Error fetching manifest: {error}[/bold red]")
-        return None
+        logger.error("Failed to fetch latest version: %s", error)
+        raise ValueError(f"Failed to fetch latest version: {error}") from error
 
 
-def _resolve_version(manifest: Manifest, version: str | None) -> str:
-    """Resolves the version string to an actual toolchain version.
+def _download_file(url: str, destination: Path, progress: Progress) -> None:
+    """Download a file with progress tracking.
 
     Args:
-        manifest: The manifest dictionary containing toolchain information.
-        version: The requested version, or None/"stable" for default.
-
-    Returns:
-        The resolved version string.
-
-    Raises:
-        ValueError: If the version cannot be resolved.
+        url: The URL to download from.
+        destination: The local path to save the file.
+        progress: Rich progress instance for tracking.
     """
-    if not version or version.lower() == "stable":
-        resolved = manifest.get("default_toolchain")
-        if not resolved:
-            raise ValueError("No default_toolchain specified in manifest")
-        return resolved
-    return version
+    destination.parent.mkdir(parents=True, exist_ok=True)
 
+    response = requests.get(url, stream=True, timeout=300)
+    response.raise_for_status()
 
-def _build_file_registry(version_info: ToolchainVersionInfo) -> dict[str, str]:
-    """Builds a Pooch registry from version info.
+    total_size = int(response.headers.get("content-length", 0))
+    task_id = progress.add_task(destination.name, total=total_size)
 
-    Args:
-        version_info: The version information from the manifest.
-
-    Returns:
-        A dictionary mapping remote filenames to SHA256 checksums.
-    """
-    return {
-        file_entry["remote_name"]: f"sha256:{file_entry['sha256']}"
-        for file_entry in version_info.get("files", [])
-        if file_entry.get("remote_name") and file_entry.get("sha256")
-    }
+    with open(destination, "wb") as file:
+        for chunk in response.iter_content(chunk_size=8192):
+            file.write(chunk)
+            progress.update(task_id, advance=len(chunk))
 
 
 def _write_active_version(version: str) -> None:
@@ -139,53 +138,64 @@ def _cleanup_old_versions(current_version: str) -> None:
 
 
 def _install_toolchain(version: str | None = None) -> None:
-    """Installs the data toolchain for the specified version.
+    """Install the data toolchain for the specified version.
 
-    Downloads and verifies all required data files (database, FAISS index, etc.)
-    using Pooch. Files are automatically decompressed and cached locally.
-    After successful installation, sets this version as the active version.
+    Downloads all required data files (database, FAISS index, BM25 indexes)
+    from remote storage. After successful installation, sets this version
+    as the active version and cleans up old versions.
 
     Args:
-        version: The version to install. If None, uses the default version.
+        version: The version to install. If None, fetches the latest version.
 
     Raises:
-        ValueError: If manifest fetch fails or version is not found.
+        ValueError: If version fetch fails or download errors occur.
     """
     console = _get_console()
 
-    manifest = _fetch_manifest()
-    if not manifest:
-        raise ValueError("Failed to fetch manifest")
-
-    resolved_version = _resolve_version(manifest, version)
-    version_info = manifest.get("toolchains", {}).get(resolved_version)
-    if not version_info:
-        available = list(manifest.get("toolchains", {}).keys())
-        raise ValueError(
-            f"Version '{resolved_version}' not found. Available: {available}"
-        )
-
-    registry = _build_file_registry(version_info)
-    base_path = version_info.get("assets_base_path_r2", "")
-    base_url = f"{Config.R2_ASSETS_BASE_URL}/{base_path}/"
-
-    file_downloader = pooch.create(
-        path=Config.CACHE_DIRECTORY / resolved_version,
-        base_url=base_url,
-        registry=registry,
-    )
-
-    # Download and decompress each file
-    for file_entry in version_info.get("files", []):
-        remote_name = file_entry.get("remote_name")
-        local_name = file_entry.get("local_name")
-        if remote_name and local_name:
-            logger.info("Downloading %s -> %s", remote_name, local_name)
-            file_downloader.fetch(
-                remote_name, processor=pooch.Decompress(name=local_name)
-            )
-
-    # Set this version as the active version and clean up old versions
+    if version:
+        resolved_version = version
+    else:
+        console.print("Fetching latest version...")
+        resolved_version = _fetch_latest_version()
+
+    console.print(f"Installing version: [bold]{resolved_version}[/bold]")
+
+    base_url = f"{Config.R2_ASSETS_BASE_URL}/assets/{resolved_version}"
+    cache_path = Config.CACHE_DIRECTORY / resolved_version
+
+    # Build list of all files to download
+    files_to_download: list[tuple[str, Path]] = []
+
+    for filename in REQUIRED_FILES:
+        url = f"{base_url}/{filename}"
+        destination = cache_path / filename
+        files_to_download.append((url, destination))
+
+    for directory_name, directory_files in BM25_DIRECTORIES.items():
+        for filename in directory_files:
+            url = f"{base_url}/{directory_name}/{filename}"
+            destination = cache_path / directory_name / filename
+            files_to_download.append((url, destination))
+
+    # Download all files with progress
+    with Progress(
+        TextColumn("[bold blue]{task.description}"),
+        BarColumn(),
+        DownloadColumn(),
+        TransferSpeedColumn(),
+        console=console,
+    ) as progress:
+        for url, destination in files_to_download:
+            if destination.exists():
+                logger.info("Skipping existing file: %s", destination.name)
+                continue
+            try:
+                _download_file(url, destination, progress)
+            except requests.exceptions.RequestException as error:
+                logger.error("Failed to download %s: %s", url, error)
+                raise ValueError(f"Failed to download {url}: {error}") from error
+
+    # Set this version as active and clean up old versions
     _write_active_version(resolved_version)
     _cleanup_old_versions(resolved_version)
 
@@ -208,29 +218,36 @@ def fetch(
         None,
         "--version",
         "-v",
-        help="Version to install (e.g., '0.1.0'). Defaults to stable/latest.",
+        help="Version to install (e.g., '20260127_103630'). Defaults to latest.",
     ),
 ) -> None:
-    """Fetches and installs the data toolchain from the remote repository.
+    """Fetch and install the data toolchain from remote storage.
 
-    Downloads the database, FAISS index, and other required data files.
-    Files are verified with SHA256 checksums and automatically decompressed.
+    Downloads the database, FAISS index, and BM25 indexes required for
+    local search. Automatically cleans up old cached versions.
     """
     _install_toolchain(version)
 
 
 @app.command("clean")
 def clean_data_toolchains() -> None:
-    """Removes all downloaded local data toolchains."""
+    """Remove all downloaded local data toolchains."""
     console = _get_console()
 
-    if not Config.CACHE_DIRECTORY.exists():
+    cache_exists = Config.CACHE_DIRECTORY.exists()
+    version_file = Config.CACHE_DIRECTORY.parent / "active_version"
+    version_exists = version_file.exists()
+
+    if not cache_exists and not version_exists:
         console.print("[yellow]No local data found to clean.[/yellow]")
         return
 
     if typer.confirm("Delete all cached data?", default=False, abort=True):
         try:
-            shutil.rmtree(Config.CACHE_DIRECTORY)
+            if cache_exists:
+                shutil.rmtree(Config.CACHE_DIRECTORY)
+            if version_exists:
+                version_file.unlink()
             console.print("[green]Data cache cleared.[/green]")
         except OSError as error:
             logger.error("Failed to clean cache directory: %s", error)

lean_explore/extract/doc_gen4.py

@@ -186,7 +186,7 @@ async def run_doc_gen4(
 
         config = PACKAGE_REGISTRY[package_name]
         workspace_path = Path("lean") / package_name
-        logger.info(f"\n{'='*50}\nPackage: {package_name}\n{'='*50}")
+        logger.info(f"\n{'=' * 50}\nPackage: {package_name}\n{'=' * 50}")
 
         if fresh:
             _clear_workspace_cache(workspace_path)

lean_explore/extract/embeddings.py

@@ -341,9 +341,7 @@ async def generate_embeddings(
         # Phase 2: Generate embeddings for remaining declarations
         logger.info("Phase 2: Generating embeddings for remaining declarations...")
         client = EmbeddingClient(model_name=model_name, max_length=max_seq_length)
-        logger.info(
-            f"Using {client.model_name} on {client.device}"
-        )
+        logger.info(f"Using {client.model_name} on {client.device}")
 
         total = len(remaining)
         total_embeddings = 0

lean_explore/extract/index.py

@@ -95,8 +95,7 @@ def _build_faiss_index(embeddings: np.ndarray, device: str) -> faiss.Index:
     nlist = max(256, int(np.sqrt(num_vectors)))
 
     logger.info(
-        f"Building FAISS IVF index for {num_vectors} vectors "
-        f"with {nlist} clusters..."
+        f"Building FAISS IVF index for {num_vectors} vectors with {nlist} clusters..."
     )
 
     # Use inner product (cosine similarity on normalized vectors)

lean_explore/extract/package_utils.py

@@ -91,11 +91,11 @@ def update_lakefile_docgen_version(lakefile_path: Path, lean_version: str) -> No
     content = lakefile_path.read_text()
 
     pattern = (
-        r'require «doc-gen4» from git\s+'
+        r"require «doc-gen4» from git\s+"
         r'"https://github\.com/leanprover/doc-gen4"(?:\s+@\s+"[^"]*")?'
     )
     replacement = (
-        f'require «doc-gen4» from git\n'
+        f"require «doc-gen4» from git\n"
         f'  "https://github.com/leanprover/doc-gen4" @ "{lean_version}"'
     )
     new_content = re.sub(pattern, replacement, content)

lean_explore/mcp/tools.py

@@ -31,6 +31,7 @@ class SearchResponseDict(TypedDict, total=False):
     count: int
     processing_time_ms: int | None
 
+
 logger = logging.getLogger(__name__)
 
 

lean_explore/search/engine.py

@@ -59,7 +59,7 @@ class SearchEngine:
         reranker_model_name: str = "Qwen/Qwen3-Reranker-0.6B",
         faiss_index_path: Path | None = None,
         faiss_ids_map_path: Path | None = None,
-        use_local_data: bool = True,
+        use_local_data: bool = False,
     ):
         """Initialize the search engine.
 
@@ -71,8 +71,9 @@ class SearchEngine:
             reranker_model_name: Name of the reranker model to use.
             faiss_index_path: Path to FAISS index. Defaults to config path.
             faiss_ids_map_path: Path to FAISS ID mapping. Defaults to config path.
-            use_local_data: If True, use DATA_DIRECTORY paths. If False, use
-                CACHE_DIRECTORY paths (for downloaded remote data).
+            use_local_data: If True, use DATA_DIRECTORY paths (for locally
+                generated data). If False, use CACHE_DIRECTORY paths (for
+                data downloaded via 'lean-explore data fetch').
         """
         self._embedding_client = embedding_client
         self._embedding_model_name = embedding_model_name
@@ -232,9 +233,7 @@ class SearchEngine:
             Map of declaration ID to semantic similarity score.
         """
         embedding_response = await self.embedding_client.embed([query], is_query=True)
-        query_embedding = np.array(
-            [embedding_response.embeddings[0]], dtype=np.float32
-        )
+        query_embedding = np.array([embedding_response.embeddings[0]], dtype=np.float32)
 
         import faiss as faiss_module
 
@@ -581,9 +580,7 @@ class SearchEngine:
             declarations_map = self._filter_by_packages(declarations_map, packages)
             # Filter boosted_scores to only include filtered declarations
             boosted_scores = [
-                (cid, score)
-                for cid, score in boosted_scores
-                if cid in declarations_map
+                (cid, score) for cid, score in boosted_scores if cid in declarations_map
             ]
             logger.info(f"Filtered to {len(declarations_map)} in {packages}")
 

lean_explore/util/reranker_client.py

@@ -86,9 +86,7 @@ class RerankerClient:
             Formatted string for the reranker model.
         """
         return (
-            f"<Instruct>: {self.instruction}\n"
-            f"<Query>: {query}\n"
-            f"<Document>: {document}"
+            f"<Instruct>: {self.instruction}\n<Query>: {query}\n<Document>: {document}"
         )
 
     @torch.no_grad()

{lean_explore-1.0.0.dist-info → lean_explore-1.0.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lean-explore
-Version: 1.0.0
+Version: 1.0.2
 Summary: A search engine for Lean 4 declarations.
 Author-email: Justin Asher <justinchadwickasher@gmail.com>
 License:                                  Apache License

{lean_explore-1.0.0.dist-info → lean_explore-1.0.2.dist-info}/RECORD

@@ -3,30 +3,30 @@ lean_explore/config.py,sha256=oZzGbpmOn1h2IZV6O9CNbw-zRasoOb8ejogkOqVp_BY,8464
 lean_explore/api/__init__.py,sha256=twql10w_SSqNbMD3tsCi0KYFp5A5n3UmUKSAfgWtwFc,341
 lean_explore/api/client.py,sha256=udvC28Sa797DxdfiAGRGgPpDMHPp3yN1ymkfwJ-wvi8,3741
 lean_explore/cli/__init__.py,sha256=JRpFfaP3DdPcPZ2MMIjJVw1RJ9gXLc8vBDa77favFlA,405
-lean_explore/cli/data_commands.py,sha256=zjlC_--KiqaueXqdKtRXxU89bKfZZmCAsOD1qJPcVo8,7362
+lean_explore/cli/data_commands.py,sha256=bpPfzYKwKhVz4gy727KdMYyZjO6UqozHS2xXiBF2l58,8084
 lean_explore/cli/display.py,sha256=t7qfP8Cdw05KM-kfYmFEeWSHNzyYuBOcvx4aCb5lFv0,5338
 lean_explore/cli/main.py,sha256=feVxTmzH0FMZGP7Urb6X43atlESJIYXgC6DJylgiKp0,3937
 lean_explore/extract/__init__.py,sha256=ZpCPPHjFCRxyHwYxQNhGpbpd_XuZOkgB9qkxa9zSUD0,184
 lean_explore/extract/__main__.py,sha256=Aquy2g-WIU7YO39UZRq0EoHgJWW-uM5G0FSK2OwDuR8,12010
-lean_explore/extract/doc_gen4.py,sha256=lnnc__YRTlSkGE8A432ecvTraWsoicHUDEQYHC5wNaw,6730
+lean_explore/extract/doc_gen4.py,sha256=5pvw-EdfDxg1NPxLpa6kQuoVUe1_TtKL1_hO13I0t78,6734
 lean_explore/extract/doc_parser.py,sha256=tb3y7409mn5DhPjR2prPh2Cm7QMf3VJy8qfir_-Y4z8,17511
-lean_explore/extract/embeddings.py,sha256=td53atXAAO15E4-MfEO2bVKdKQrz32-EzkfZXDpE-0A,11897
+lean_explore/extract/embeddings.py,sha256=atZzaM2wZBSZflL2sTbStGUSd1WKjyex218fMxG_CWc,11875
 lean_explore/extract/github.py,sha256=1Zyyl2u1esULJ3KFHHL8F80zcxLDx1Fh8u_gxCmBTi8,3499
-lean_explore/extract/index.py,sha256=cCZsLePSSGqHKINTifGaRp95RjIQQ9q2cxTlYnLlV0Y,10093
+lean_explore/extract/index.py,sha256=ctkhXIPV-IV6bKa6TzSeaXNof3E6HZxrLsnFLxS_Fi0,10081
 lean_explore/extract/informalize.py,sha256=AtjRQUvvaL22yn_D8Ik0HS3Xv-s6U9mwTv0sWMWs8oM,22209
 lean_explore/extract/package_config.py,sha256=-09icflXgioPzpnXNPhpTPkKixxBmWCLNsxEY50Uq2g,1890
 lean_explore/extract/package_registry.py,sha256=N-M3WYry77P80pjt12lpFcCuPW1M9SgxiCd_N8-EoXw,1565
-lean_explore/extract/package_utils.py,sha256=61d8EgCVdXZ44gfT9OeVwfBYhqjrbPsCwVVjQ2Zvylg,3400
+lean_explore/extract/package_utils.py,sha256=SBD4gbCA5enldm2m3AoL4Dti1FsSs-ShQYeYMjyOADg,3400
 lean_explore/extract/types.py,sha256=Sp6sYuioTE_Gs0Z0lbq4h7OMyAnaZafdLV8UGtKn-zs,583
 lean_explore/mcp/__init__.py,sha256=YO0RM466ik2jQk8YMDITzkm3AHPtjEhn7Wm7rOusUXo,462
 lean_explore/mcp/app.py,sha256=PJ2HwX6VyTqKejuI1G8Ld4aO9XWp9hT5H8loaA5g0Lc,2173
 lean_explore/mcp/server.py,sha256=Lf3SCn8ghPNkZ3BybHh3VCXn91F-yX6RSRke1rvC7Pk,8234
-lean_explore/mcp/tools.py,sha256=hs2lo66GcvGWbOGFEvFouXKwHrWo7rjftkuHpW4MYIY,4514
+lean_explore/mcp/tools.py,sha256=iOpJkezDDIBGfAKU5xfqVjrGQEP0MLGHHdkqHEnLPDE,4515
 lean_explore/models/__init__.py,sha256=G2Xeld_DADq-hhxm1K1CrEPeAM3ylHU2ckCh42Ogxro,321
 lean_explore/models/search_db.py,sha256=_a6B6FpqUevyHvW4KmNLeziiznIuxftpMUy0AtSDBJE,2673
 lean_explore/models/search_types.py,sha256=VcFGrK5Z12Cg9f2R-Y6GXGlh2SunyBAJaZ5I4DG0AUw,1442
 lean_explore/search/__init__.py,sha256=0k_iHe5xrurepznk7NzMYz10QFbK10ydMlpFlsuyFSc,1216
-lean_explore/search/engine.py,sha256=zP1jvxmknATXjzBDdrACmGhpwsFvtv2zcPOBErOUC-c,23645
+lean_explore/search/engine.py,sha256=oAsqiltBEXsbur0t77-zG8ATgcZ-8vKX8vvdoPNZsv0,23660
 lean_explore/search/scoring.py,sha256=VkH-kpGheX14_tf8uJYBOp0nrG05_JJLmv7_0QdfAQk,4168
 lean_explore/search/service.py,sha256=6CWN-U5jxv7cTzc7ffitgzNMn3k59LfirpteC4xsvSE,1915
 lean_explore/search/tokenization.py,sha256=1EHd3dbJLwnmrj2SmdU1W8WoyCUnzhJB5gzmZLpWifs,1831
@@ -34,10 +34,10 @@ lean_explore/util/__init__.py,sha256=07QNedHV2KNBRMfQKzkw4nbgZn5-W5_YRtqpeUnSokg
 lean_explore/util/embedding_client.py,sha256=6XJGJrGTXAiefDr-E1j_SPHTTZMIJYi62PwXXLdSLDQ,3098
 lean_explore/util/logging.py,sha256=hF8YPi-1I6DdC1B_yROXA6u5GG14IIhD0Nym2FfqgRA,649
 lean_explore/util/openrouter_client.py,sha256=C_0HLO5o1seYjGl2zn6897i2onK7CdI6XxtE3cWb3Os,1926
-lean_explore/util/reranker_client.py,sha256=7a5qTA7Dza_E4KvwWL6I5dRLjHrRTB8I2CynKJ0AmKc,6106
-lean_explore-1.0.0.dist-info/licenses/LICENSE,sha256=l4QLw1kIvEOjUktmmKm4dycK1E249Qs2s2AQTYbMXpY,11354
-lean_explore-1.0.0.dist-info/METADATA,sha256=sLZlcyWFOoPRS6MHUjsN0oqgP92KQJj4fEmEbzVbo7k,17084
-lean_explore-1.0.0.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
-lean_explore-1.0.0.dist-info/entry_points.txt,sha256=FuKSRE7GmI9B_kM-xoiWEJj2dQ4upqhHnw8qH1vcjW8,59
-lean_explore-1.0.0.dist-info/top_level.txt,sha256=h51BKWrFvB7iym-IlaNAAHX5MZfA8Gmg-aDuXGo0fQ8,13
-lean_explore-1.0.0.dist-info/RECORD,,
+lean_explore/util/reranker_client.py,sha256=kLCTGPMQuphjwAj0PPi9KXpSzDP7o9JRQJpTbmWGiMs,6074
+lean_explore-1.0.2.dist-info/licenses/LICENSE,sha256=l4QLw1kIvEOjUktmmKm4dycK1E249Qs2s2AQTYbMXpY,11354
+lean_explore-1.0.2.dist-info/METADATA,sha256=MzBiz91-USMmnybIjXQoUrr0ieU1MHOmyFaFdeDPVDc,17084
+lean_explore-1.0.2.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+lean_explore-1.0.2.dist-info/entry_points.txt,sha256=FuKSRE7GmI9B_kM-xoiWEJj2dQ4upqhHnw8qH1vcjW8,59
+lean_explore-1.0.2.dist-info/top_level.txt,sha256=h51BKWrFvB7iym-IlaNAAHX5MZfA8Gmg-aDuXGo0fQ8,13
+lean_explore-1.0.2.dist-info/RECORD,,