lean-explore 1.0.0__tar.gz → 1.0.1__tar.gz

{lean_explore-1.0.0 → lean_explore-1.0.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lean-explore
-Version: 1.0.0
+Version: 1.0.1
 Summary: A search engine for Lean 4 declarations.
 Author-email: Justin Asher <justinchadwickasher@gmail.com>
 License:                                  Apache License

{lean_explore-1.0.0 → lean_explore-1.0.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "lean-explore"
-version = "1.0.0"
+version = "1.0.1"
 authors = [
     { name = "Justin Asher", email = "justinchadwickasher@gmail.com" },
 ]

lean_explore-1.0.1/src/lean_explore/cli/data_commands.py

@@ -0,0 +1,259 @@
+# src/lean_explore/cli/data_commands.py
+
+"""Manages local Lean Explore data toolchains.
+
+Provides CLI commands to download, install, and clean data files (database,
+FAISS index, BM25 indexes, etc.) from remote storage.
+"""
+
+import logging
+import shutil
+from pathlib import Path
+
+import requests
+import typer
+from rich.console import Console
+from rich.progress import (
+    BarColumn,
+    DownloadColumn,
+    Progress,
+    TextColumn,
+    TransferSpeedColumn,
+)
+
+from lean_explore.config import Config
+
+logger = logging.getLogger(__name__)
+
+app = typer.Typer(
+    name="data",
+    help="Manage local data toolchains for Lean Explore (e.g., download, list, "
+    "select, clean).",
+    no_args_is_help=True,
+)
+
+# Files required for the search engine (relative to version directory)
+REQUIRED_FILES: list[str] = [
+    "lean_explore.db",
+    "informalization_faiss.index",
+    "informalization_faiss_ids_map.json",
+    "bm25_ids_map.json",
+]
+
+# BM25 index directories and their contents
+BM25_DIRECTORIES: dict[str, list[str]] = {
+    "bm25_name_raw": [
+        "data.csc.index.npy",
+        "indices.csc.index.npy",
+        "indptr.csc.index.npy",
+        "nonoccurrence_array.index.npy",
+        "params.index.json",
+        "vocab.index.json",
+    ],
+    "bm25_name_spaced": [
+        "data.csc.index.npy",
+        "indices.csc.index.npy",
+        "indptr.csc.index.npy",
+        "nonoccurrence_array.index.npy",
+        "params.index.json",
+        "vocab.index.json",
+    ],
+}
+
+
+def _get_console() -> Console:
+    """Create a Rich console instance for output."""
+    return Console()
+
+
+def _fetch_latest_version() -> str:
+    """Fetch the latest version identifier from remote storage.
+
+    Returns:
+        The version string (e.g., "20260127_103630").
+
+    Raises:
+        ValueError: If the latest version cannot be fetched.
+    """
+    latest_url = f"{Config.R2_ASSETS_BASE_URL}/assets/latest.txt"
+    try:
+        response = requests.get(latest_url, timeout=10)
+        response.raise_for_status()
+        return response.text.strip()
+    except requests.exceptions.RequestException as error:
+        logger.error("Failed to fetch latest version: %s", error)
+        raise ValueError(f"Failed to fetch latest version: {error}") from error
+
+
+def _download_file(url: str, destination: Path, progress: Progress) -> None:
+    """Download a file with progress tracking.
+
+    Args:
+        url: The URL to download from.
+        destination: The local path to save the file.
+        progress: Rich progress instance for tracking.
+    """
+    destination.parent.mkdir(parents=True, exist_ok=True)
+
+    response = requests.get(url, stream=True, timeout=300)
+    response.raise_for_status()
+
+    total_size = int(response.headers.get("content-length", 0))
+    task_id = progress.add_task(destination.name, total=total_size)
+
+    with open(destination, "wb") as file:
+        for chunk in response.iter_content(chunk_size=8192):
+            file.write(chunk)
+            progress.update(task_id, advance=len(chunk))
+
+
+def _write_active_version(version: str) -> None:
+    """Write the active version to the version file.
+
+    Args:
+        version: The version string to write.
+    """
+    version_file = Config.CACHE_DIRECTORY.parent / "active_version"
+    version_file.parent.mkdir(parents=True, exist_ok=True)
+    version_file.write_text(version)
+    logger.info("Set active version to: %s", version)
+
+
+def _cleanup_old_versions(current_version: str) -> None:
+    """Remove all cached versions except the current one.
+
+    Args:
+        current_version: The version to keep.
+    """
+    if not Config.CACHE_DIRECTORY.exists():
+        return
+
+    for item in Config.CACHE_DIRECTORY.iterdir():
+        if item.is_dir() and item.name != current_version:
+            logger.info("Removing old version: %s", item.name)
+            try:
+                shutil.rmtree(item)
+            except OSError as error:
+                logger.warning("Failed to remove %s: %s", item.name, error)
+
+
+def _install_toolchain(version: str | None = None) -> None:
+    """Install the data toolchain for the specified version.
+
+    Downloads all required data files (database, FAISS index, BM25 indexes)
+    from remote storage. After successful installation, sets this version
+    as the active version and cleans up old versions.
+
+    Args:
+        version: The version to install. If None, fetches the latest version.
+
+    Raises:
+        ValueError: If version fetch fails or download errors occur.
+    """
+    console = _get_console()
+
+    if version:
+        resolved_version = version
+    else:
+        console.print("Fetching latest version...")
+        resolved_version = _fetch_latest_version()
+
+    console.print(f"Installing version: [bold]{resolved_version}[/bold]")
+
+    base_url = f"{Config.R2_ASSETS_BASE_URL}/assets/{resolved_version}"
+    cache_path = Config.CACHE_DIRECTORY / resolved_version
+
+    # Build list of all files to download
+    files_to_download: list[tuple[str, Path]] = []
+
+    for filename in REQUIRED_FILES:
+        url = f"{base_url}/{filename}"
+        destination = cache_path / filename
+        files_to_download.append((url, destination))
+
+    for directory_name, directory_files in BM25_DIRECTORIES.items():
+        for filename in directory_files:
+            url = f"{base_url}/{directory_name}/{filename}"
+            destination = cache_path / directory_name / filename
+            files_to_download.append((url, destination))
+
+    # Download all files with progress
+    with Progress(
+        TextColumn("[bold blue]{task.description}"),
+        BarColumn(),
+        DownloadColumn(),
+        TransferSpeedColumn(),
+        console=console,
+    ) as progress:
+        for url, destination in files_to_download:
+            if destination.exists():
+                logger.info("Skipping existing file: %s", destination.name)
+                continue
+            try:
+                _download_file(url, destination, progress)
+            except requests.exceptions.RequestException as error:
+                logger.error("Failed to download %s: %s", url, error)
+                raise ValueError(f"Failed to download {url}: {error}") from error
+
+    # Set this version as active and clean up old versions
+    _write_active_version(resolved_version)
+    _cleanup_old_versions(resolved_version)
+
+    console.print(f"[green]Installed data for version {resolved_version}[/green]")
+
+
+@app.callback()
+def main() -> None:
+    """Lean-Explore data CLI.
+
+    This callback exists only to prevent Typer from treating the first
+    sub-command as a *default* command when there is otherwise just one.
+    """
+    pass
+
+
+@app.command()
+def fetch(
+    version: str = typer.Option(
+        None,
+        "--version",
+        "-v",
+        help="Version to install (e.g., '20260127_103630'). Defaults to latest.",
+    ),
+) -> None:
+    """Fetch and install the data toolchain from remote storage.
+
+    Downloads the database, FAISS index, and BM25 indexes required for
+    local search. Automatically cleans up old cached versions.
+    """
+    _install_toolchain(version)
+
+
+@app.command("clean")
+def clean_data_toolchains() -> None:
+    """Remove all downloaded local data toolchains."""
+    console = _get_console()
+
+    cache_exists = Config.CACHE_DIRECTORY.exists()
+    version_file = Config.CACHE_DIRECTORY.parent / "active_version"
+    version_exists = version_file.exists()
+
+    if not cache_exists and not version_exists:
+        console.print("[yellow]No local data found to clean.[/yellow]")
+        return
+
+    if typer.confirm("Delete all cached data?", default=False, abort=True):
+        try:
+            if cache_exists:
+                shutil.rmtree(Config.CACHE_DIRECTORY)
+            if version_exists:
+                version_file.unlink()
+            console.print("[green]Data cache cleared.[/green]")
+        except OSError as error:
+            logger.error("Failed to clean cache directory: %s", error)
+            console.print(f"[bold red]Error cleaning data: {error}[/bold red]")
+            raise typer.Exit(code=1)
+
+
+if __name__ == "__main__":
+    app()

{lean_explore-1.0.0 → lean_explore-1.0.1}/src/lean_explore/extract/doc_gen4.py

@@ -186,7 +186,7 @@ async def run_doc_gen4(
 
         config = PACKAGE_REGISTRY[package_name]
         workspace_path = Path("lean") / package_name
-        logger.info(f"\n{'='*50}\nPackage: {package_name}\n{'='*50}")
+        logger.info(f"\n{'=' * 50}\nPackage: {package_name}\n{'=' * 50}")
 
         if fresh:
             _clear_workspace_cache(workspace_path)

{lean_explore-1.0.0 → lean_explore-1.0.1}/src/lean_explore/extract/embeddings.py

@@ -341,9 +341,7 @@ async def generate_embeddings(
         # Phase 2: Generate embeddings for remaining declarations
         logger.info("Phase 2: Generating embeddings for remaining declarations...")
         client = EmbeddingClient(model_name=model_name, max_length=max_seq_length)
-        logger.info(
-            f"Using {client.model_name} on {client.device}"
-        )
+        logger.info(f"Using {client.model_name} on {client.device}")
 
         total = len(remaining)
         total_embeddings = 0

{lean_explore-1.0.0 → lean_explore-1.0.1}/src/lean_explore/extract/index.py

@@ -95,8 +95,7 @@ def _build_faiss_index(embeddings: np.ndarray, device: str) -> faiss.Index:
     nlist = max(256, int(np.sqrt(num_vectors)))
 
     logger.info(
-        f"Building FAISS IVF index for {num_vectors} vectors "
-        f"with {nlist} clusters..."
+        f"Building FAISS IVF index for {num_vectors} vectors with {nlist} clusters..."
     )
 
     # Use inner product (cosine similarity on normalized vectors)

{lean_explore-1.0.0 → lean_explore-1.0.1}/src/lean_explore/extract/package_utils.py

@@ -91,11 +91,11 @@ def update_lakefile_docgen_version(lakefile_path: Path, lean_version: str) -> No
     content = lakefile_path.read_text()
 
     pattern = (
-        r'require «doc-gen4» from git\s+'
+        r"require «doc-gen4» from git\s+"
         r'"https://github\.com/leanprover/doc-gen4"(?:\s+@\s+"[^"]*")?'
     )
     replacement = (
-        f'require «doc-gen4» from git\n'
+        f"require «doc-gen4» from git\n"
         f'  "https://github.com/leanprover/doc-gen4" @ "{lean_version}"'
     )
     new_content = re.sub(pattern, replacement, content)

{lean_explore-1.0.0 → lean_explore-1.0.1}/src/lean_explore/mcp/tools.py

@@ -31,6 +31,7 @@ class SearchResponseDict(TypedDict, total=False):
     count: int
     processing_time_ms: int | None
 
+
 logger = logging.getLogger(__name__)
 
 

{lean_explore-1.0.0 → lean_explore-1.0.1}/src/lean_explore/search/engine.py

@@ -232,9 +232,7 @@ class SearchEngine:
             Map of declaration ID to semantic similarity score.
         """
         embedding_response = await self.embedding_client.embed([query], is_query=True)
-        query_embedding = np.array(
-            [embedding_response.embeddings[0]], dtype=np.float32
-        )
+        query_embedding = np.array([embedding_response.embeddings[0]], dtype=np.float32)
 
         import faiss as faiss_module
 
@@ -581,9 +579,7 @@ class SearchEngine:
             declarations_map = self._filter_by_packages(declarations_map, packages)
             # Filter boosted_scores to only include filtered declarations
             boosted_scores = [
-                (cid, score)
-                for cid, score in boosted_scores
-                if cid in declarations_map
+                (cid, score) for cid, score in boosted_scores if cid in declarations_map
             ]
             logger.info(f"Filtered to {len(declarations_map)} in {packages}")
 

{lean_explore-1.0.0 → lean_explore-1.0.1}/src/lean_explore/util/reranker_client.py

@@ -86,9 +86,7 @@ class RerankerClient:
             Formatted string for the reranker model.
         """
         return (
-            f"<Instruct>: {self.instruction}\n"
-            f"<Query>: {query}\n"
-            f"<Document>: {document}"
+            f"<Instruct>: {self.instruction}\n<Query>: {query}\n<Document>: {document}"
         )
 
     @torch.no_grad()

{lean_explore-1.0.0 → lean_explore-1.0.1}/src/lean_explore.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lean-explore
-Version: 1.0.0
+Version: 1.0.1
 Summary: A search engine for Lean 4 declarations.
 Author-email: Justin Asher <justinchadwickasher@gmail.com>
 License:                                  Apache License

lean_explore-1.0.0/src/lean_explore/cli/data_commands.py

@@ -1,242 +0,0 @@
-# src/lean_explore/cli/data_commands.py
-
-"""Manages local Lean Explore data toolchains.
-
-Provides CLI commands to download, install, and clean data files (database,
-FAISS index, etc.) from remote storage using Pooch for checksums and caching.
-"""
-
-import logging
-import shutil
-from typing import TypedDict
-
-import pooch
-import requests
-import typer
-from rich.console import Console
-
-from lean_explore.config import Config
-
-
-class ManifestFileEntry(TypedDict):
-    """A file entry in the manifest's toolchain version."""
-
-    remote_name: str
-    local_name: str
-    sha256: str
-
-
-class ToolchainVersionInfo(TypedDict):
-    """Version information for a specific toolchain in the manifest."""
-
-    assets_base_path_r2: str
-    files: list[ManifestFileEntry]
-
-
-class Manifest(TypedDict):
-    """Remote data manifest structure."""
-
-    default_toolchain: str
-    toolchains: dict[str, ToolchainVersionInfo]
-
-logger = logging.getLogger(__name__)
-
-app = typer.Typer(
-    name="data",
-    help="Manage local data toolchains for Lean Explore (e.g., download, list, "
-    "select, clean).",
-    no_args_is_help=True,
-)
-
-
-def _get_console() -> Console:
-    """Create a Rich console instance for output."""
-    return Console()
-
-
-def _fetch_manifest() -> Manifest | None:
-    """Fetches the remote data manifest.
-
-    Returns:
-        The manifest dictionary, or None if fetch fails.
-    """
-    console = _get_console()
-    try:
-        response = requests.get(Config.MANIFEST_URL, timeout=10)
-        response.raise_for_status()
-        return response.json()
-    except requests.exceptions.RequestException as error:
-        logger.error("Failed to fetch manifest: %s", error)
-        console.print(f"[bold red]Error fetching manifest: {error}[/bold red]")
-        return None
-
-
-def _resolve_version(manifest: Manifest, version: str | None) -> str:
-    """Resolves the version string to an actual toolchain version.
-
-    Args:
-        manifest: The manifest dictionary containing toolchain information.
-        version: The requested version, or None/"stable" for default.
-
-    Returns:
-        The resolved version string.
-
-    Raises:
-        ValueError: If the version cannot be resolved.
-    """
-    if not version or version.lower() == "stable":
-        resolved = manifest.get("default_toolchain")
-        if not resolved:
-            raise ValueError("No default_toolchain specified in manifest")
-        return resolved
-    return version
-
-
-def _build_file_registry(version_info: ToolchainVersionInfo) -> dict[str, str]:
-    """Builds a Pooch registry from version info.
-
-    Args:
-        version_info: The version information from the manifest.
-
-    Returns:
-        A dictionary mapping remote filenames to SHA256 checksums.
-    """
-    return {
-        file_entry["remote_name"]: f"sha256:{file_entry['sha256']}"
-        for file_entry in version_info.get("files", [])
-        if file_entry.get("remote_name") and file_entry.get("sha256")
-    }
-
-
-def _write_active_version(version: str) -> None:
-    """Write the active version to the version file.
-
-    Args:
-        version: The version string to write.
-    """
-    version_file = Config.CACHE_DIRECTORY.parent / "active_version"
-    version_file.parent.mkdir(parents=True, exist_ok=True)
-    version_file.write_text(version)
-    logger.info("Set active version to: %s", version)
-
-
-def _cleanup_old_versions(current_version: str) -> None:
-    """Remove all cached versions except the current one.
-
-    Args:
-        current_version: The version to keep.
-    """
-    if not Config.CACHE_DIRECTORY.exists():
-        return
-
-    for item in Config.CACHE_DIRECTORY.iterdir():
-        if item.is_dir() and item.name != current_version:
-            logger.info("Removing old version: %s", item.name)
-            try:
-                shutil.rmtree(item)
-            except OSError as error:
-                logger.warning("Failed to remove %s: %s", item.name, error)
-
-
-def _install_toolchain(version: str | None = None) -> None:
-    """Installs the data toolchain for the specified version.
-
-    Downloads and verifies all required data files (database, FAISS index, etc.)
-    using Pooch. Files are automatically decompressed and cached locally.
-    After successful installation, sets this version as the active version.
-
-    Args:
-        version: The version to install. If None, uses the default version.
-
-    Raises:
-        ValueError: If manifest fetch fails or version is not found.
-    """
-    console = _get_console()
-
-    manifest = _fetch_manifest()
-    if not manifest:
-        raise ValueError("Failed to fetch manifest")
-
-    resolved_version = _resolve_version(manifest, version)
-    version_info = manifest.get("toolchains", {}).get(resolved_version)
-    if not version_info:
-        available = list(manifest.get("toolchains", {}).keys())
-        raise ValueError(
-            f"Version '{resolved_version}' not found. Available: {available}"
-        )
-
-    registry = _build_file_registry(version_info)
-    base_path = version_info.get("assets_base_path_r2", "")
-    base_url = f"{Config.R2_ASSETS_BASE_URL}/{base_path}/"
-
-    file_downloader = pooch.create(
-        path=Config.CACHE_DIRECTORY / resolved_version,
-        base_url=base_url,
-        registry=registry,
-    )
-
-    # Download and decompress each file
-    for file_entry in version_info.get("files", []):
-        remote_name = file_entry.get("remote_name")
-        local_name = file_entry.get("local_name")
-        if remote_name and local_name:
-            logger.info("Downloading %s -> %s", remote_name, local_name)
-            file_downloader.fetch(
-                remote_name, processor=pooch.Decompress(name=local_name)
-            )
-
-    # Set this version as the active version and clean up old versions
-    _write_active_version(resolved_version)
-    _cleanup_old_versions(resolved_version)
-
-    console.print(f"[green]Installed data for version {resolved_version}[/green]")
-
-
-@app.callback()
-def main() -> None:
-    """Lean-Explore data CLI.
-
-    This callback exists only to prevent Typer from treating the first
-    sub-command as a *default* command when there is otherwise just one.
-    """
-    pass
-
-
-@app.command()
-def fetch(
-    version: str = typer.Option(
-        None,
-        "--version",
-        "-v",
-        help="Version to install (e.g., '0.1.0'). Defaults to stable/latest.",
-    ),
-) -> None:
-    """Fetches and installs the data toolchain from the remote repository.
-
-    Downloads the database, FAISS index, and other required data files.
-    Files are verified with SHA256 checksums and automatically decompressed.
-    """
-    _install_toolchain(version)
-
-
-@app.command("clean")
-def clean_data_toolchains() -> None:
-    """Removes all downloaded local data toolchains."""
-    console = _get_console()
-
-    if not Config.CACHE_DIRECTORY.exists():
-        console.print("[yellow]No local data found to clean.[/yellow]")
-        return
-
-    if typer.confirm("Delete all cached data?", default=False, abort=True):
-        try:
-            shutil.rmtree(Config.CACHE_DIRECTORY)
-            console.print("[green]Data cache cleared.[/green]")
-        except OSError as error:
-            logger.error("Failed to clean cache directory: %s", error)
-            console.print(f"[bold red]Error cleaning data: {error}[/bold red]")
-            raise typer.Exit(code=1)
-
-
-if __name__ == "__main__":
-    app()