lean-explore 0.1.4__tar.gz → 0.2.0__tar.gz

{lean_explore-0.1.4 → lean_explore-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lean-explore
-Version: 0.1.4
+Version: 0.2.0
 Summary: A project to explore and rank Lean mathematical declarations.
 Author-email: Justin Asher <justinchadwickasher@gmail.com>
 License:                                  Apache License
@@ -228,8 +228,9 @@ Requires-Dist: sqlalchemy>=2.0
 Requires-Dist: numpy>=1.20
 Requires-Dist: faiss-cpu>=1.7
 Requires-Dist: sentence-transformers>=2.2.0
-Requires-Dist: rapidfuzz>=3.0.0
 Requires-Dist: filelock>=3.0.0
+Requires-Dist: nltk>=3.6
+Requires-Dist: rank-bm25>=0.2.2
 Requires-Dist: httpx>=0.23.0
 Requires-Dist: pydantic>=2.0
 Requires-Dist: typer[all]>=0.9.0
@@ -237,6 +238,7 @@ Requires-Dist: toml>=0.10.0
 Requires-Dist: openai-agents>=0.0.16
 Requires-Dist: mcp>=1.9.0
 Requires-Dist: tqdm>=4.60
+Requires-Dist: requests>=2.25.0
 Dynamic: license-file
 
 # LeanExplore
@@ -261,7 +263,7 @@ If you use LeanExplore in your research or work, please cite it as follows:
 
 **General Citation:**
 
-Justin Asher. (2025). *LeanExplore: A search engine for Lean 4 declarations*. LeanExplore.com. Retrieved from [http://www.leanexplore.com](http://www.leanexplore.com) (GitHub: [https://github.com/justincasher/lean-explore](https://github.com/justincasher/lean-explore)).
+Justin Asher. (2025). *LeanExplore: A search engine for Lean 4 declarations*. LeanExplore.com. (GitHub: [https://github.com/justincasher/lean-explore](https://github.com/justincasher/lean-explore)).
 
 **BibTeX Entry:**
 
@@ -270,7 +272,6 @@ Justin Asher. (2025). *LeanExplore: A search engine for Lean 4 declarations*. Le
   author = {Asher, Justin},
   title = {{LeanExplore: A search engine for Lean 4 declarations}},
   year = {2025},
-  publisher = {LeanExplore.com},
   url = {http://www.leanexplore.com},
   note = {GitHub repository: https://github.com/justincasher/lean-explore}
 }

{lean_explore-0.1.4 → lean_explore-0.2.0}/README.md

@@ -20,7 +20,7 @@ If you use LeanExplore in your research or work, please cite it as follows:
 
 **General Citation:**
 
-Justin Asher. (2025). *LeanExplore: A search engine for Lean 4 declarations*. LeanExplore.com. Retrieved from [http://www.leanexplore.com](http://www.leanexplore.com) (GitHub: [https://github.com/justincasher/lean-explore](https://github.com/justincasher/lean-explore)).
+Justin Asher. (2025). *LeanExplore: A search engine for Lean 4 declarations*. LeanExplore.com. (GitHub: [https://github.com/justincasher/lean-explore](https://github.com/justincasher/lean-explore)).
 
 **BibTeX Entry:**
 
@@ -29,8 +29,7 @@ Justin Asher. (2025). *LeanExplore: A search engine for Lean 4 declarations*. Le
   author = {Asher, Justin},
   title = {{LeanExplore: A search engine for Lean 4 declarations}},
   year = {2025},
-  publisher = {LeanExplore.com},
   url = {http://www.leanexplore.com},
   note = {GitHub repository: https://github.com/justincasher/lean-explore}
 }
-```
+```

{lean_explore-0.1.4 → lean_explore-0.2.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "lean-explore"
-version = "0.1.4"
+version = "0.2.0"
 authors = [
     { name = "Justin Asher", email = "justinchadwickasher@gmail.com" },
 ]
@@ -35,8 +35,9 @@ dependencies = [
     "numpy>=1.20",
     "faiss-cpu>=1.7",
     "sentence-transformers>=2.2.0",
-    "rapidfuzz>=3.0.0",
     "filelock>=3.0.0",
+    "nltk>=3.6",
+    "rank-bm25>=0.2.2",
 
     # API Client / Shared Data Models
     "httpx>=0.23.0",
@@ -52,6 +53,7 @@ dependencies = [
 
     # Utilities
     "tqdm>=4.60",
+    "requests>=2.25.0",
 ]
 
 [project.urls]

{lean_explore-0.1.4 → lean_explore-0.2.0}/src/lean_explore/cli/data_commands.py

@@ -5,6 +5,7 @@
 This module includes functions to fetch toolchain data (database, FAISS index, etc.)
 from a remote source (Cloudflare R2), verify its integrity, decompress it,
 and place it in the appropriate local directory for the application to use.
+It also provides a command to clean up this downloaded data.
 """
 
 import gzip
@@ -32,7 +33,7 @@ from lean_explore import defaults  # For R2 URLs and local paths
 app = typer.Typer(
     name="data",
     help="Manage local data toolchains for Lean Explore (e.g., download, list, "
-    "select).",
+    "select, clean).",
     no_args_is_help=True,
 )
 
@@ -167,10 +168,6 @@ def _download_file_with_progress(
                         "reported size for progress bar if available, otherwise "
                         "expected size.[/yellow]"
                     )
-                # Prefer expected_size_bytes if it's provided and server doesn't send
-                # Content-Length or if we want to strictly adhere to manifest size for
-                # progress. However, for live progress, server's content-length is
-                # usually more accurate for what's being transferred.
                 if (
                     total_size_from_header == 0
                 ):  # If server didn't provide content-length
@@ -201,13 +198,11 @@ def _download_file_with_progress(
         finally:
             r.close()
 
-        # Sanity check after download
         actual_downloaded_size = destination_path.stat().st_size
         if (
             total_size_from_header > 0
             and actual_downloaded_size != total_size_from_header
         ):
-            # This might indicate an incomplete download if not all bytes were written.
             console.print(
                 f"[orange3]Warning: For [cyan]{description}[/cyan], downloaded size "
                 f"({actual_downloaded_size} bytes) differs from Content-Length header "
@@ -258,7 +253,6 @@ def _verify_sha256_checksum(file_path: pathlib.Path, expected_checksum: str) ->
     sha256_hash = hashlib.sha256()
     try:
         with open(file_path, "rb") as f:
-            # Read and update hash string value in blocks of 4K
             for byte_block in iter(lambda: f.read(4096), b""):
                 sha256_hash.update(byte_block)
         calculated_checksum = sha256_hash.hexdigest()
@@ -328,28 +322,18 @@ def main() -> None:
 
 
 @app.command()
-def fetch(
-    version: str = typer.Argument(
-        None,
-        help=(
-            "The toolchain version to fetch (e.g., 'stable', '0.1.0'). "
-            "'stable' will attempt to use the 'default_toolchain' from the manifest."
-        ),
-        show_default=False,
-    ),
-) -> None:
-    """Fetches and installs a specified data version from the remote repository.
-
-    Downloads necessary assets like the database and FAISS index, verifies their
-    integrity via SHA256 checksums, decompresses them, and places them into the
-    appropriate local directory (e.g., ~/.lean_explore/data/toolchains/<version>/).
+def fetch() -> None:
+    """Fetches and installs the default data toolchain from the remote repository.
+
+    This command identifies the 'default_toolchain' (often aliased as 'stable')
+    from the remote manifest, then downloads necessary assets like the database
+    and FAISS index. It verifies their integrity via SHA256 checksums,
+    decompresses them, and places them into the appropriate local versioned
+    directory (e.g., ~/.lean_explore/data/toolchains/<default_version>/).
     """
-    console.rule(
-        f"[bold blue]Fetching Lean Explore Data Toolchain: {version}[/bold blue]"
-    )
+    console.rule("[bold blue]Fetching Default Lean Explore Data Toolchain[/bold blue]")
 
-    if version is None:
-        version = "stable"
+    version_to_request = "stable"  # Always fetch the stable/default version
 
     # 1. Fetch and Parse Manifest
     console.print(f"Fetching data manifest from {defaults.R2_MANIFEST_DEFAULT_URL}...")
@@ -362,12 +346,12 @@ def fetch(
     console.print("[green]Manifest fetched successfully.[/green]")
 
     # 2. Resolve Target Version from Manifest
-    version_info = _resolve_toolchain_version_info(manifest_data, version)
+    version_info = _resolve_toolchain_version_info(manifest_data, version_to_request)
     if not version_info:
         # _resolve_toolchain_version_info already prints detailed errors
         raise typer.Exit(code=1)
 
-    resolved_version_key = version_info["_resolved_key"]  # Key like "0.1.0"
+    resolved_version_key = version_info["_resolved_key"]  # Key like "0.1.0" or "0.2.0"
     console.print(
         f"Processing toolchain version: [bold yellow]{resolved_version_key}"
         "[/bold yellow] "
@@ -400,12 +384,8 @@ def fetch(
         local_name = file_entry.get("local_name")
         remote_name = file_entry.get("remote_name")
         expected_checksum = file_entry.get("sha256")
-        expected_size_compressed = file_entry.get(
-            "size_bytes_compressed"
-        )  # This is size of .gz
-        assets_r2_path_prefix = version_info.get(
-            "assets_base_path_r2", ""
-        )  # e.g., "assets/0.1.0/"
+        expected_size_compressed = file_entry.get("size_bytes_compressed")
+        assets_r2_path_prefix = version_info.get("assets_base_path_r2", "")
 
         if not all([local_name, remote_name, expected_checksum]):
             console.print(
@@ -418,7 +398,7 @@ def fetch(
         console.rule(f"[bold cyan]Processing: {local_name}[/bold cyan]")
 
         final_local_path = local_version_dir / local_name
-        temp_download_path = local_version_dir / remote_name  # Path for the .gz file
+        temp_download_path = local_version_dir / remote_name
 
         remote_url = (
             defaults.R2_ASSETS_BASE_URL.rstrip("/")
@@ -473,9 +453,7 @@ def fetch(
             )
             if final_local_path.exists():
                 final_local_path.unlink(missing_ok=True)
-            if (
-                temp_download_path.exists()
-            ):  # Ensure .gz is also removed on decompress failure
+            if temp_download_path.exists():
                 temp_download_path.unlink(missing_ok=True)
             continue
 
@@ -500,7 +478,87 @@ def fetch(
         raise typer.Exit(code=1)
 
 
+@app.command("clean")
+def clean_data_toolchains() -> None:
+    """Removes all downloaded local data toolchains.
+
+    This command deletes all version-specific subdirectories and their contents
+    within the local toolchains storage directory (typically located at
+    ~/.lean_explore/data/toolchains/).
+
+    Configuration files will not be affected.
+    """
+    toolchains_dir = defaults.LEAN_EXPLORE_TOOLCHAINS_BASE_DIR
+    console.print(
+        f"Attempting to clean local data toolchains from: [dim]{toolchains_dir}[/dim]"
+    )
+
+    if not toolchains_dir.exists() or not any(toolchains_dir.iterdir()):
+        console.print("[yellow]No local toolchain data found to clean.[/yellow]")
+        raise typer.Exit(code=0)
+
+    console.print(
+        "[bold yellow]\nThis will delete all downloaded database files and other "
+        "toolchain assets stored locally.[/bold yellow]"
+    )
+    if not typer.confirm(
+        "Are you sure you want to proceed?",
+        default=False,
+        abort=True,  # Typer will exit if user chooses 'no' (the default)
+    ):
+        # This line is effectively not reached if user aborts.
+        # Kept for logical structure understanding, but Typer handles the abort.
+        return
+
+    console.print(f"\nCleaning data from {toolchains_dir}...")
+    deleted_items_count = 0
+    errors_encountered = False
+    try:
+        for item_path in toolchains_dir.iterdir():
+            try:
+                if item_path.is_dir():
+                    shutil.rmtree(item_path)
+                    console.print(f"  Removed directory: [dim]{item_path.name}[/dim]")
+                    deleted_items_count += 1
+                elif item_path.is_file():  # Handle stray files if any
+                    item_path.unlink()
+                    console.print(f"  Removed file: [dim]{item_path.name}[/dim]")
+                    deleted_items_count += 1
+            except OSError as e:
+                console.print(
+                    f"[bold red]  Error removing {item_path.name}: {e}[/bold red]"
+                )
+                errors_encountered = True
+
+        console.print("")  # Add a newline for better formatting after item list
+
+        if errors_encountered:
+            console.print(
+                "[bold orange3]Data cleaning process completed with some errors. "
+                "Please review messages above.[/bold orange3]"
+            )
+            raise typer.Exit(code=1)
+        elif deleted_items_count > 0:
+            console.print(
+                "[bold green]All local toolchain data has been successfully "
+                "cleaned.[/bold green]"
+            )
+        else:
+            # This case might occur if the directory contained no items
+            # that were directories or files, or if it became empty
+            # between the initial check and this point.
+            console.print(
+                "[yellow]No items were deleted. The toolchain directory might "
+                "have been empty or contained unexpected item types.[/yellow]"
+            )
+
+    except OSError as e:  # Error iterating the directory itself
+        console.print(
+            f"[bold red]An error occurred while accessing toolchain directory "
+            f"for cleaning: {e}[/bold red]"
+        )
+        raise typer.Exit(code=1)
+
+
 if __name__ == "__main__":
-    # This allows testing `python -m lean_explore.cli.data_commands fetch stable`
-    # For actual CLI use, this app will be mounted in `main.py`.
     app()

{lean_explore-0.1.4 → lean_explore-0.2.0}/src/lean_explore/defaults.py

@@ -31,7 +31,7 @@ LEAN_EXPLORE_TOOLCHAINS_BASE_DIR: Final[pathlib.Path] = (
 # In future enhancements, this could be determined dynamically
 # or from user configuration.
 # For now, it's set to the initial version of data provided ("0.1.0").
-DEFAULT_ACTIVE_TOOLCHAIN_VERSION: Final[str] = "0.1.0"
+DEFAULT_ACTIVE_TOOLCHAIN_VERSION: Final[str] = "0.2.0"
 
 # Path to the data directory for the currently active toolchain version.
 # Example: ~/.lean_explore/data/toolchains/0.1.0/
@@ -98,20 +98,17 @@ DEFAULT_EMBEDDING_MODEL_NAME: Final[str] = "BAAI/bge-base-en-v1.5"
 # FAISS Search Parameters
 DEFAULT_FAISS_K: Final[int] = 100  # Number of nearest neighbors from FAISS
 DEFAULT_FAISS_NPROBE: Final[int] = 200  # For IVF-type FAISS indexes
+DEFAULT_FAISS_OVERSAMPLING_FACTOR: Final[int] = (
+    3  # Factor to multiply faiss_k by when package filters are active.
+)
 
 # Scoring and Ranking Parameters
 DEFAULT_SEM_SIM_THRESHOLD: Final[float] = 0.525
-DEFAULT_PAGERANK_WEIGHT: Final[float] = 1.0
-DEFAULT_TEXT_RELEVANCE_WEIGHT: Final[float] = 0.2
-DEFAULT_NAME_MATCH_WEIGHT: Final[float] = 0.5
+DEFAULT_PAGERANK_WEIGHT: Final[float] = 0.2
+DEFAULT_TEXT_RELEVANCE_WEIGHT: Final[float] = 1.0
+DEFAULT_NAME_MATCH_WEIGHT: Final[float] = 1.0  # Ensuring float for consistency
 
 # Output Parameters
 DEFAULT_RESULTS_LIMIT: Final[int] = (
     50  # Default number of final results to display/return
 )
-
-
-# --- Other Constants (if any emerge) ---
-# Example: If your application needs other hardcoded default values,
-# they can be added here.
-# DEFAULT_SOME_OTHER_PARAMETER: Final[Any] = "some_value"