lean-explore 0.1.4__py3-none-any.whl → 0.2.1__py3-none-any.whl

lean_explore/cli/config_utils.py

@@ -6,7 +6,8 @@ This module provides functions to save and load user-specific settings,
 such as API keys for Lean Explore and OpenAI, from a configuration
 file stored in the user's home directory. It handles file creation,
 parsing, and sets secure permissions for files containing sensitive
-information.
+information. It also supports loading API keys from environment
+variables as a fallback if they are not found in the configuration file.
 """
 
 import logging
@@ -23,12 +24,14 @@ _APP_CONFIG_DIR_NAME: str = "leanexplore"
 _CONFIG_FILENAME: str = "config.toml"
 
 # Define keys for Lean Explore API section
-_LEAN_EXPLORE_API_SECTION_NAME: str = "lean_explore_api"  # Renamed for clarity
+_LEAN_EXPLORE_API_SECTION_NAME: str = "lean_explore_api"
 _LEAN_EXPLORE_API_KEY_NAME: str = "key"
+_LEAN_EXPLORE_API_KEY_ENV_VAR: str = "LEANEXPLORE_API_KEY"
 
 # Define keys for OpenAI API section
 _OPENAI_API_SECTION_NAME: str = "openai"
-_OPENAI_API_KEY_NAME: str = "api_key"  # Using a distinct key name for clarity
+_OPENAI_API_KEY_NAME: str = "api_key"
+_OPENAI_API_KEY_ENV_VAR: str = "OPENAI_API_KEY"
 
 
 def get_config_file_path() -> pathlib.Path:
@@ -83,7 +86,6 @@ def _load_config_data(config_file_path: pathlib.Path) -> Dict[str, Any]:
                 "Configuration file %s is corrupted. Treating as empty.",
                 config_file_path,
             )
-            # Potentially back up corrupted file before returning empty
         except Exception as e:
             logger.error(
                 "Error reading existing config file %s: %s",
@@ -91,7 +93,6 @@ def _load_config_data(config_file_path: pathlib.Path) -> Dict[str, Any]:
                 e,
                 exc_info=True,
             )
-            # Decide if to proceed with empty or raise further
     return config_data
 
 
@@ -162,9 +163,7 @@ def save_api_key(api_key: str) -> bool:
         if _save_config_data(config_file_path, config_data):
             logger.info("Lean Explore API key saved to %s", config_file_path)
             return True
-    except (
-        Exception
-    ) as e:  # Catch any exception from _ensure_config_dir_exists or broad issues
+    except Exception as e:
         logger.error(
             "General error during Lean Explore API key saving process: %s",
             e,
@@ -174,50 +173,86 @@ def save_api_key(api_key: str) -> bool:
 
 
 def load_api_key() -> Optional[str]:
-    """Loads the Lean Explore API key from the user's configuration file.
+    """Loads the Lean Explore API key.
+
+    It first checks the user's configuration file (typically
+    ~/.config/leanexplore/config.toml under the section
+    `lean_explore_api` with key `key`). If a valid, non-empty API key
+    is found there, it is returned.
+
+    If the API key is not found in the configuration file, is empty,
+    or is not a string, this function then checks the environment
+    variable `LEAN_EXPLORE_API_KEY`. If this environment variable is
+    set to a non-empty string, its value is returned.
+
+    If the API key is not found or is invalid in both locations,
+    None is returned.
 
     Returns:
         Optional[str]: The Lean Explore API key string if found and valid,
             otherwise None.
     """
     config_file_path = get_config_file_path()
-    if not config_file_path.exists() or not config_file_path.is_file():
-        logger.debug(
-            "Configuration file not found at %s for Lean Explore API key.",
-            config_file_path,
-        )
-        return None
 
-    try:
-        config_data = _load_config_data(config_file_path)
-        api_key = config_data.get(_LEAN_EXPLORE_API_SECTION_NAME, {}).get(
-            _LEAN_EXPLORE_API_KEY_NAME
-        )
-
-        if api_key and isinstance(api_key, str):
-            logger.debug(
-                "Lean Explore API key loaded successfully from %s", config_file_path
-            )
-            return api_key
-        elif api_key:  # Found but not a string
-            logger.warning(
-                "Lean Explore API key found in %s but is not a valid string.",
-                config_file_path,
+    # 1. Try loading from config file
+    if config_file_path.exists() and config_file_path.is_file():
+        try:
+            config_data = _load_config_data(config_file_path)
+            key_value = config_data.get(_LEAN_EXPLORE_API_SECTION_NAME, {}).get(
+                _LEAN_EXPLORE_API_KEY_NAME
             )
-        else:  # Not found under the expected keys
-            logger.debug(
-                "Lean Explore API key not found under section '%s', key '%s' in %s",
-                _LEAN_EXPLORE_API_SECTION_NAME,
-                _LEAN_EXPLORE_API_KEY_NAME,
+
+            if isinstance(key_value, str) and key_value:  # Non-empty string
+                logger.debug(
+                    "Lean Explore API key loaded from configuration file %s",
+                    config_file_path,
+                )
+                return key_value
+            elif key_value is not None:  # Present but not a valid non-empty string
+                logger.warning(
+                    "Lean Explore API key found in %s but is not a valid "
+                    "non-empty string. "
+                    "Will check environment variable %s.",
+                    config_file_path,
+                    _LEAN_EXPLORE_API_KEY_ENV_VAR,
+                )
+        except Exception as e:  # Catch unexpected errors during config processing
+            logger.error(
+                "Error processing configuration file %s for Lean Explore API key: %s. "
+                "Will check environment variable %s.",
                 config_file_path,
+                e,
+                _LEAN_EXPLORE_API_KEY_ENV_VAR,
+                exc_info=True,
             )
-    except Exception as e:  # Catch any other unexpected errors during loading
-        logger.error(
-            "Unexpected error loading Lean Explore API key from %s: %s",
+    else:
+        logger.debug(
+            "Configuration file %s not found. Will check environment "
+            "variable %s for Lean Explore API key.",
             config_file_path,
-            e,
-            exc_info=True,
+            _LEAN_EXPLORE_API_KEY_ENV_VAR,
         )
+
+    # 2. Try loading from environment variable
+    api_key_from_env = os.getenv(_LEAN_EXPLORE_API_KEY_ENV_VAR)
+
+    if isinstance(api_key_from_env, str) and api_key_from_env:  # Non-empty string
+        logger.debug(
+            "Lean Explore API key loaded from environment variable %s",
+            _LEAN_EXPLORE_API_KEY_ENV_VAR,
+        )
+        return api_key_from_env
+    elif api_key_from_env is not None:  # Env var exists but is empty string
+        logger.debug(
+            "Environment variable %s for Lean Explore API key is set but empty.",
+            _LEAN_EXPLORE_API_KEY_ENV_VAR,
+        )
+
+    logger.debug(
+        "Lean Explore API key not found in configuration file or "
+        "valid in environment variable %s.",
+        _LEAN_EXPLORE_API_KEY_ENV_VAR,
+    )
     return None
 
 
@@ -262,7 +297,7 @@ def delete_api_key() -> bool:
                 "Lean Explore API key not found in %s, no deletion performed.",
                 config_file_path,
             )
-            return True  # Key wasn't there, so considered successful
+            return True
 
     except Exception as e:
         logger.error(
@@ -316,46 +351,84 @@ def save_openai_api_key(api_key: str) -> bool:
 
 
 def load_openai_api_key() -> Optional[str]:
-    """Loads the OpenAI API key from the user's configuration file.
+    """Loads the OpenAI API key.
+
+    It first checks the user's configuration file (typically
+    ~/.config/leanexplore/config.toml under the section
+    `openai` with key `api_key`). If a valid, non-empty API key
+    is found there, it is returned.
+
+    If the API key is not found in the configuration file, is empty,
+    or is not a string, this function then checks the environment
+    variable `OPENAI_API_KEY`. If this environment variable is
+    set to a non-empty string, its value is returned.
+
+    If the API key is not found or is invalid in both locations,
+    None is returned.
 
     Returns:
         Optional[str]: The OpenAI API key string if found and valid, otherwise None.
     """
     config_file_path = get_config_file_path()
-    if not config_file_path.exists() or not config_file_path.is_file():
-        logger.debug(
-            "Configuration file not found at %s for OpenAI API key.", config_file_path
-        )
-        return None
 
-    try:
-        config_data = _load_config_data(config_file_path)
-        api_key = config_data.get(_OPENAI_API_SECTION_NAME, {}).get(
-            _OPENAI_API_KEY_NAME
-        )
-
-        if api_key and isinstance(api_key, str):
-            logger.debug("OpenAI API key loaded successfully from %s", config_file_path)
-            return api_key
-        elif api_key:  # Found but not a string
-            logger.warning(
-                "OpenAI API key found in %s but is not a valid string.",
-                config_file_path,
+    # 1. Try loading from config file
+    if config_file_path.exists() and config_file_path.is_file():
+        try:
+            config_data = _load_config_data(config_file_path)
+            key_value = config_data.get(_OPENAI_API_SECTION_NAME, {}).get(
+                _OPENAI_API_KEY_NAME
             )
-        else:  # Not found under the expected keys
-            logger.debug(
-                "OpenAI API key not found under section '%s', key '%s' in %s",
-                _OPENAI_API_SECTION_NAME,
-                _OPENAI_API_KEY_NAME,
+
+            if isinstance(key_value, str) and key_value:  # Non-empty string
+                logger.debug(
+                    "OpenAI API key loaded from configuration file %s",
+                    config_file_path,
+                )
+                return key_value
+            elif key_value is not None:  # Present but not a valid non-empty string
+                logger.warning(
+                    "OpenAI API key found in %s but is not a valid non-empty string. "
+                    "Will check environment variable %s.",
+                    config_file_path,
+                    _OPENAI_API_KEY_ENV_VAR,
+                )
+        except Exception as e:  # Catch unexpected errors during config processing
+            logger.error(
+                "Error processing configuration file %s for OpenAI API key: %s. "
+                "Will check environment variable %s.",
                 config_file_path,
+                e,
+                _OPENAI_API_KEY_ENV_VAR,
+                exc_info=True,
             )
-    except Exception as e:
-        logger.error(
-            "Unexpected error loading OpenAI API key from %s: %s",
+    else:
+        logger.debug(
+            "Configuration file %s not found. Will check environment "
+            "variable %s for OpenAI API key.",
             config_file_path,
-            e,
-            exc_info=True,
+            _OPENAI_API_KEY_ENV_VAR,
         )
+
+    # 2. Try loading from environment variable
+    api_key_from_env = os.getenv(_OPENAI_API_KEY_ENV_VAR)
+
+    if isinstance(api_key_from_env, str) and api_key_from_env:  # Non-empty string
+        logger.debug(
+            "OpenAI API key loaded from environment variable %s",
+            _OPENAI_API_KEY_ENV_VAR,
+        )
+        return api_key_from_env
+    elif api_key_from_env is not None:  # Env var exists but is empty string
+        logger.debug(
+            "Environment variable %s for OpenAI API key is set but empty.",
+            _OPENAI_API_KEY_ENV_VAR,
+        )
+
+    logger.debug(
+        "OpenAI API key not found in configuration file or valid in "
+        "environment variable %s.",
+        _OPENAI_API_KEY_ENV_VAR,
+    )
     return None
 
 
@@ -396,7 +469,7 @@ def delete_openai_api_key() -> bool:
                 "OpenAI API key not found in %s, no deletion performed.",
                 config_file_path,
             )
-            return True  # Key wasn't there, so considered successful
+            return True
 
     except Exception as e:
         logger.error(

lean_explore/cli/data_commands.py

@@ -5,6 +5,7 @@
 This module includes functions to fetch toolchain data (database, FAISS index, etc.)
 from a remote source (Cloudflare R2), verify its integrity, decompress it,
 and place it in the appropriate local directory for the application to use.
+It also provides a command to clean up this downloaded data.
 """
 
 import gzip
@@ -32,7 +33,7 @@ from lean_explore import defaults  # For R2 URLs and local paths
 app = typer.Typer(
     name="data",
     help="Manage local data toolchains for Lean Explore (e.g., download, list, "
-    "select).",
+    "select, clean).",
     no_args_is_help=True,
 )
 
@@ -167,10 +168,6 @@ def _download_file_with_progress(
                         "reported size for progress bar if available, otherwise "
                         "expected size.[/yellow]"
                     )
-                # Prefer expected_size_bytes if it's provided and server doesn't send
-                # Content-Length or if we want to strictly adhere to manifest size for
-                # progress. However, for live progress, server's content-length is
-                # usually more accurate for what's being transferred.
                 if (
                     total_size_from_header == 0
                 ):  # If server didn't provide content-length
@@ -201,13 +198,11 @@ def _download_file_with_progress(
         finally:
             r.close()
 
-        # Sanity check after download
         actual_downloaded_size = destination_path.stat().st_size
         if (
             total_size_from_header > 0
             and actual_downloaded_size != total_size_from_header
         ):
-            # This might indicate an incomplete download if not all bytes were written.
             console.print(
                 f"[orange3]Warning: For [cyan]{description}[/cyan], downloaded size "
                 f"({actual_downloaded_size} bytes) differs from Content-Length header "
@@ -258,7 +253,6 @@ def _verify_sha256_checksum(file_path: pathlib.Path, expected_checksum: str) ->
     sha256_hash = hashlib.sha256()
     try:
         with open(file_path, "rb") as f:
-            # Read and update hash string value in blocks of 4K
             for byte_block in iter(lambda: f.read(4096), b""):
                 sha256_hash.update(byte_block)
         calculated_checksum = sha256_hash.hexdigest()
@@ -328,28 +322,18 @@ def main() -> None:
 
 
 @app.command()
-def fetch(
-    version: str = typer.Argument(
-        None,
-        help=(
-            "The toolchain version to fetch (e.g., 'stable', '0.1.0'). "
-            "'stable' will attempt to use the 'default_toolchain' from the manifest."
-        ),
-        show_default=False,
-    ),
-) -> None:
-    """Fetches and installs a specified data version from the remote repository.
-
-    Downloads necessary assets like the database and FAISS index, verifies their
-    integrity via SHA256 checksums, decompresses them, and places them into the
-    appropriate local directory (e.g., ~/.lean_explore/data/toolchains/<version>/).
+def fetch() -> None:
+    """Fetches and installs the default data toolchain from the remote repository.
+
+    This command identifies the 'default_toolchain' (often aliased as 'stable')
+    from the remote manifest, then downloads necessary assets like the database
+    and FAISS index. It verifies their integrity via SHA256 checksums,
+    decompresses them, and places them into the appropriate local versioned
+    directory (e.g., ~/.lean_explore/data/toolchains/<default_version>/).
     """
-    console.rule(
-        f"[bold blue]Fetching Lean Explore Data Toolchain: {version}[/bold blue]"
-    )
+    console.rule("[bold blue]Fetching Default Lean Explore Data Toolchain[/bold blue]")
 
-    if version is None:
-        version = "stable"
+    version_to_request = "stable"  # Always fetch the stable/default version
 
     # 1. Fetch and Parse Manifest
     console.print(f"Fetching data manifest from {defaults.R2_MANIFEST_DEFAULT_URL}...")
@@ -362,12 +346,12 @@ def fetch(
     console.print("[green]Manifest fetched successfully.[/green]")
 
     # 2. Resolve Target Version from Manifest
-    version_info = _resolve_toolchain_version_info(manifest_data, version)
+    version_info = _resolve_toolchain_version_info(manifest_data, version_to_request)
     if not version_info:
         # _resolve_toolchain_version_info already prints detailed errors
         raise typer.Exit(code=1)
 
-    resolved_version_key = version_info["_resolved_key"]  # Key like "0.1.0"
+    resolved_version_key = version_info["_resolved_key"]  # Key like "0.1.0" or "0.2.0"
     console.print(
         f"Processing toolchain version: [bold yellow]{resolved_version_key}"
         "[/bold yellow] "
@@ -400,12 +384,8 @@ def fetch(
         local_name = file_entry.get("local_name")
         remote_name = file_entry.get("remote_name")
         expected_checksum = file_entry.get("sha256")
-        expected_size_compressed = file_entry.get(
-            "size_bytes_compressed"
-        )  # This is size of .gz
-        assets_r2_path_prefix = version_info.get(
-            "assets_base_path_r2", ""
-        )  # e.g., "assets/0.1.0/"
+        expected_size_compressed = file_entry.get("size_bytes_compressed")
+        assets_r2_path_prefix = version_info.get("assets_base_path_r2", "")
 
         if not all([local_name, remote_name, expected_checksum]):
             console.print(
@@ -418,7 +398,7 @@ def fetch(
         console.rule(f"[bold cyan]Processing: {local_name}[/bold cyan]")
 
         final_local_path = local_version_dir / local_name
-        temp_download_path = local_version_dir / remote_name  # Path for the .gz file
+        temp_download_path = local_version_dir / remote_name
 
         remote_url = (
             defaults.R2_ASSETS_BASE_URL.rstrip("/")
@@ -473,9 +453,7 @@ def fetch(
             )
             if final_local_path.exists():
                 final_local_path.unlink(missing_ok=True)
-            if (
-                temp_download_path.exists()
-            ):  # Ensure .gz is also removed on decompress failure
+            if temp_download_path.exists():
                 temp_download_path.unlink(missing_ok=True)
             continue
 
@@ -500,7 +478,87 @@ def fetch(
         raise typer.Exit(code=1)
 
 
+@app.command("clean")
+def clean_data_toolchains() -> None:
+    """Removes all downloaded local data toolchains.
+
+    This command deletes all version-specific subdirectories and their contents
+    within the local toolchains storage directory (typically located at
+    ~/.lean_explore/data/toolchains/).
+
+    Configuration files will not be affected.
+    """
+    toolchains_dir = defaults.LEAN_EXPLORE_TOOLCHAINS_BASE_DIR
+    console.print(
+        f"Attempting to clean local data toolchains from: [dim]{toolchains_dir}[/dim]"
+    )
+
+    if not toolchains_dir.exists() or not any(toolchains_dir.iterdir()):
+        console.print("[yellow]No local toolchain data found to clean.[/yellow]")
+        raise typer.Exit(code=0)
+
+    console.print(
+        "[bold yellow]\nThis will delete all downloaded database files and other "
+        "toolchain assets stored locally.[/bold yellow]"
+    )
+    if not typer.confirm(
+        "Are you sure you want to proceed?",
+        default=False,
+        abort=True,  # Typer will exit if user chooses 'no' (the default)
+    ):
+        # This line is effectively not reached if user aborts.
+        # Kept for logical structure understanding, but Typer handles the abort.
+        return
+
+    console.print(f"\nCleaning data from {toolchains_dir}...")
+    deleted_items_count = 0
+    errors_encountered = False
+    try:
+        for item_path in toolchains_dir.iterdir():
+            try:
+                if item_path.is_dir():
+                    shutil.rmtree(item_path)
+                    console.print(f"  Removed directory: [dim]{item_path.name}[/dim]")
+                    deleted_items_count += 1
+                elif item_path.is_file():  # Handle stray files if any
+                    item_path.unlink()
+                    console.print(f"  Removed file: [dim]{item_path.name}[/dim]")
+                    deleted_items_count += 1
+            except OSError as e:
+                console.print(
+                    f"[bold red]  Error removing {item_path.name}: {e}[/bold red]"
+                )
+                errors_encountered = True
+
+        console.print("")  # Add a newline for better formatting after item list
+
+        if errors_encountered:
+            console.print(
+                "[bold orange3]Data cleaning process completed with some errors. "
+                "Please review messages above.[/bold orange3]"
+            )
+            raise typer.Exit(code=1)
+        elif deleted_items_count > 0:
+            console.print(
+                "[bold green]All local toolchain data has been successfully "
+                "cleaned.[/bold green]"
+            )
+        else:
+            # This case might occur if the directory contained no items
+            # that were directories or files, or if it became empty
+            # between the initial check and this point.
+            console.print(
+                "[yellow]No items were deleted. The toolchain directory might "
+                "have been empty or contained unexpected item types.[/yellow]"
+            )
+
+    except OSError as e:  # Error iterating the directory itself
+        console.print(
+            f"[bold red]An error occurred while accessing toolchain directory "
+            f"for cleaning: {e}[/bold red]"
+        )
+        raise typer.Exit(code=1)
+
+
 if __name__ == "__main__":
-    # This allows testing `python -m lean_explore.cli.data_commands fetch stable`
-    # For actual CLI use, this app will be mounted in `main.py`.
     app()

lean_explore/defaults.py

@@ -31,7 +31,7 @@ LEAN_EXPLORE_TOOLCHAINS_BASE_DIR: Final[pathlib.Path] = (
 # In future enhancements, this could be determined dynamically
 # or from user configuration.
 # For now, it's set to the initial version of data provided ("0.1.0").
-DEFAULT_ACTIVE_TOOLCHAIN_VERSION: Final[str] = "0.1.0"
+DEFAULT_ACTIVE_TOOLCHAIN_VERSION: Final[str] = "0.2.0"
 
 # Path to the data directory for the currently active toolchain version.
 # Example: ~/.lean_explore/data/toolchains/0.1.0/
@@ -98,20 +98,17 @@ DEFAULT_EMBEDDING_MODEL_NAME: Final[str] = "BAAI/bge-base-en-v1.5"
 # FAISS Search Parameters
 DEFAULT_FAISS_K: Final[int] = 100  # Number of nearest neighbors from FAISS
 DEFAULT_FAISS_NPROBE: Final[int] = 200  # For IVF-type FAISS indexes
+DEFAULT_FAISS_OVERSAMPLING_FACTOR: Final[int] = (
+    3  # Factor to multiply faiss_k by when package filters are active.
+)
 
 # Scoring and Ranking Parameters
 DEFAULT_SEM_SIM_THRESHOLD: Final[float] = 0.525
-DEFAULT_PAGERANK_WEIGHT: Final[float] = 1.0
-DEFAULT_TEXT_RELEVANCE_WEIGHT: Final[float] = 0.2
-DEFAULT_NAME_MATCH_WEIGHT: Final[float] = 0.5
+DEFAULT_PAGERANK_WEIGHT: Final[float] = 0.2
+DEFAULT_TEXT_RELEVANCE_WEIGHT: Final[float] = 1.0
+DEFAULT_NAME_MATCH_WEIGHT: Final[float] = 1.0  # Ensuring float for consistency
 
 # Output Parameters
 DEFAULT_RESULTS_LIMIT: Final[int] = (
     50  # Default number of final results to display/return
 )
-
-
-# --- Other Constants (if any emerge) ---
-# Example: If your application needs other hardcoded default values,
-# they can be added here.
-# DEFAULT_SOME_OTHER_PARAMETER: Final[Any] = "some_value"