llmboost-hub 0.1.1__py3-none-any.whl

llmboost_hub/utils/lookup_cache.py

@@ -0,0 +1,123 @@
+import os
+import time
+import json
+import io
+import pandas as pd
+import requests
+import click
+
+from llmboost_hub.utils.config import config
+
+
+def _cache_is_fresh(path: str, ttl_seconds: int = 60) -> bool:
+    try:
+        mtime = os.path.getmtime(path)
+        return (time.time() - mtime) < ttl_seconds
+    except Exception:
+        return False
+
+
+def _write_cache_from_df(cache_path: str, df: pd.DataFrame, verbose: bool = False) -> None:
+    try:
+        if cache_path.lower().endswith(".json"):
+            df.to_json(cache_path, orient="records")
+        else:
+            df.to_csv(cache_path, index=False)
+    except Exception:
+        # best-effort cache
+        pass
+    if verbose:
+        click.echo(f"Refreshed cache at {cache_path}")
+
+
+def _load_df_from_cache(cache_path: str, verbose: bool = False) -> pd.DataFrame:
+    try:
+        if cache_path.lower().endswith(".json"):
+            with open(cache_path, "r", encoding="utf-8") as fh:
+                data = json.load(fh)
+            return pd.DataFrame(data)
+        else:
+            return pd.read_csv(cache_path)
+    except Exception:
+        if verbose:
+            click.echo(f"Failed to load cache from {cache_path}")
+        return pd.DataFrame()
+
+
+def _fetch_csv(endpoint: str, params: dict, verbose: bool = False) -> pd.DataFrame:
+    # if verbose:
+    #     click.echo(f"Downloading CSV from {endpoint} with params {params}")
+    resp = requests.get(endpoint, params=params, timeout=10)
+    if resp.status_code != 200:
+        raise click.ClickException(f"Lookup failed ({resp.status_code}): {resp.text}")
+    try:
+        return pd.read_csv(io.StringIO(resp.text))
+    except Exception:
+        raise click.ClickException("Lookup returned invalid CSV")
+
+
+def load_lookup_df(
+    endpoint: str,
+    query: str,
+    verbose: bool = False,
+    local_only: bool = False,
+    skip_cache_update: bool = False,
+) -> pd.DataFrame:
+    """
+    Use the local cache (`config.LBH_LOOKUP_CACHE`) if fresh (age < `config.LBH_LOOKUP_CACHE_TTL`);
+    otherwise fetch from `endpoint`.
+
+    On fetch success:
+        - Return DataFrame.
+
+    On fetch failure:
+        - Fall back to the cache (even if stale)
+
+    On cache failure:
+        - Return empty DataFrame.
+
+    Args:
+        endpoint: CSV endpoint URL.
+        query: Optional query string to pass to the endpoint (e.g., `q`).
+        verbose: If True, log cache and fetch behavior.
+        local_only: If True, skip remote fetch and use only local cache.
+        skip_cache_update: If True, do not update cache after successful fetch. (Only applies if `local_only=False`.)
+
+    Returns:
+        A pandas DataFrame with columns normalized to lower-case (may be empty).
+    """
+    cache_path = config.LBH_LOOKUP_CACHE
+
+    # Fresh cache
+    if os.path.exists(cache_path) and _cache_is_fresh(
+        cache_path, ttl_seconds=config.LBH_LOOKUP_CACHE_TTL
+    ):
+        if verbose:
+            click.echo(f"Using fresh cache at {cache_path}")
+        return _load_df_from_cache(cache_path, verbose=verbose)
+
+    # Skip fetch remote if local_only
+    if not local_only:
+        try:
+            df = _fetch_csv(endpoint, params={"q": query}, verbose=verbose)
+            df.columns = [str(c).strip().lower() for c in df.columns]
+            if not skip_cache_update:
+                _write_cache_from_df(cache_path, df, verbose=verbose)
+            return df
+        except click.ClickException as e:
+            if verbose:
+                click.echo(f"Remote lookup failed: {e}. Falling back to cache/sample.")
+        except Exception as e:
+            if verbose:
+                click.echo(f"Remote error: {e}. Falling back to cache/sample.")
+
+    # Fallback to cache (even if stale)
+    if os.path.exists(cache_path):
+        if verbose:
+            click.echo(f"Using cached data from {cache_path}")
+        return _load_df_from_cache(cache_path, verbose=verbose)
+
+    # Last resort: generate empty DataFrame
+    if verbose:
+        click.echo(f"No cache found at {cache_path}.")
+    return pd.DataFrame()

llmboost_hub/utils/model_utils.py

@@ -0,0 +1,76 @@
+import os
+from typing import List
+
+
+def path_has_files(path: str) -> bool:
+    """
+    Return True if 'path' exists, is a directory, and contains at least one file
+    in its subtree (current dir or any nested subdirectory).
+
+    Args:
+        path: Absolute or relative directory path.
+
+    Returns:
+        True if there is at least one regular file somewhere under 'path'; False otherwise.
+    """
+    # Fast-fail if not a directory
+    if not os.path.isdir(path):
+        return False
+    # Walk the tree and short-circuit as soon as we see any file
+    for _, _, files in os.walk(path):
+        if files:
+            return True
+    return False
+
+
+def is_model_downloaded(models_root: str, model_id: str) -> bool:
+    """
+    Check if the models root contains a downloaded model directory for `model_id`.
+
+    Args:
+        models_root: Root directory that stores models (e.g., `config.LBH_MODELS`).
+        model_id: Model identifier; may include `org/name` or plain name.
+
+    Returns:
+        True if `models_root/model_id` exists and contains at least one file; False otherwise.
+    """
+    return path_has_files(os.path.join(models_root, str(model_id or "")))
+
+
+def get_model_name_from_model_id(model_id: str) -> str:
+    """
+    Get name of model from a Hugging Face style model id.
+    Applies following transformations:
+        - Remove repo/organization prefix (if present; splits on '/')
+    Converts any 'org/name' to just 'name'.
+
+    Examples:
+        'Org/Model_Name' -> 'Model_name'
+        'model.name'     -> 'model.name'
+
+    Args:
+        model_id: Model id such as 'org/name' or 'name'.
+
+    Returns:
+        Normalized model name.
+    """
+    parts = str(model_id or "").split("/")
+    return parts[-1]  # get last part after '/'
+
+
+def get_repo_from_model_id(model_id: str) -> str:
+    """
+    Extract the organization/user (repo owner) from a Hugging Face style model id.
+
+    Examples:
+        'org/name' -> 'org'
+        'name'     -> ''  (no org present)
+
+    Args:
+        model_id: Model id such as 'org/name' or 'name'.
+
+    Returns:
+        The repo/organization segment if present; otherwise an empty string.
+    """
+    parts = str(model_id or "").split("/")
+    return parts[-2] if len(parts) >= 2 else ""

llmboost_hub/utils/signature.py

@@ -0,0 +1,3 @@
+# Copyright 2024, MangoBoost, Inc. All rights reserved.
+
+BUILD_DATE = "9999-99-99"

llmboost_hub-0.1.1.dist-info/METADATA

@@ -0,0 +1,203 @@
+Metadata-Version: 2.4
+Name: llmboost_hub
+Version: 0.1.1
+Summary: A lightweight CLI tool for managing LLMBoost model images and environments.
+Author-email: Harish Kambhampaty <harish.kambhampaty@mangoboost.io>
+License: Copyright 2025, MangoBoost, Inc. All rights reserved.
+        
+        Redistribution and modification in source and binary forms is 
+        strictly prohibited.
+        
+        THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+        EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+        IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+        PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+        CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+        EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+        PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+        PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+        OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+        (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+        OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+        
+Project-URL: Homepage, https://github.com/mangoboost/llmboost_hub
+Project-URL: Documentation, https://docs.mangoboost.io/llmboost_hub
+Project-URL: Source, https://github.com/mangoboost/llmboost_hub
+Keywords: LLM,Docker,CLI,HPC,AI,LLMBoost
+Classifier: Programming Language :: Python :: 3
+Classifier: Operating System :: OS Independent
+Classifier: Intended Audience :: Developers
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: click>=8.0
+Requires-Dist: requests>=2.31
+Requires-Dist: tabulate>=0.9
+Requires-Dist: rich>=13.7
+Requires-Dist: docker>=7.0
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: pandas>=2.0
+Requires-Dist: licensing
+Requires-Dist: bcrypt>=4.0
+Requires-Dist: huggingface_hub>=0.23
+Requires-Dist: black>=23.1.0
+Provides-Extra: build
+Requires-Dist: build>=1.0.0; extra == "build"
+Requires-Dist: wheel>=0.42; extra == "build"
+Requires-Dist: setuptools>=68.0; extra == "build"
+Requires-Dist: twine>=4.0; extra == "build"
+Provides-Extra: compile
+Requires-Dist: nuitka>=2.1; extra == "compile"
+Provides-Extra: obfuscate
+Requires-Dist: pyarmor>=8.5; extra == "obfuscate"
+Dynamic: license-file
+
+# [LLMBoost Hub (lbh)](https://docs.mangoboost.io/llmboost_hub/)
+
+Manage LLMBoost™ model containers and environments to run, serve, and tune large language models.
+
+---
+
+## Pre-requisites
+
+### Dependencies:
+- Python 3.10+
+- Docker 27.3.1+
+- NVIDIA GPU: [nvidia-docker2](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html) or AMD GPU: [ROCm 6.3+](https://rocm.docs.amd.com/en/latest/Installation_Guide/Installation-Guide.html)
+
+### Install LLMBoost Hub:
+
+```bash
+pip install llmboost_hub
+
+# Verify installation
+lbh --version
+```
+
+Note: This document uses `lbh` interchangeably with `llmboost_hub`.
+
+### Login to Hugging Face and Docker:
+```bash
+huggingface-cli login     # or set HF_TOKEN env var
+docker login -u <your_docker_username>
+```
+
+---
+
+## Quick start
+
+One-liner to start serving a model (automatically downloads image and model, if needed):
+```bash
+lbh serve <Repo/Model-Name> # Full model name (including repository or organization name) must match the name from https://huggingface.co
+```
+
+For example:
+```bash
+lbh serve meta-llama/Llama-3.1-8B-Instruct
+```
+
+Basic workflow:
+```bash
+lbh login # authenticate LLMBoost license
+lbh search <model> # find supported models, regex-style match
+lbh prep <Repo/Model-Name> # download image and model assets
+lbh run <Repo/Model-Name> # start container
+lbh serve <Repo/Model-Name> # start LLMBoost server inside container
+lbh test <Repo/Model-Name> # send test request
+lbh stop <Repo/Model-Name> # stop container
+```
+
+For more details, see the [Command Reference](#command-reference) section below.
+
+Shell completions:
+```bash
+eval "$(lbh completions)"                 # current shell
+lbh completions [--venv|--profile]        # persist for venv or profile
+```
+
+---
+
+## Configuration
+
+*llmboost_hub* uses the following environment variables:
+
+- `LBH_HOME`: base directory for all *llmboost_hub* data. (defaults: (host) `~/.llmboost_hub` <- (container) `/llmboost_hub`)
+- `LBH_MODELS`: directory for storing and retrieving model assets. (default: `$LBH_HOME/models`)
+- `LBH_WORKSPACE`: mounted user workspace for manually transferring files out of containers. (defaults: (host) `$LBH_HOME/workspace` <- (container) `/user_workspace`)
+
+Notes:
+- A configuation file is stored at `$LBH_HOME/config.yaml` with all the above mentioned settings (and other advanced settings). 
+  - Precedence order for settings: Environment variables > Configuration file > Defaults
+- `LBH_HOME` can only be changed by setting the env var (or in `~/.bashrc`). 
+  - WARNING: Changing `LBH_HOME` will cause a new data directory to be used, and all configuration will be reset.
+- `HF_TOKEN` is injected automatically when set.
+
+---
+
+## Command Reference
+
+Use `lbh -h` for a summary of all commands, and `lbh [COMMAND] -h` for help with a specific command and all available options.
+
+Use `lbh -v [COMMAND]` for verbose output with any command; shows useful diagnostic info for troubleshooting.
+
+- `lbh login`
+  - Reads `$LBH_LICENSE_PATH` or prompts for a token.
+  - Validates online and saves the license file.
+
+- `lbh search <model>`
+  - Fetches latest models supported by LLMBoost.
+  - Filters to available GPU.
+
+- `lbh list [model]`
+  - Lists local images joined with lookup.
+  - Shows status: 
+    - pending: model not prepared; docker image or model assets missing
+    - stopped: model prepared but container not running
+    - running: container running but idling
+    - initializing: container running and starting LLMBoost server
+    - serving: LLMBoost server ready to accept requests
+    - tuning: autotuner running
+
+- `lbh prep <Repo/Model-Name> [--only-verify] [--fresh]`
+  - Pulls the image and downloads HF assets.
+  - `--only-verify` checks digests and sizes.
+  - `--fresh` removes existing image and re-downloads model assets from Hugging Face.
+
+- `lbh run <Repo/Model-Name> [OPTIONS] -- [DOCKER FLAGS...]`
+  - Resolves and starts the container detached.
+  - Mounts `$LBH_HOME` and `$LBH_WORKSPACE`. Injects HF_TOKEN.
+  - NVIDIA GPUs use `--gpus all`. AMD maps `/dev/dri` and `/dev/kfd`.
+  - Useful options: 
+    - `--image <image>`: override docker image.
+    - `--model_path <model_path>`: override model assets path.
+    - `--restart`: restarts container, if already running.
+    - Pass extra docker flags after `--`.
+
+- `lbh serve <Repo/Model-Name> [--host 0.0.0.0] [--port 8080] [--detached] [--force]`
+  - Starts LLMBoost server inside the container.
+  - Waits until ready, unless `--detached`.
+  - `--force` skips GPU utilization checks (use if GPU utilization is incorrectly reported by NVidia or AMD GPU drivers).
+
+- `lbh test <Repo/Model-Name> [--query "..."] [-t N] [--host 127.0.0.1] [--port 8080]`
+  - Sends a test request to `/v1/chat/completions`.
+
+- `lbh attach <Repo/Model-Name> [-c <container name or ID>]`
+  - Opens a shell in the running container.
+
+- `lbh stop <Repo/Model-Name> [-c <container name or ID>]`
+  - Stops the container.
+
+- `lbh status [model]`
+  - Shows status and model.
+
+- `lbh tune <Repo/Model-Name> [--metrics throughput] [--detached] [--image <image>]`
+  - Runs the autotuner. 
+  - Store results to `$LBH_HOME/inference.db`, and loads this on next `lbh serve`.
+
+---
+
+## Support
+
+- Docs: https://docs.mangoboost.io/llmboost_hub/
+- Website: https://docs.mangoboost.io/llmboost_hub/
+- Email: support@mangoboost.io

llmboost_hub-0.1.1.dist-info/RECORD

@@ -0,0 +1,31 @@
+llmboost_hub/cli.py,sha256=dmaQcfbDGpmzfdfS0w7nV92aObUPFBlWf7vf-WLMgz8,1293
+llmboost_hub/commands/attach.py,sha256=9thBR2RJuMbDRoxjCo4KodAsFWqjd6NhBiSrWOvUho4,2619
+llmboost_hub/commands/chat.py,sha256=A0fnInJB5qHqRRBNW0nOsK75zcZCvnzNrscOw8fKSas,2225
+llmboost_hub/commands/completions.py,sha256=y__rLnB-bLQqqZVnXyjwU9CqNv-fcoOjwqTEApRtAUo,8066
+llmboost_hub/commands/list.py,sha256=bCABrBQvIn_V4W6Y45BEmI87Z5Zf1onXOFzIOgyc8W8,10146
+llmboost_hub/commands/login.py,sha256=gqQONUUk9VbErK1TBHbVrSmq__7tk-wtQUksJxQJCrg,2612
+llmboost_hub/commands/prep.py,sha256=31YskfZfzNP2PqeP-3k7sOZYvAq6AhviGV3bBGAfj2s,22164
+llmboost_hub/commands/run.py,sha256=T8CUXGoAJjJRE4rB9RHFOiRMYTuv9XjoZnsI5yuXqUY,17103
+llmboost_hub/commands/search.py,sha256=9YVWjuww2nNndSoBeIpIyW9iwSFZJXt48xATt92DjT0,6035
+llmboost_hub/commands/serve.py,sha256=tcVJE8hZJnV5GCb4vi58KpRs-Kc9Yrz7ICZvWjpsqS8,9618
+llmboost_hub/commands/status.py,sha256=JXAHkRWzn6WKIC4I_KEGjT9WSQuPUJ4e1PwH9ZiiA6E,1193
+llmboost_hub/commands/stop.py,sha256=HFIA8jc06Y4kquNHP8zuNAX1ll8ZraHJ77GlWt3BHW0,1973
+llmboost_hub/commands/test_cmd.py,sha256=Vc5WjpapFYzJtOqBbEd-56E2_OvfJOCrwymzM7P8rs4,1707
+llmboost_hub/commands/tune.py,sha256=kJNICp4ezOBMrShNtOFCOIZj4pnwv9JE0nHa7ZT9IIE,12107
+llmboost_hub/utils/config.py,sha256=KPR5pOWlzDbXqFdb_afSres39IMZCOrCW3Xm-qzaQf8,7602
+llmboost_hub/utils/container_utils.py,sha256=zZEHBqj9bQ628CLhTGeaUggCsNEXwQbOdU81GTseDTo,4313
+llmboost_hub/utils/fs_utils.py,sha256=vcRnvh4pNB2TsU2D5962lB9UEqD0xkPR5w0lpYD6xwI,1016
+llmboost_hub/utils/generate_sample_lookup.py,sha256=-osdHYe_SbvOIsFMRi5hFU00qpllBuqItthLd09hVxs,4108
+llmboost_hub/utils/gpu_info.py,sha256=bqFSDYyGbobG-wYIlBJYzSnf4HoSbp0JYiAzKwqfTg8,7635
+llmboost_hub/utils/license_checker.py,sha256=hJXvetEeMAzN4xACsUSLNPM7VYVhGtn_HvUENO34d4o,13300
+llmboost_hub/utils/license_wrapper.py,sha256=AhJHZmdkVUGUlohTfwBkYF8zsp6pXx7Owo9EL8Qgw-w,3024
+llmboost_hub/utils/llmboost_version.py,sha256=J-j-u0itpEFT6irdmWmixQqYMadNl1X91TxUmoiLHMI,22
+llmboost_hub/utils/lookup_cache.py,sha256=IZ5S-FMn4WfGlW7dpmL14w8sdfJD8wtn51iGnQCntpY,4029
+llmboost_hub/utils/model_utils.py,sha256=818M0d1Y8pQ0gL15bvc1SzAouQhf7jl6zSAu07MgBSM,2243
+llmboost_hub/utils/signature.py,sha256=TlukT_dJ9xXNvV6LUggABMC5-yajPAT7OqmFP9Vg_z0,83
+llmboost_hub-0.1.1.dist-info/licenses/LICENSE,sha256=Bs8AHb5xcW2e0Wh2rAnxkHkVtG6nO1GKuP01mvhNSiw,881
+llmboost_hub-0.1.1.dist-info/METADATA,sha256=rMiyXf7VqXbBmKrVJi0-ccVE-XcmAHcgkksBXNYyeHY,7702
+llmboost_hub-0.1.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+llmboost_hub-0.1.1.dist-info/entry_points.txt,sha256=zOGSTbgg7ct5h-x-MkhAFfCP-ljKLgXoucw5B1GmuUo,83
+llmboost_hub-0.1.1.dist-info/top_level.txt,sha256=GiutvBTlDMdnX4lMved7C4gi60bjCzqS7vYlKzm_f0w,13
+llmboost_hub-0.1.1.dist-info/RECORD,,

llmboost_hub-0.1.1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.9.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

llmboost_hub-0.1.1.dist-info/entry_points.txt

@@ -0,0 +1,3 @@
+[console_scripts]
+lbh = llmboost_hub.cli:main
+llmboost_hub = llmboost_hub.cli:main

llmboost_hub-0.1.1.dist-info/licenses/LICENSE

@@ -0,0 +1,16 @@
+Copyright 2025, MangoBoost, Inc. All rights reserved.
+
+Redistribution and modification in source and binary forms is 
+strictly prohibited.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

llmboost_hub-0.1.1.dist-info/top_level.txt

@@ -0,0 +1 @@
+llmboost_hub