kalavai-client 0.7.4__py3-none-any.whl → 0.7.5__py3-none-any.whl

kalavai_client/__init__.py

@@ -1,2 +1,2 @@
 
-__version__ = "0.7.4"
+__version__ = "0.7.5"

kalavai_client/assets/default_pool_config.yaml

@@ -3,18 +3,19 @@ server:
   ip_address: null
   location: null
   name: "kalavai_cluster"
+  mtu: 1280
 
 core:
   # Deploy systems
   deploy:
   - volcano
   - cert_manager
-  - rocm
+  #- rocm
   - watcher
   - hami
   #- lago
   #- opencost
-  - prometheus
+  #- prometheus
   #- kuberay
   #- minio
   # "Kalavai API endpoint"

kalavai_client/assets/docker-compose-template.yaml

@@ -4,6 +4,7 @@ services:
     image: gravitl/netclient:v0.90.0
     container_name: {{vpn_name}}
     platform: linux/{{target_platform}}
+    privileged: true
     cap_add:
       - NET_ADMIN
       - SYS_MODULE
@@ -11,6 +12,7 @@ services:
     environment:
       - HOST_NAME={{node_name}}
       - TOKEN={{vpn_token}}
+      - MTU={{mtu}}
     volumes:
       - /dev/net/tun:/dev/net/tun
     restart: unless-stopped
@@ -58,6 +60,7 @@ services:
       {% if node_labels %}
       --extra="{{node_labels}}"
       {% endif %}
+      --mtu={{mtu}}
 
     # volumes:
     # - {{k3s_path}}:/var/lib/rancher/k3s # Persist data

kalavai_client/assets/model_deployment_values.yaml

@@ -0,0 +1,68 @@
+Qwen/Qwen3-4B-Instruct-2507-FP8:
+  size: 6 # should include weights + context window + batch size memory requirements
+  cuda:
+    template: vllm
+    values:
+      gpu_backend: "cuda"
+      litellm_key: ""
+      workers: ""
+      model_id: Qwen/Qwen3-4B-Instruct-2507-FP8
+      working_memory: 14
+      hf_token: ""
+      cpus: 2
+      gpus: 1
+      memory: 12
+      tensor_parallel_size: 1
+      pipeline_parallel_size: 1
+      tool_call_parser: hermes
+      extra: "--dtype float16 --enforce-eager --quantization fp8"
+  rocm:
+    template: vllm
+    values:
+      gpu_backend: "rocm"
+      litellm_key: ""
+      workers: ""
+      model_id: Qwen/Qwen3-4B-Instruct-2507-FP8
+      working_memory: 14
+      hf_token: ""
+      cpus: 2
+      gpus: 1
+      memory: 12
+      tensor_parallel_size: 1
+      pipeline_parallel_size: 1
+      tool_call_parser: hermes
+      extra: "--dtype float16 --enforce-eager"
+Qwen/Qwen3-0.6B:
+  size: 4 # should include weights + context window + batch size memory requirements
+  cuda:
+    template: vllm
+    values:
+      gpu_backend: "cuda"
+      litellm_key: ""
+      workers: ""
+      model_id: Qwen/Qwen3-0.6B
+      working_memory: 8
+      hf_token: ""
+      cpus: 2
+      gpus: 1
+      memory: 8
+      tensor_parallel_size: 1
+      pipeline_parallel_size: 1
+      tool_call_parser: hermes
+      extra: "--dtype float16"
+  rocm:
+    template: vllm
+    values:
+      gpu_backend: "rocm"
+      litellm_key: ""
+      workers: ""
+      model_id: Qwen/Qwen3-0.6B
+      working_memory: 8
+      hf_token: ""
+      cpus: 2
+      gpus: 1
+      memory: 8
+      tensor_parallel_size: 1
+      pipeline_parallel_size: 1
+      tool_call_parser: hermes
+      extra: "--dtype float16"

kalavai_client/bridge_api.py

@@ -59,6 +59,7 @@ from kalavai_client.core import (
     add_node_labels,
     get_node_labels,
     generate_worker_package,
+    get_deployment_values,
     TokenType
 )
 from kalavai_client.utils import (
@@ -446,6 +447,20 @@ def job_deploy(request: DeployJobRequest, api_key: str = Depends(verify_api_key)
     )
     return result
 
+@app.get("/get_deployment_values",
+    operation_id="get_deployment_values",
+    summary="Get deployment template values for a given model",
+    description="Given a model id from Huggingface, return the deployment template values required to load the model instance in the pool, including number of workers, number of gpus and gpu backend.",
+    tags=["job_management"],
+    response_description="Deployment template values")
+def get_deployment_template_values(model_id: str, api_key: str = Depends(verify_api_key)):
+    """
+    Get the deployment template values for a given model id:
+    
+    - **model_id**: Model id from Huggingface type mode
+    """
+    return get_deployment_values(model_id=model_id)
+
 @app.post("/delete_job",
     operation_id="delete_job",
     summary="Terminate and remove a job from the pool",

kalavai_client/cli.py

@@ -62,7 +62,8 @@ from kalavai_client.core import (
     uncordon_nodes,
     TokenType,
     unregister_pool,
-    update_pool
+    update_pool,
+    get_deployment_values
 )
 from kalavai_client.utils import (
     check_gpu_drivers,
@@ -396,7 +397,7 @@ def pool__list(*others, user_only=False):
 
 
 @arguably.command
-def pool__start(*others,  pool_config_file=None, apps: list=None, platform="amd64", ip_address: str=None, location: str=None, app_values: str=None, pool_config_values: str=None, non_interactive: bool=False):
+def pool__start(*others,  pool_config_file=None, apps: list=None, mtu: str=None, platform="amd64", ip_address: str=None, location: str=None, app_values: str=None, pool_config_values: str=None, non_interactive: bool=False):
 
     """
     Start Kalavai pool and start/resume sharing resources.
@@ -439,7 +440,8 @@ def pool__start(*others,  pool_config_file=None, apps: list=None, platform="amd6
         target_platform=platform,
         pool_config_file=pool_config_file,
         apps=apps,
-        num_gpus=input_gpus(non_interactive=non_interactive)
+        num_gpus=input_gpus(non_interactive=non_interactive),
+        mtu=mtu
     )
 
     if "warning" in result:
@@ -482,7 +484,7 @@ def pool__token(*others, admin=False, user=False, worker=False):
     return join_token
 
 @arguably.command
-def pool__check_token(token, *others, public=False):
+def pool__check_token(token, *others, public=False, verbose=False):
     """
     Utility to check the validity of a join token
     """
@@ -490,12 +492,14 @@ def pool__check_token(token, *others, public=False):
     if "error" in result:
         console.log(f"[red]Error in token: {result}")
         return False
+    if verbose:
+        console.log(json.dumps(result["data"], indent=2))
     
     console.log("[green]Token format is correct")
     return True
 
 @arguably.command
-def pool__join(token, *others, platform="amd64", node_name=None, non_interactive=False):
+def pool__join(token, *others, mtu=None, platform="amd64", node_name=None, non_interactive=False):
     """
     Join Kalavai pool and start/resume sharing resources.
 
@@ -549,7 +553,8 @@ def pool__join(token, *others, platform="amd64", node_name=None, non_interactive
         token=token,
         node_name=node_name,
         num_gpus=num_gpus,
-        ip_address=ip_address
+        ip_address=ip_address,
+        mtu=mtu
     )
     if "error" in result:
         console.log(f"[red]Error when connecting: {result}")
@@ -1165,6 +1170,12 @@ def job__delete(name, *others, force_namespace: str=None):
         console.log(f"{result}")
 
 
+@arguably.command
+def job__model_requirements(model_id: str, *others):
+    values = get_deployment_values(model_id=model_id)
+    console.log(values)
+
+
 @arguably.command
 def job__estimate(
     *others, 
@@ -1180,6 +1191,7 @@ def job__estimate(
     precision_bytes = {
         "fp32": 4,
         "fp16": 2,
+        "fp8": 1,
         "int8": 1,
         "int4": 0.5
     }

kalavai_client/core.py

@@ -2,7 +2,7 @@ import os
 import yaml
 import time
 from collections import defaultdict
-import math
+import random
 import uuid
 import socket
 import ipaddress
@@ -69,8 +69,7 @@ from kalavai_client.env import (
     USER_NODE_LABEL,
     DEFAULT_WATCHER_PORT,
     HELM_APPS_FILE,
-    HELM_APPS_VALUES,
-    POOL_CONFIG_DEFAULT_VALUES,
+    MODEL_DEPLOYMENT_VALUES_MAPPING,
     POOL_CONFIG_TEMPLATE,
     FORBIDEDEN_IPS,
     DEFAULT_POOL_CONFIG_TEMPLATE
@@ -105,6 +104,70 @@ class TokenType(Enum):
     WORKER = 2
 
 
+def get_deployment_values(model_id: str):
+    """
+    Given a model ID and the resources in the pool, identify key
+    computing values required to deploy the model.
+    - GPU_BACKEND: rocm or cuda
+    - WORKERS: number of nodes to use
+    - 
+    """
+    # get hardcoded deployment values (per model)
+    with open(MODEL_DEPLOYMENT_VALUES_MAPPING, "r") as f:
+        mapping = yaml.safe_load(f)
+
+    def _parse_memory_str(memory: str):
+        memory = memory.replace("G", "")
+        return int(memory)
+    
+    def _get_num_workers(memory_values: list[int], size):
+        workers = 0
+        available_memory = 0
+        for gpu_mem in memory_values:
+            available_memory += gpu_mem
+            workers += 1
+            if available_memory >= size:
+                break
+        return workers
+    
+    # get resources
+    if model_id in mapping:
+        model_size = mapping[model_id]["size"]
+        # get gpus and extract available memory
+        nvidia_gpu_mems = []
+        amd_gpu_mems = []
+        backends = set()
+        for node_name, gpus in load_gpu_models():
+            for gpu in gpus["gpus"]:
+                if "nvidia" in gpu["model"].lower():
+                    nvidia_gpu_mems.append(_parse_memory_str(gpu["memory"]))
+                    backends.add("cuda")
+                else:
+                    amd_gpu_mems.append(_parse_memory_str(gpu["memory"]))
+                    backends.add("rocm")
+        nvidia_gpu_mems = sorted(nvidia_gpu_mems, reverse=False)
+        amd_gpu_mems = sorted(amd_gpu_mems, reverse=False)
+        # calculate num workers required
+        if sum(nvidia_gpu_mems) >= model_size and sum(amd_gpu_mems) < model_size:
+            gpu_backend = "cuda"
+            num_workers = _get_num_workers(memory_values=nvidia_gpu_mems, size=model_size)
+        elif sum(amd_gpu_mems) >= model_size and sum(nvidia_gpu_mems) < model_size:
+            gpu_backend = "rocm"
+            num_workers = _get_num_workers(memory_values=amd_gpu_mems, size=model_size)
+        else:
+            gpu_backend = random.choice(list(backends))
+            num_workers = _get_num_workers(
+                memory_values=amd_gpu_mems if gpu_backend == "rocm" else nvidia_gpu_mems,
+                size=model_size
+            )
+        # populate selected template
+        mapping[model_id][gpu_backend]["values"]["workers"] = num_workers
+        mapping[model_id][gpu_backend]["values"]["pipeline_parallel_size"] = num_workers
+
+        return mapping[model_id][gpu_backend]
+    return None
+
+
 def set_schedulable(schedulable, node_names):
     """
     Delete job in the cluster
@@ -545,7 +608,7 @@ def check_token(token, public=False):
         if public:
             if data[PUBLIC_LOCATION_KEY] is None:
                 raise ValueError("Token is not valid for public pools. Did you start the cluster with a public_location?")
-        return {"status": True}
+        return {"status": True, "data": data}
     except Exception as e:
         return {"error": str(e)}
     
@@ -671,7 +734,8 @@ def join_pool(
         num_gpus=None,
         node_name=None,
         ip_address=None,
-        target_platform="amd64"
+        target_platform="amd64",
+        mtu="1420"
 ):
     compatibility = check_worker_compatibility()
     if len(compatibility["issues"]) > 0:
@@ -714,6 +778,7 @@ def join_pool(
         pool_token=kalavai_token,
         num_gpus=num_gpus,
         vpn_token=public_location,
+        mtu=mtu,
         node_name=node_name,
         node_labels=node_labels)
     
@@ -763,6 +828,7 @@ def create_pool(
         token_mode: TokenType=TokenType.USER,
         num_gpus: int=-1,
         node_name: str=None,
+        mtu: str=None,
         apps: list=[]
     ):
 
@@ -794,6 +860,7 @@ def create_pool(
         ip_address = config_values["server"]["ip_address"] if ip_address is None else ip_address
         location = config_values["server"]["location"] if location is None else location
         target_platform = config_values["server"]["platform"] if target_platform is None else target_platform
+        mtu = config_values["server"]["mtu"] if mtu is None else mtu
         app_values = config_values["core"]
         post_config_values = config_values["pool"]
         deploy_apps = {
@@ -805,6 +872,8 @@ def create_pool(
         return {"error": f"Error when loading pool config. Missing format? {str(e)}"}
 
     # Generate docker compose recipe
+    if ip_address is None:
+        ip_address = "0.0.0.0"
     generate_compose_config(
         target_platform=target_platform,
         role="server",
@@ -812,7 +881,8 @@ def create_pool(
         node_ip_address=ip_address,
         num_gpus=num_gpus,
         node_name=node_name,
-        node_labels=node_labels
+        node_labels=node_labels,
+        mtu=mtu
     )
 
     # start server

kalavai_client/env.py

@@ -56,6 +56,7 @@ USER_WORKSPACE_TEMPLATE = resource_path("kalavai_client/assets/user_workspace.ya
 DEFAULT_USER_WORKSPACE_VALUES = resource_path("kalavai_client/assets/user_workspace_values.yaml")
 POOL_CONFIG_TEMPLATE = resource_path("kalavai_client/assets/pool_config_template.yaml")
 POOL_CONFIG_DEFAULT_VALUES = resource_path("kalavai_client/assets/pool_config_values.yaml")
+MODEL_DEPLOYMENT_VALUES_MAPPING = resource_path("kalavai_client/assets/model_deployment_values.yaml")
 # user specific config files
 USER_TEMPLATES_FOLDER = user_path("templates", create_path=True)
 USER_LOCAL_SERVER_FILE = user_path(".server")

kalavai_client/utils.py

@@ -129,7 +129,7 @@ def extract_auth_token(headers):
         return {"error": str(e)}
 
 
-def generate_compose_config(role, node_name, target_platform="amd64", write_to_file=True, node_ip_address="0.0.0.0", num_gpus=0, node_labels=None, pool_ip=None, vpn_token=None, pool_token=None):
+def generate_compose_config(role, node_name, mtu="1420", target_platform="amd64", write_to_file=True, node_ip_address="0.0.0.0", num_gpus=0, node_labels=None, pool_ip=None, vpn_token=None, pool_token=None):
     
     if node_labels is not None:
         node_labels = " ".join([f"--node-label {key}={value}" for key, value in node_labels.items()])
@@ -140,6 +140,7 @@ def generate_compose_config(role, node_name, target_platform="amd64", write_to_f
         "service_name": DEFAULT_CONTAINER_NAME,
         "vpn": vpn_token is not None,
         "vpn_name": DEFAULT_VPN_CONTAINER_NAME,
+        "mtu": mtu,
         "node_ip_address": node_ip_address,
         "pool_ip": pool_ip,
         "pool_token": pool_token,

{kalavai_client-0.7.4.dist-info → kalavai_client-0.7.5.dist-info}/METADATA

@@ -1,20 +1,21 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.4
 Name: kalavai-client
-Version: 0.7.4
+Version: 0.7.5
 Summary: Client app for kalavai platform
-License: Apache-2.0
+License-Expression: Apache-2.0
+License-File: LICENSE
 Keywords: LLM,platform
 Author: Carlos Fernandez Musoles
 Author-email: carlos@kalavai.net
 Maintainer: Carlos Fernandez Musoles
 Maintainer-email: carlos@kalavai.net
 Requires-Python: >=3.10
-Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
 Provides-Extra: dev
 Requires-Dist: Pillow (==10.3.0)
 Requires-Dist: arguably (>=1.2.5)

kalavai_client-0.7.5.dist-info/RECORD

@@ -0,0 +1,25 @@
+kalavai_client/__init__.py,sha256=2yNV33mxa4mw0NRZdX1rF9R0X_uQaQl59WtFaMqS_ss,22
+kalavai_client/__main__.py,sha256=WQUfxvRsBJH5gsCJg8pLz95QnZIj7Ol8psTO77m0QE0,73
+kalavai_client/assets/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+kalavai_client/assets/apps.yaml,sha256=z5lLrBbR8KVP5Ou3pbD6cELYWATs9VJuRu-PPe1GMNk,7007
+kalavai_client/assets/default_pool_config.yaml,sha256=FLfT0i-xPgEZhleTOZmfyFYrB9fAEMOmZ0YGLyJr0OA,1879
+kalavai_client/assets/docker-compose-gui.yaml,sha256=OAVO0ohaCpDB9FGeih0yAbVNwUfDtaCzssZ25uiuJyA,787
+kalavai_client/assets/docker-compose-template.yaml,sha256=8giwNUVbVGxF_YppNMIsS7evWZ8qb4n2enK22u-x6Pk,1920
+kalavai_client/assets/model_deployment_values.yaml,sha256=PKD9DGJnRRwYfk6pO4cmGN9gIgylHX3IFW2sDweEGdU,1698
+kalavai_client/assets/nginx.conf,sha256=drVVCg8GHucz7hmt_BI6giAhK92OV71257NTs3LthwM,225
+kalavai_client/assets/pool_config_template.yaml,sha256=MhBZQsEMKrBgbUVSKgIGmXWhybeGKG6l5XvJb38y5GI,577
+kalavai_client/assets/user_workspace.yaml,sha256=wDvlMYknOPABAEo0dsQwU7bac8iubjAG9tdkFbJZ5Go,476
+kalavai_client/assets/user_workspace_values.yaml,sha256=G0HOzQUxrDMCwuW9kbWUZaKMzDDPVwDwzBHCL2Xi2ZM,542
+kalavai_client/auth.py,sha256=EB3PMvKUn5_KAQkezkEHEt-OMZXyfkZguIQlUFkEHcA,3243
+kalavai_client/bridge_api.py,sha256=6NGRIbu3CtWyT75XCFKkLSfllF54vGJmYBjnX9qiWX8,27657
+kalavai_client/bridge_models.py,sha256=bq6vQNTI1py7e_1YgnBZhorFsAKoBqBVN7nRukCuQRE,2960
+kalavai_client/cli.py,sha256=7EdEHjJxoS0lgmWFyT72k7rl3-qJ7ODz9ju_wZGYB1I,48312
+kalavai_client/cluster.py,sha256=Z2PIXbZuSAv9xmw-MyZP1M41BpVMpirLzG51bqGA-zc,13548
+kalavai_client/core.py,sha256=BUlCOeQnXTAqwjD4rxWIshk9Sf_s-QJy9UB6cUbcECo,38906
+kalavai_client/env.py,sha256=0L5gfEo5KY8gflrW-rSADx10ffDa-8gXmmrGWztKUd8,3099
+kalavai_client/utils.py,sha256=L0FQJT8EucabPAbxGwzMqeawJdj4wTZqmmZ0OdsyVwM,13470
+kalavai_client-0.7.5.dist-info/METADATA,sha256=u04xcVwDMzNCp-xU_6TE791B5xQlSc2jnqnzRB587vI,13175
+kalavai_client-0.7.5.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
+kalavai_client-0.7.5.dist-info/entry_points.txt,sha256=9T6D45gxwzfVbglMm1r6XPdXuuZdHfy_7fCeu2jUphc,50
+kalavai_client-0.7.5.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+kalavai_client-0.7.5.dist-info/RECORD,,

{kalavai_client-0.7.4.dist-info → kalavai_client-0.7.5.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 2.1.3
+Generator: poetry-core 2.2.1
 Root-Is-Purelib: true
 Tag: py3-none-any

kalavai_client-0.7.4.dist-info/RECORD

@@ -1,24 +0,0 @@
-kalavai_client/__init__.py,sha256=tEb-PZ0vatpsK8g60stVimfhqVV4BkN49tzmy1jwTv8,22
-kalavai_client/__main__.py,sha256=WQUfxvRsBJH5gsCJg8pLz95QnZIj7Ol8psTO77m0QE0,73
-kalavai_client/assets/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-kalavai_client/assets/apps.yaml,sha256=z5lLrBbR8KVP5Ou3pbD6cELYWATs9VJuRu-PPe1GMNk,7007
-kalavai_client/assets/default_pool_config.yaml,sha256=zfzefofkG9rld9OxSFWbvcVGkz5TNRq0mw6M8X0tVYA,1865
-kalavai_client/assets/docker-compose-gui.yaml,sha256=OAVO0ohaCpDB9FGeih0yAbVNwUfDtaCzssZ25uiuJyA,787
-kalavai_client/assets/docker-compose-template.yaml,sha256=vW7GhOl_PaUodehJk8qajOlE0deZXrPc7qizg5SeYyc,1859
-kalavai_client/assets/nginx.conf,sha256=drVVCg8GHucz7hmt_BI6giAhK92OV71257NTs3LthwM,225
-kalavai_client/assets/pool_config_template.yaml,sha256=MhBZQsEMKrBgbUVSKgIGmXWhybeGKG6l5XvJb38y5GI,577
-kalavai_client/assets/user_workspace.yaml,sha256=wDvlMYknOPABAEo0dsQwU7bac8iubjAG9tdkFbJZ5Go,476
-kalavai_client/assets/user_workspace_values.yaml,sha256=G0HOzQUxrDMCwuW9kbWUZaKMzDDPVwDwzBHCL2Xi2ZM,542
-kalavai_client/auth.py,sha256=EB3PMvKUn5_KAQkezkEHEt-OMZXyfkZguIQlUFkEHcA,3243
-kalavai_client/bridge_api.py,sha256=0TvAGgsyfMkbcVqoPKsjhXQLo06WetBH93mZH-pOM7U,26921
-kalavai_client/bridge_models.py,sha256=bq6vQNTI1py7e_1YgnBZhorFsAKoBqBVN7nRukCuQRE,2960
-kalavai_client/cli.py,sha256=JDzBZF63S4GGiII47QWbPgKPlYP2mNDcrucj9s-QMUI,47968
-kalavai_client/cluster.py,sha256=Z2PIXbZuSAv9xmw-MyZP1M41BpVMpirLzG51bqGA-zc,13548
-kalavai_client/core.py,sha256=-YITkLcwhkwJLU1Z-I9yEW_e4QVgt8-sQ8A0_uMW7f4,36184
-kalavai_client/env.py,sha256=Uu4ojOsssATfKAC4dKMbR3_kc6hRN6w9ZRsaC7DvzAo,2997
-kalavai_client/utils.py,sha256=1mz-dzoJhZ9GJKU7jiGYBC1tP37SXHvxToMqqEir8R0,13438
-kalavai_client-0.7.4.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-kalavai_client-0.7.4.dist-info/METADATA,sha256=RKD51W-Kd_aX5LhW1YbKLhcsRWrndU3aorWL2NXhbWY,13154
-kalavai_client-0.7.4.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-kalavai_client-0.7.4.dist-info/entry_points.txt,sha256=9T6D45gxwzfVbglMm1r6XPdXuuZdHfy_7fCeu2jUphc,50
-kalavai_client-0.7.4.dist-info/RECORD,,