kalavai-client 0.6.12__tar.gz → 0.6.14__tar.gz

{kalavai_client-0.6.12 → kalavai_client-0.6.14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: kalavai-client
-Version: 0.6.12
+Version: 0.6.14
 Summary: Client app for kalavai platform
 License: Apache-2.0
 Keywords: LLM,platform

kalavai_client-0.6.14/kalavai_client/__init__.py

@@ -0,0 +1,2 @@
+
+__version__ = "0.6.14"

{kalavai_client-0.6.12 → kalavai_client-0.6.14}/kalavai_client/assets/apps.yaml

@@ -152,7 +152,7 @@ releases:
     - name: replicas
       value: 1
     - name: image_tag
-      value: "v2025.05.2"
+      value: "v2025.06.6"
     - name: deployment.in_cluster
       value: "True"
     - name: deployment.kalavai_username_key

{kalavai_client-0.6.12 → kalavai_client-0.6.14}/kalavai_client/assets/docker-compose-template.yaml

@@ -18,6 +18,7 @@ services:
 {%if command %}
   {{service_name}}:
     image: docker.io/bundenth/kalavai-runner:gpu-latest
+    pull_policy: always
     container_name: {{service_name}}
   {% if vpn %}
     depends_on:
@@ -35,6 +36,9 @@ services:
     {% endif %}
       --node_name="{{node_name}}"
       --node_ip="{{node_ip_address}}"
+    {% if random_suffix %}
+      --random_suffix="{{random_suffix}}"
+    {% endif %}
       {% if command == "server" %}
       --port_range="30000-32767"
       {% else %}

kalavai_client-0.6.14/kalavai_client/bridge_api.py

@@ -0,0 +1,518 @@
+"""
+Core kalavai service.
+Used as a bridge between the kalavai-client app and the reflex frontend
+"""
+from fastapi import FastAPI, HTTPException, Depends
+from starlette.requests import Request
+import uvicorn
+
+from kalavai_client.bridge_models import (
+    CreatePoolRequest,
+    InvitesRequest,
+    JoinPoolRequest,
+    StopPoolRequest,
+    DeployJobRequest,
+    DeleteJobRequest,
+    JobDetailsRequest,
+    NodesActionRequest,
+    NodeLabelsRequest,
+    GetNodeLabelsRequest
+)
+from kalavai_client.core import (
+    create_pool,
+    join_pool,
+    attach_to_pool,
+    send_invites,
+    stop_pool,
+    fetch_devices,
+    fetch_resources,
+    fetch_job_names,
+    fetch_gpus,
+    fetch_job_details,
+    fetch_job_logs,
+    fetch_job_templates,
+    fetch_job_defaults,
+    deploy_job,
+    delete_job,
+    authenticate_user,
+    load_user_session,
+    user_logout,
+    is_connected,
+    list_available_pools,
+    is_agent_running,
+    is_server,
+    pause_agent,
+    resume_agent,
+    get_ip_addresses,
+    get_pool_token,
+    delete_nodes,
+    cordon_nodes,
+    uncordon_nodes,
+    add_node_labels,
+    get_node_labels,
+    TokenType
+)
+from kalavai_client.utils import load_user_id
+
+app = FastAPI(
+    title="Kalavai Bridge API",
+    description="API for managing Kalavai pools, jobs, and nodes",
+    version="1.0.0",
+    docs_url="/docs",
+    redoc_url="/redoc",
+)
+
+################################
+## API Key Validation methods ##
+################################
+async def verify_api_key(request: Request):
+    """
+    Verify the API key from the request headers.
+    The API key must match the user ID.
+    """
+    user_id = load_user_id()
+    if user_id is None:
+        return None
+    api_key = request.headers.get("X-API-KEY")
+    if api_key != user_id:
+        raise HTTPException(status_code=401, detail="Request requires API Key")
+    return api_key
+
+@app.post("/create_pool", 
+    summary="Create a new pool",
+    description="Creates a new pool with the specified configuration",
+    response_description="Result of pool creation")
+def pool_create(request: CreatePoolRequest, api_key: str = Depends(verify_api_key)):
+    """
+    Create a new pool with the following parameters:
+    
+    - **cluster_name**: Name of the cluster
+    - **ip_address**: IP address for the pool
+    - **app_values**: Application configuration values
+    - **num_gpus**: Number of GPUs to allocate
+    - **node_name**: Name of the node
+    - **only_registered_users**: Whether to restrict to registered users
+    - **location**: Location of the pool
+    - **description**: Pool description
+    - **token_mode**: Token type for authentication
+    - **frontend**: Whether this is a frontend request
+    """
+    result = create_pool(
+        cluster_name=request.cluster_name,
+        ip_address=request.ip_address,
+        app_values=request.app_values,
+        num_gpus=request.num_gpus,
+        node_name=request.node_name,
+        only_registered_users=request.only_registered_users,
+        location=request.location,
+        description=request.description,
+        token_mode=request.token_mode,
+        frontend=request.frontend
+    )
+    return result
+
+@app.post("/join_pool",
+    summary="Join an existing pool",
+    description="Join a pool using a token",
+    response_description="Result of joining the pool")
+def pool_join(request: JoinPoolRequest, api_key: str = Depends(verify_api_key)):
+    """
+    Join a pool with the following parameters:
+    
+    - **token**: Pool join token
+    - **ip_address**: IP address for the node
+    - **node_name**: Name of the node
+    - **num_gpus**: Number of GPUs to allocate
+    - **frontend**: Whether this is a frontend request
+    """
+    result = join_pool(
+        token=request.token,
+        num_gpus=request.num_gpus,
+        node_name=request.node_name,
+        ip_address=request.ip_address,
+        frontend=request.frontend
+    )
+    return result
+
+@app.post("/attach_to_pool",
+    summary="Attach to an existing pool",
+    description="Attach to a pool using a token",
+    response_description="Result of attaching to the pool")
+def pool_attach(request: JoinPoolRequest, api_key: str = Depends(verify_api_key)):
+    """
+    Attach to a pool with the following parameters:
+    
+    - **token**: Pool token
+    - **node_name**: Name of the node
+    - **frontend**: Whether this is a frontend request
+    """
+    result = attach_to_pool(
+        token=request.token,
+        node_name=request.node_name,
+        frontend=request.frontend
+    )
+    return result
+
+@app.post("/stop_pool",
+    summary="Stop a pool",
+    description="Stop the current pool",
+    response_description="Result of stopping the pool")
+def pool_stop(request: StopPoolRequest, api_key: str = Depends(verify_api_key)):
+    """
+    Stop the pool with the following parameters:
+    
+    - **skip_node_deletion**: Whether to skip node deletion
+    """
+    result = stop_pool(
+        skip_node_deletion=request.skip_node_deletion
+    )
+    return result
+
+@app.post("/delete_nodes",
+    summary="Delete nodes",
+    description="Delete specified nodes from the pool",
+    response_description="Result of node deletion")
+def device_delete(request: NodesActionRequest, api_key: str = Depends(verify_api_key)):
+    """
+    Delete nodes with the following parameters:
+    
+    - **nodes**: List of node names to delete
+    """
+    result = delete_nodes(
+        nodes=request.nodes
+    )
+    return result
+
+@app.post("/cordon_nodes",
+    summary="Cordon nodes",
+    description="Mark nodes as unschedulable",
+    response_description="Result of cordoning nodes")
+def device_cordon(request: NodesActionRequest, api_key: str = Depends(verify_api_key)):
+    """
+    Cordon nodes with the following parameters:
+    
+    - **nodes**: List of node names to cordon
+    """
+    result = cordon_nodes(
+        nodes=request.nodes
+    )
+    return result
+
+@app.post("/uncordon_nodes",
+    summary="Uncordon nodes",
+    description="Mark nodes as schedulable",
+    response_description="Result of uncordoning nodes")
+def device_uncordon(request: NodesActionRequest, api_key: str = Depends(verify_api_key)):
+    """
+    Uncordon nodes with the following parameters:
+    
+    - **nodes**: List of node names to uncordon
+    """
+    result = uncordon_nodes(
+        nodes=request.nodes
+    )
+    return result
+
+@app.get("/get_pool_token",
+    summary="Get pool token",
+    description="Get a token for the pool",
+    response_description="Pool token")
+def get_token(mode: int, api_key: str = Depends(verify_api_key)):
+    """
+    Get pool token with the following parameters:
+    
+    - **mode**: Token type mode
+    """
+    return get_pool_token(mode=TokenType(mode))
+
+@app.get("/fetch_devices",
+    summary="Fetch devices",
+    description="Get list of available devices",
+    response_description="List of devices")
+def get_devices(api_key: str = Depends(verify_api_key)):
+    """Get list of available devices"""
+    return fetch_devices()
+
+@app.post("/send_pool_invites",
+    summary="Send pool invites",
+    description="Send invites to join the pool",
+    response_description="Result of sending invites")
+def send_pool_invites(request: InvitesRequest, api_key: str = Depends(verify_api_key)):
+    """
+    Send pool invites with the following parameters:
+    
+    - **invitees**: List of invitee identifiers
+    """
+    return send_invites(invitees=request.invitees)
+
+@app.get("/fetch_resources",
+    summary="Fetch resources",
+    description="Get available resources",
+    response_description="Resource information")
+def resources(api_key: str = Depends(verify_api_key)):
+    """Get available resources"""
+    return fetch_resources()
+
+@app.get("/fetch_job_names",
+    summary="Fetch job names",
+    description="Get list of job names",
+    response_description="List of job names")
+def job_names(api_key: str = Depends(verify_api_key)):
+    """Get list of job names"""
+    return fetch_job_names()
+
+@app.get("/fetch_gpus",
+    summary="Fetch GPUs",
+    description="Get list of available GPUs",
+    response_description="List of GPUs")
+def gpus(available: bool = False, api_key: str = Depends(verify_api_key)):
+    """
+    Get list of GPUs with the following parameters:
+    
+    - **available**: Whether to show only available GPUs
+    """
+    return fetch_gpus(available=available)
+
+@app.post("/fetch_job_details",
+    summary="Fetch job details",
+    description="Get details for specified jobs",
+    response_description="Job details")
+def job_details(request: JobDetailsRequest, api_key: str = Depends(verify_api_key)):
+    """
+    Get job details with the following parameters:
+    
+    - **jobs**: List of jobs to get details for
+    """
+    return fetch_job_details(jobs=request.jobs)
+
+@app.get("/fetch_job_logs",
+    summary="Fetch job logs",
+    description="Get logs for a specific job",
+    response_description="Job logs")
+def job_logs(
+    job_name: str,
+    force_namespace: str = None,
+    pod_name: str = None,
+    tail: int = 100,
+    api_key: str = Depends(verify_api_key)
+):
+    """
+    Get job logs with the following parameters:
+    
+    - **job_name**: Name of the job
+    - **force_namespace**: Optional namespace override
+    - **pod_name**: Optional pod name
+    - **tail**: Number of log lines to return
+    """
+    return fetch_job_logs(
+        job_name=job_name,
+        force_namespace=force_namespace,
+        pod_name=pod_name,
+        tail=tail
+    )
+
+@app.get("/fetch_job_templates",
+    summary="Fetch job templates",
+    description="Get available job templates",
+    response_description="List of job templates")
+def job_templates(api_key: str = Depends(verify_api_key)):
+    """Get available job templates"""
+    return fetch_job_templates()
+
+@app.get("/fetch_job_defaults",
+    summary="Fetch job defaults",
+    description="Get default values for a job template",
+    response_description="Job default values")
+def job_templates(name: str, api_key: str = Depends(verify_api_key)):
+    """
+    Get job defaults with the following parameters:
+    
+    - **name**: Name of the job template
+    """
+    return fetch_job_defaults(name=name)
+
+@app.post("/deploy_job",
+    summary="Deploy job",
+    description="Deploy a new job",
+    response_description="Result of job deployment")
+def job_deploy(request: DeployJobRequest, api_key: str = Depends(verify_api_key)):
+    """
+    Deploy a job with the following parameters:
+    
+    - **template_name**: Name of the job template
+    - **values**: Job configuration values
+    - **force_namespace**: Optional namespace override
+    - **target_labels**: Optional target node labels
+    """
+    result = deploy_job(
+        template_name=request.template_name,
+        values_dict=request.values,
+        force_namespace=request.force_namespace,
+        target_labels=request.target_labels
+    )
+    return result
+
+@app.post("/delete_job",
+    summary="Delete job",
+    description="Delete a job",
+    response_description="Result of job deletion")
+def job_delete(request: DeleteJobRequest, api_key: str = Depends(verify_api_key)):
+    """
+    Delete a job with the following parameters:
+    
+    - **name**: Name of the job to delete
+    - **force_namespace**: Optional namespace override
+    """
+    result = delete_job(
+        name=request.name,
+        force_namespace=request.force_namespace
+    )
+    return result
+
+@app.get("/authenticate_user",
+    summary="Authenticate user",
+    description="Authenticate a user",
+    response_description="Authentication result")
+def user_authenticate(user_id: str, api_key: str = Depends(verify_api_key)):
+    """
+    Authenticate user with the following parameters:
+    
+    - **user_id**: User identifier
+    """
+    result = authenticate_user(
+        user_id=user_id
+    )
+    return result
+
+@app.get("/load_user_session",
+    summary="Load user session",
+    description="Load the current user session",
+    response_description="User session information")
+def user_session(api_key: str = Depends(verify_api_key)):
+    """Load the current user session"""
+    result = load_user_session()
+    return result
+
+@app.get("/user_logout",
+    summary="User logout",
+    description="Log out the current user",
+    response_description="Logout result")
+def logout_user():
+    """Log out the current user"""
+    result = user_logout()
+    return result
+
+@app.get("/is_connected",
+    summary="Check connection",
+    description="Check if connected to a pool",
+    response_description="Connection status")
+def pool_connected():
+    """Check if connected to a pool"""
+    result = is_connected()
+    return result
+
+@app.get("/is_agent_running",
+    summary="Check agent status",
+    description="Check if the agent is running",
+    response_description="Agent status")
+def agent_running():
+    """Check if the agent is running"""
+    result = is_agent_running()
+    return result
+
+@app.get("/is_server",
+    summary="Check server status",
+    description="Check if running as server",
+    response_description="Server status")
+def server():
+    """Check if running as server"""
+    result = is_server()
+    return result
+
+@app.post("/pause_agent",
+    summary="Pause agent",
+    description="Pause the agent",
+    response_description="Result of pausing agent")
+def agent_pause():
+    """Pause the agent"""
+    result = pause_agent()
+    return result
+
+@app.post("/resume_agent",
+    summary="Resume agent",
+    description="Resume the agent",
+    response_description="Result of resuming agent")
+def agent_resume():
+    """Resume the agent"""
+    result = resume_agent()
+    return result
+
+@app.get("/get_ip_addresses",
+    summary="Get IP addresses",
+    description="Get available IP addresses",
+    response_description="List of IP addresses")
+def ip_addresses(subnet: str = None, api_key: str = Depends(verify_api_key)):
+    """
+    Get IP addresses with the following parameters:
+    
+    - **subnet**: Optional subnet to filter by
+    """
+    result = get_ip_addresses(subnet=subnet)
+    return result
+
+@app.get("/list_available_pools",
+    summary="List available pools",
+    description="Get list of available pools",
+    response_description="List of available pools")
+def pool_connected(user_only: bool = False, api_key: str = Depends(verify_api_key)):
+    """
+    List available pools with the following parameters:
+    
+    - **user_only**: Whether to show only user's pools
+    """
+    result = list_available_pools(user_only=user_only)
+    return result
+
+@app.post("/add_node_labels",
+    summary="Add node labels",
+    description="Add labels to a node",
+    response_description="Result of adding labels")
+def node_labels(request: NodeLabelsRequest, api_key: str = Depends(verify_api_key)):
+    """
+    Add node labels with the following parameters:
+    
+    - **node_name**: Name of the node
+    - **labels**: Dictionary of labels to add
+    """
+    result = add_node_labels(
+        node_name=request.node_name,
+        labels=request.labels
+    )
+    return result
+
+@app.post("/get_node_labels",
+    summary="Get node labels",
+    description="Get labels for specified nodes",
+    response_description="Node labels")
+def node_labels_get(request: GetNodeLabelsRequest, api_key: str = Depends(verify_api_key)):
+    """
+    Get node labels with the following parameters:
+    
+    - **node_names**: List of node names to get labels for
+    """
+    result = get_node_labels(
+        node_names=request.node_names
+    )
+    return result
+
+def run_api(host="0.0.0.0", port=8001, log_level="critical"):
+    uvicorn.run(
+        app,
+        host=host,
+        port=port,
+        log_level=log_level
+    )
+
+if __name__ == "__main__":
+    run_api()
+    

kalavai_client-0.6.14/kalavai_client/bridge_models.py

@@ -0,0 +1,53 @@
+from pydantic import BaseModel, Field
+from typing import List, Dict, Optional
+
+from kalavai_client.core import Job, TokenType
+
+
+class InvitesRequest(BaseModel):
+    invitees: list[str] = Field(description="List of user identifiers to invite to the pool")
+
+class CreatePoolRequest(BaseModel):
+    cluster_name: str = Field(description="Name of the cluster to create")
+    ip_address: str = Field(description="IP address for the pool")
+    app_values: dict = Field(None, description="Application configuration values")
+    num_gpus: int = Field(None, description="Number of GPUs to allocate")
+    node_name: str = Field(None, description="Name of the node")
+    only_registered_users: bool = Field(False, description="Whether to restrict access to registered users only")
+    location: str = Field(None, description="Geographic location of the pool")
+    token_mode: TokenType = Field(TokenType.USER, description="Token type for authentication")
+    description: str = Field("", description="Description of the pool")
+    frontend: bool = Field(False, description="Whether this is a frontend request")
+
+class NodesActionRequest(BaseModel):
+    nodes: list[str] = Field(description="List of node names to perform the action on")
+
+class JoinPoolRequest(BaseModel):
+    token: str = Field(description="Token to join the pool")
+    ip_address: str = Field(None, description="IP address for the node")
+    node_name: str = Field(None, description="Name of the node")
+    num_gpus: int = Field(None, description="Number of GPUs to allocate")
+    frontend: bool = Field(False, description="Whether this is a frontend request")
+class JobDetailsRequest(BaseModel):
+    jobs: list[Job] = Field(description="List of jobs to get details for")
+
+
+class StopPoolRequest(BaseModel):
+    skip_node_deletion: bool = Field(False, description="Whether to skip node deletion when stopping the pool")
+
+class DeployJobRequest(BaseModel):
+    template_name: str = Field(description="Name of the job template to use")
+    values: dict = Field(description="Job configuration values")
+    force_namespace: str = Field(None, description="Optional namespace override")
+    target_labels: dict[str, str] = Field(None, description="Optional target node labels")
+
+class DeleteJobRequest(BaseModel):
+    name: str = Field(description="Name of the job to delete")
+    force_namespace: str = Field(None, description="Optional namespace override")
+
+class NodeLabelsRequest(BaseModel):
+    node_name: str = Field(description="Name of the node to add labels to")
+    labels: Dict[str, str] = Field(description="Dictionary of labels to add to the node")
+
+class GetNodeLabelsRequest(BaseModel):
+    node_names: List[str] = Field(description="List of node names to get labels for")

{kalavai_client-0.6.12 → kalavai_client-0.6.14}/kalavai_client/cli.py

@@ -186,7 +186,9 @@ def input_gpus(non_interactive=False):
     try:
         has_gpus = check_gpu_drivers()
         if has_gpus:
-            max_gpus = int(run_cmd("nvidia-smi -L | wc -l").decode())
+            max_gpus = len(
+                [r for r in run_cmd("nvidia-smi -L").decode().split("\n") if len(r.strip())>0]
+            )
             if non_interactive:
                 num_gpus = max_gpus
             else:

{kalavai_client-0.6.12 → kalavai_client-0.6.14}/kalavai_client/cluster.py

@@ -1,4 +1,5 @@
 import os
+import platform
 import time
 from pathlib import Path
 from abc import ABC, abstractmethod
@@ -96,7 +97,7 @@ class dockerCluster(Cluster):
         # wait for container to be setup
         while True:
             try:
-                run_cmd(f"docker cp {self.container_name}:/etc/rancher/k3s/k3s.yaml {self.kubeconfig_file} >/dev/null 2>&1")
+                run_cmd(f"docker cp {self.container_name}:/etc/rancher/k3s/k3s.yaml {self.kubeconfig_file}", hide_output=True)
                 break
             except:
                 pass
@@ -115,15 +116,15 @@ class dockerCluster(Cluster):
     def update_dependencies(self, dependencies_file=None, debug=False, retries=3):
         if dependencies_file is not None:
             self.dependencies_file = dependencies_file
-        if debug:
-            output = ""
-        else:
-            output = " >/dev/null 2>&1"
         while True:
             try:
                 home = user_path("")
-                run_cmd(f"docker run --rm --net=host -v {home}:{home} ghcr.io/helmfile/helmfile:v0.169.2 helmfile sync --file {self.dependencies_file} --kubeconfig {self.kubeconfig_file} {output}")
-                #run_cmd(f"helmfile sync --file {self.dependencies_file} --kubeconfig {self.kubeconfig_file} {output}")
+                # convert path on host to path on container (will be different in windows os)
+                target_path = "/cache/kalavai"
+                kubeconfig_path = f"{target_path}/{Path(self.kubeconfig_file).name}"
+                dependencies_path = f"{target_path}/{Path(self.dependencies_file).name}"
+
+                run_cmd(f"docker run --rm --net=host -v {home}:{target_path} ghcr.io/helmfile/helmfile:v0.169.2 helmfile sync --file {dependencies_path} --kubeconfig {kubeconfig_path}", hide_output=not debug)
                 break
             except Exception as e:
                 if retries > 0:
@@ -142,11 +143,18 @@ class dockerCluster(Cluster):
     def is_agent_running(self):
         if not os.path.isfile(self.compose_file):
             return False
-        status = self.container_name in run_cmd(f"docker compose -f {self.compose_file} ps --services --status=running").decode()
-        if not status:
+        try:
+            status = self.container_name in run_cmd(f"docker compose -f {self.compose_file} ps --services --status=running").decode()
+            if not status:
+                return False
+            if "windows" in platform.system().lower():
+                status = (0 == os.system(f'docker exec {self.container_name} ps aux | findstr /n /c:"k3s server" /c:"k3s agent"'))
+            else:
+                status = (0 == os.system(f'docker exec {self.container_name} ps aux | grep -v grep | grep -E "k3s (server|agent)"'))
+            return status
+        except Exception as e:
+            print(f"Error when checking agent. Is Docker installed and running?\n\n{str(e)}")
             return False
-        status = (0 == os.system(f'docker exec {self.container_name} ps aux | grep -v grep | grep -E "k3s (server|agent)"'))
-        return status
 
     def is_seed_node(self):
         if not os.path.isfile(self.compose_file):
@@ -154,7 +162,7 @@ class dockerCluster(Cluster):
         if not self.is_agent_running():
             return False
         try:
-            run_cmd(f"docker container exec {self.container_name} cat /var/lib/rancher/k3s/server/node-token >/dev/null 2>&1")
+            run_cmd(f"docker container exec {self.container_name} cat /var/lib/rancher/k3s/server/node-token", hide_output=True)
             return True
         except:
             return False
@@ -162,8 +170,12 @@ class dockerCluster(Cluster):
     def is_cluster_init(self):
         if not os.path.isfile(self.compose_file):
             return False
-        status = self.container_name in run_cmd(f"docker compose -f {self.compose_file} ps --services --all").decode()
-        return status
+        try:
+            status = self.container_name in run_cmd(f"docker compose -f {self.compose_file} ps --services --all").decode()
+            return status
+        except Exception as e:
+            print(f"Error when checking cluster. Is Docker installed and running?\n\n{str(e)}")
+            return False
 
     def pause_agent(self):
         status = False
@@ -177,7 +189,6 @@ class dockerCluster(Cluster):
     def restart_agent(self):
         try:
             run_cmd(f'docker compose -f {self.compose_file} start')
-
         except:
             pass
         time.sleep(5)
@@ -186,7 +197,6 @@ class dockerCluster(Cluster):
     def get_cluster_token(self):
         if self.is_seed_node():
             return run_cmd(f"docker container exec {self.container_name} cat /var/lib/rancher/k3s/server/node-token").decode()
-            #return run_cmd("sudo k3s token create --kubeconfig /etc/rancher/k3s/k3s.yaml --ttl 0").decode()
         else:
             return None
     
@@ -231,7 +241,7 @@ class k3sCluster(Cluster):
             flannel_iface = f"--flannel-iface {self.default_flannel_iface}"
         else:
             flannel_iface = ""
-        run_cmd(f'curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION="{self.kube_version}" INSTALL_K3S_EXEC="server --node-ip {ip_address} --node-external-ip {ip_address} {flannel_iface} --flannel-backend wireguard-native {node_labels}" sh - >/dev/null 2>&1')
+        run_cmd(f'curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION="{self.kube_version}" INSTALL_K3S_EXEC="server --node-ip {ip_address} --node-external-ip {ip_address} {flannel_iface} --flannel-backend wireguard-native {node_labels}" sh - ', hide_output=True)
         run_cmd(f"sudo cp /etc/rancher/k3s/k3s.yaml {self.kubeconfig_file}")
         run_cmd(f"sudo chown $USER {self.kubeconfig_file}")
 
@@ -245,8 +255,8 @@ class k3sCluster(Cluster):
             flannel_iface = f"--flannel-iface {self.default_flannel_iface}"
         else:
             flannel_iface = ""
-        command = f'curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION="{self.kube_version}" INSTALL_K3S_EXEC="agent --token {token} --server https://{url}:6443 --node-name {node_name} --node-ip {ip_address} --node-external-ip {ip_address} {flannel_iface} {node_labels}" sh - >/dev/null 2>&1'
-        run_cmd(command)
+        command = f'curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION="{self.kube_version}" INSTALL_K3S_EXEC="agent --token {token} --server https://{url}:6443 --node-name {node_name} --node-ip {ip_address} --node-external-ip {ip_address} {flannel_iface} {node_labels}" sh - '
+        run_cmd(command, hide_output=True)
         
 
     def update_dependencies(self, dependencies_file=None, debug=False, retries=3):
@@ -270,13 +280,13 @@ class k3sCluster(Cluster):
 
     def remove_agent(self):
         try:
-            run_cmd('/usr/local/bin/k3s-uninstall.sh >/dev/null 2>&1')
-            run_cmd('sudo rm -r /etc/rancher/node/ >/dev/null 2>&1')
+            run_cmd('/usr/local/bin/k3s-uninstall.sh', hide_output=True)
+            run_cmd('sudo rm -r /etc/rancher/node/', hide_output=True)
             return True
         except:
             pass
         try:
-            run_cmd('/usr/local/bin/k3s-agent-uninstall.sh >/dev/null 2>&1')
+            run_cmd('/usr/local/bin/k3s-agent-uninstall.sh', hide_output=True)
             return True
         except:
             pass
@@ -296,12 +306,12 @@ class k3sCluster(Cluster):
     def pause_agent(self):
         status = False
         try:
-            run_cmd('sudo systemctl stop k3s >/dev/null 2>&1')
+            run_cmd('sudo systemctl stop k3s', hide_output=True)
             status = True
         except:
             pass
         try:
-            run_cmd('sudo systemctl stop k3s-agent >/dev/null 2>&1')
+            run_cmd('sudo systemctl stop k3s-agent', hide_output=True)
             status = True
         except:
             pass
@@ -309,11 +319,11 @@ class k3sCluster(Cluster):
 
     def restart_agent(self):
         try:
-            run_cmd('sudo systemctl start k3s >/dev/null 2>&1')
+            run_cmd('sudo systemctl start k3s', hide_output=True)
         except:
             pass
         try:
-            run_cmd('sudo systemctl start k3s-agent >/dev/null 2>&1')
+            run_cmd('sudo systemctl start k3s-agent', hide_output=True)
         except:
             pass
         return self.is_agent_running()

{kalavai_client-0.6.12 → kalavai_client-0.6.14}/kalavai_client/core.py

@@ -33,6 +33,7 @@ from kalavai_client.utils import (
     get_public_seeds,
     load_template,
     is_storage_compatible,
+    get_max_gpus,
     NODE_NAME_KEY,
     MANDATORY_TOKEN_FIELDS,
     PUBLIC_LOCATION_KEY,
@@ -157,7 +158,7 @@ def check_seed_compatibility():
     logs = []
     # docker
     try:
-        run_cmd("docker version >/dev/null 2>&1")
+        run_cmd("docker ps", hide_output=True)
     except:
         logs.append("[red]Docker not installed. Install instructions:\n")
         logs.append("   Linux: https://docs.docker.com/engine/install/\n")
@@ -170,7 +171,7 @@ def check_worker_compatibility():
     logs = []
     # docker
     try:
-        run_cmd("docker version >/dev/null 2>&1")
+        run_cmd("docker ps", hide_output=True)
     except:
         logs.append("[red]Docker not installed. Install instructions:\n")
         logs.append("   Linux: https://docs.docker.com/engine/install/\n")
@@ -594,16 +595,6 @@ def attach_to_pool(token, node_name=None):
 
     return cluster_name
 
-def get_max_gpus():
-    try:
-        has_gpus = check_gpu_drivers()
-        if has_gpus:
-            return int(run_cmd("nvidia-smi -L | wc -l").decode())
-        else:
-            return 0
-    except:
-        return 0
-
 def generate_worker_package(num_gpus=0, node_name=None, ip_address="0.0.0.0", storage_compatible=True):
     # get pool data from token  
     token = get_pool_token(mode=TokenType.WORKER)
@@ -772,10 +763,9 @@ def create_pool(
         node_name=node_name,
         node_labels=node_labels
     )
-    
+
     # start server
     CLUSTER.start_seed_node()
-    
     while not CLUSTER.is_agent_running():
         time.sleep(10)
     

{kalavai_client-0.6.12 → kalavai_client-0.6.14}/kalavai_client/utils.py

@@ -73,17 +73,20 @@ KALAVAI_AUTH = KalavaiAuth(
 
 ####### Methods to check OS compatibility ########
 def check_gpu_drivers():
-    value = run_cmd("command -v nvidia-smi")
-    if len(value.decode("utf-8")) == 0:
-        # no nvidia installed, no need to check nvidia any further
-        return False
-    else:
-        # check drivers are set correctly
-        try:
-            value = run_cmd("nvidia-smi")
-            return True
-        except:
-            raise ("Nvidia not configured properly. Please check your drivers are installed and configured")
+    value = run_cmd("nvidia-smi", hide_output=True)
+    return len(value.decode("utf-8")) == 0
+
+def get_max_gpus():
+    try:
+        has_gpus = check_gpu_drivers()
+        if has_gpus:
+            return len(
+                [r for r in run_cmd("nvidia-smi -L").decode().split("\n") if len(r.strip())>0]
+            )
+        else:
+            return 0
+    except:
+        return 0
 
 def is_storage_compatible():
     """
@@ -92,6 +95,9 @@ def is_storage_compatible():
     Exclude: WSL
     """
     try:
+        import platform
+        if "windows" in platform.system().lower():
+            return True
         flagged = any([
             "microsoft" in run_cmd("cat /proc/version").decode().lower()
         ])
@@ -120,6 +126,7 @@ def generate_compose_config(role, node_name, write_to_file=True, node_ip_address
         "num_gpus": num_gpus,
         "k3s_path": f"{CONTAINER_HOST_PATH}/{rand_suffix}/k3s",
         "etc_path": f"{CONTAINER_HOST_PATH}/{rand_suffix}/etc",
+        "random_suffix": rand_suffix,
         "node_labels": node_labels,
         "flannel_iface": DEFAULT_FLANNEL_IFACE if vpn_token is not None else "",
         "user_id": load_user_id()
@@ -231,9 +238,17 @@ def validate_poolconfig(poolconfig_file):
             return False
     return True
 
-def run_cmd(command):
+def run_cmd(command, hide_output=False):
     try:
-        return_value = subprocess.check_output(command, shell=True, executable="/bin/bash")
+        import platform
+        if "windows" in platform.system().lower():
+            if hide_output:
+                command = command + " > $nul 2>&1"
+            return_value = subprocess.check_output(command, shell=True)
+        else:
+            if hide_output:
+                command = command + " >/dev/null 2>&1"
+            return_value = subprocess.check_output(command, shell=True, executable="/bin/bash")
         return return_value
     except OSError as error:
         return error # for exit code

{kalavai_client-0.6.12 → kalavai_client-0.6.14}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name            = "kalavai-client"
-version         = "0.6.12"
+version         = "0.6.14"
 authors = [
   {name = "Carlos Fernandez Musoles", email = "carlos@kalavai.net"}
 ]

kalavai_client-0.6.12/kalavai_client/__init__.py

@@ -1,2 +0,0 @@
-
-__version__ = "0.6.12"

kalavai_client-0.6.12/kalavai_client/bridge_api.py

@@ -1,276 +0,0 @@
-"""
-Core kalavai service.
-Used as a bridge between the kalavai-client app and the reflex frontend
-"""
-from fastapi import FastAPI, HTTPException, Depends
-from starlette.requests import Request
-import uvicorn
-
-from kalavai_client.bridge_models import (
-    CreatePoolRequest,
-    InvitesRequest,
-    JoinPoolRequest,
-    StopPoolRequest,
-    DeployJobRequest,
-    DeleteJobRequest,
-    JobDetailsRequest,
-    NodesActionRequest,
-    NodeLabelsRequest,
-    GetNodeLabelsRequest
-)
-from kalavai_client.core import (
-    create_pool,
-    join_pool,
-    attach_to_pool,
-    send_invites,
-    stop_pool,
-    fetch_devices,
-    fetch_resources,
-    fetch_job_names,
-    fetch_gpus,
-    fetch_job_details,
-    fetch_job_logs,
-    fetch_job_templates,
-    fetch_job_defaults,
-    deploy_job,
-    delete_job,
-    authenticate_user,
-    load_user_session,
-    user_logout,
-    is_connected,
-    list_available_pools,
-    is_agent_running,
-    is_server,
-    pause_agent,
-    resume_agent,
-    get_ip_addresses,
-    get_pool_token,
-    delete_nodes,
-    cordon_nodes,
-    uncordon_nodes,
-    add_node_labels,
-    get_node_labels,
-    TokenType
-)
-from kalavai_client.utils import load_user_id
-
-app = FastAPI()
-
-################################
-## API Key Validation methods ##
-################################
-async def verify_api_key(request: Request):
-    user_id = load_user_id()
-    if user_id is None:
-        return None
-    api_key = request.headers.get("X-API-KEY")
-    if api_key != user_id:
-        raise HTTPException(status_code=401, detail="Request requires API Key")
-    return api_key
-
-@app.post("/create_pool")
-def pool_create(request: CreatePoolRequest, api_key: str = Depends(verify_api_key)):
-    result = create_pool(
-        cluster_name=request.cluster_name,
-        ip_address=request.ip_address,
-        app_values=request.app_values,
-        num_gpus=request.num_gpus,
-        node_name=request.node_name,
-        only_registered_users=request.only_registered_users,
-        location=request.location,
-        description=request.description,
-        token_mode=request.token_mode,
-        frontend=request.frontend
-    )
-    return result
-
-@app.post("/join_pool")
-def pool_join(request: JoinPoolRequest, api_key: str = Depends(verify_api_key)):
-    result = join_pool(
-        token=request.token,
-        num_gpus=request.num_gpus,
-        node_name=request.node_name,
-        ip_address=request.ip_address,
-        frontend=request.frontend
-    )
-    return result
-
-@app.post("/attach_to_pool")
-def pool_attach(request: JoinPoolRequest, api_key: str = Depends(verify_api_key)):
-    result = attach_to_pool(
-        token=request.token,
-        node_name=request.node_name,
-        frontend=request.frontend
-    )
-    return result
-
-@app.post("/stop_pool")
-def pool_stop(request: StopPoolRequest, api_key: str = Depends(verify_api_key)):
-    result = stop_pool(
-        skip_node_deletion=request.skip_node_deletion
-    )
-    return result
-
-@app.post("/delete_nodes")
-def device_delete(request: NodesActionRequest, api_key: str = Depends(verify_api_key)):
-    result = delete_nodes(
-        nodes=request.nodes
-    )
-    return result
-
-@app.post("/cordon_nodes")
-def device_cordon(request: NodesActionRequest, api_key: str = Depends(verify_api_key)):
-    result = cordon_nodes(
-        nodes=request.nodes
-    )
-    return result
-
-@app.post("/uncordon_nodes")
-def device_uncordon(request: NodesActionRequest, api_key: str = Depends(verify_api_key)):
-    result = uncordon_nodes(
-        nodes=request.nodes
-    )
-    return result
-
-@app.get("/get_pool_token")
-def get_token(mode: int, api_key: str = Depends(verify_api_key)):
-
-    return get_pool_token(mode=TokenType(mode))
-
-@app.get("/fetch_devices")
-def get_devices(api_key: str = Depends(verify_api_key)):
-    return fetch_devices()
-
-@app.post("/send_pool_invites")
-def send_pool_invites(request: InvitesRequest, api_key: str = Depends(verify_api_key)):
-    return send_invites(invitees=request.invitees)
-
-@app.get("/fetch_resources")
-def resources(api_key: str = Depends(verify_api_key)):
-    return fetch_resources()
-
-@app.get("/fetch_job_names")
-def job_names(api_key: str = Depends(verify_api_key)):
-    return fetch_job_names()
-
-@app.get("/fetch_gpus")
-def gpus(available: bool = False, api_key: str = Depends(verify_api_key)):
-    return fetch_gpus(available=available)
-
-@app.post("/fetch_job_details")
-def job_details(request: JobDetailsRequest, api_key: str = Depends(verify_api_key)):
-    return fetch_job_details(jobs=request.jobs)
-
-@app.get("/fetch_job_logs")
-def job_logs(job_name: str, force_namespace: str=None, pod_name: str=None, tail: int=100, api_key: str = Depends(verify_api_key)):
-    return fetch_job_logs(
-        job_name=job_name,
-        force_namespace=force_namespace,
-        pod_name=pod_name,
-        tail=tail
-    )
-
-@app.get("/fetch_job_templates")
-def job_templates(api_key: str = Depends(verify_api_key)):
-    return fetch_job_templates()
-
-@app.get("/fetch_job_defaults")
-def job_templates(name: str, api_key: str = Depends(verify_api_key)):
-    return fetch_job_defaults(name=name)
-
-@app.post("/deploy_job")
-def job_deploy(request: DeployJobRequest, api_key: str = Depends(verify_api_key)):
-    result = deploy_job(
-        template_name=request.template_name,
-        values_dict=request.values,
-        force_namespace=request.force_namespace,
-        target_labels=request.target_labels
-    )
-    return result
-
-@app.post("/delete_job")
-def job_delete(request: DeleteJobRequest, api_key: str = Depends(verify_api_key)):
-    result = delete_job(
-        name=request.name,
-        force_namespace=request.force_namespace
-    )
-    return result
-
-@app.get("/authenticate_user")
-def user_authenticate(user_id: str, api_key: str = Depends(verify_api_key)):
-    result = authenticate_user(
-        user_id=user_id
-    )
-    return result
-
-@app.get("/load_user_session")
-def user_session(api_key: str = Depends(verify_api_key)):
-    result = load_user_session()
-    return result
-
-@app.get("/user_logout")
-def logout_user():
-    result = user_logout()
-    return result
-
-@app.get("/is_connected")
-def pool_connected():
-    result = is_connected()
-    return result
-
-@app.get("/is_agent_running")
-def agent_running():
-    result = is_agent_running()
-    return result
-
-@app.get("/is_server")
-def server():
-    result = is_server()
-    return result
-
-@app.post("/pause_agent")
-def agent_pause():
-    result = pause_agent()
-    return result
-
-@app.post("/resume_agent")
-def agent_resume():
-    result = resume_agent()
-    return result
-
-@app.get("/get_ip_addresses")
-def ip_addresses(subnet: str=None, api_key: str = Depends(verify_api_key)):
-    result = get_ip_addresses(subnet=subnet)
-    return result
-
-@app.get("/list_available_pools")
-def pool_connected(user_only: bool=False, api_key: str = Depends(verify_api_key)):
-    result = list_available_pools(user_only=user_only)
-    return result
-
-@app.post("/add_node_labels")
-def node_labels(request: NodeLabelsRequest, api_key: str = Depends(verify_api_key)):
-    result = add_node_labels(
-        node_name=request.node_name,
-        labels=request.labels
-    )
-    return result
-
-@app.post("/get_node_labels")
-def node_labels_get(request: GetNodeLabelsRequest, api_key: str = Depends(verify_api_key)):
-    result = get_node_labels(
-        node_names=request.node_names
-    )
-    return result
-
-def run_api(host="0.0.0.0", port=8001, log_level="critical"):
-    uvicorn.run(
-        app,
-        host=host,
-        port=port,
-        log_level=log_level
-    )
-
-if __name__ == "__main__":
-    run_api()
-    

kalavai_client-0.6.12/kalavai_client/bridge_models.py

@@ -1,53 +0,0 @@
-from pydantic import BaseModel
-from typing import List, Dict, Optional
-
-from kalavai_client.core import Job, TokenType
-
-
-class InvitesRequest(BaseModel):
-    invitees: list[str]
-
-class CreatePoolRequest(BaseModel):
-    cluster_name: str
-    ip_address: str
-    app_values: dict = None
-    num_gpus: int = None
-    node_name: str = None
-    only_registered_users: bool = False
-    location: str = None
-    token_mode: TokenType = TokenType.USER
-    description: str = ""
-    frontend: bool = False
-
-class NodesActionRequest(BaseModel):
-    nodes: list[str]
-
-class JoinPoolRequest(BaseModel):
-    token: str
-    ip_address: str = None
-    node_name: str = None
-    num_gpus: int = None
-    frontend: bool = False
-class JobDetailsRequest(BaseModel):
-    jobs: list[Job]
-
-
-class StopPoolRequest(BaseModel):
-    skip_node_deletion: bool = False
-
-class DeployJobRequest(BaseModel):
-    template_name: str
-    values: dict
-    force_namespace: str = None
-    target_labels: dict[str, str] = None
-
-class DeleteJobRequest(BaseModel):
-    name: str
-    force_namespace: str = None
-
-class NodeLabelsRequest(BaseModel):
-    node_name: str
-    labels: Dict[str, str]
-
-class GetNodeLabelsRequest(BaseModel):
-    node_names: List[str]