kalavai-client 0.5.7__py2.py3-none-any.whl → 0.5.9__py2.py3-none-any.whl

kalavai_client/__init__.py

@@ -1,2 +1,2 @@
 
-__version__ = "0.5.7"
+__version__ = "0.5.9"

kalavai_client/assets/apps.yaml

@@ -189,13 +189,13 @@ releases:
       value: "1"
     - name: devicePlugin.deviceSplitCount
       value: "1"
-    - name: scheduler.customWebhook.port
-      value: "30498"
-    - name: scheduler.service.schedulerPort
-      value: "30498"
-    - name: scheduler.service.monitorPort
-      value: "30493"
-    - name: devicePlugin.service.httpPort
-      value: "30492"
+    # - name: scheduler.customWebhook.port
+    #   value: "30498"
+    # - name: scheduler.service.schedulerPort
+    #   value: "30498"
+    # - name: scheduler.service.monitorPort
+    #   value: "30493"
+    # - name: devicePlugin.service.httpPort
+    #   value: "30492"
     
 

kalavai_client/assets/docker-compose-template.yaml

@@ -1,55 +1,106 @@
 services:
-   {{service_name}}-{{command}}:
-      image: bundenth/kalavai-runner:gpu-latest
-      container_name: {{service_name}}
-      hostname: {{hostname}}
-      privileged: true
-      restart: unless-stopped
-      ports:
-      - "6443:6443" # kube server
-      - "8472:8472" # flannel vxlan
-      - "51820:51820" # flannel wireguard
-{% if command == "server" %}
-      - "30000-30500:30000-30500"
+{% if vpn %}
+  {{vpn_name}}:
+    image: gravitl/netclient:v0.30.0
+    container_name: {{vpn_name}}
+    cap_add:
+      - NET_ADMIN
+      - SYS_MODULE
+    network_mode: host
+    # networks:
+    # - custom-network
+    # ports:
+    # # https://docs.k3s.io/installation/requirements#inbound-rules-for-k3s-nodes
+    # - "6443:6443" # kube server
+    # - "10250:10250" # worker balancer
+    # - "8472:8472/udp" # flannel vxlan
+    # - "51820-51830:51820-51830" # flannel wireguard
+    # {% if command == "server" %}
+    # - "30000-30500:30000-30500"
+    # {% endif %}
+    environment:
+      - HOST_NAME={{node_name}}
+      - IFACE_NAME={{flannel_iface}}
+      - TOKEN={{vpn_token}}
+    volumes:
+      - /dev/net/tun:/dev/net/tun
+    restart: unless-stopped
+#   nginx:
+#     image: nginx:latest
+#     ports:
+#       - "{{redirect_source_port}}:{{redirect_source_port}}"
+#     restart: unless-stopped
+#     networks:
+#       - custom-network
+#     volumes:
+#       - {{nginx_path}}/nginx.conf:/etc/nginx/nginx.conf
 {% endif %}
-      networks:
-      - custom-network
-      command: >
-        {{command}}
-{% if command == "server" %}
-        --flannel-backend wireguard-native
-        --service-node-port-range "30000-30500"
-{% else %}
-        --server {{pool_ip}}
-        --token {{token}}
-{% endif %}
-        --node-label role={{command}}
-{% if node_labels %}
-        {{node_labels}}
-{% endif %}
-{% if num_gpus and num_gpus > 0 %}
-        --node-label gpu=on
-{% else %}
-        --node-label gpu=off
-{% endif %}
-        --node-ip {{ip_address}}
-        --node-external-ip {{ip_address}}
-{% if flannel_iface %}
-        --flannel-iface {{flannel_iface}}
-{% endif %}
-      volumes:
-      - {{k3s_path}}:/var/lib/rancher/k3s # Persist data
-      - {{etc_path}}:/etc/rancher/k3s # Config files
-{% if num_gpus and num_gpus > 0 %}
-      deploy:
-        resources:
-          reservations:
-            devices:
-            - driver: nvidia
-              count: {{num_gpus}}
-              capabilities: [gpu]
+
+# run worker only if command is set
+{%if command %}
+  {{service_name}}:
+    image: docker.io/bundenth/kalavai-runner:gpu-latest
+    container_name: {{service_name}}
+  {% if vpn %}
+    depends_on:
+    - {{vpn_name}}
+    network_mode: "service:{{vpn_name}}"
+  {% else %}
+    network_mode: host
+    # hostname: {{node_name}}
+    # networks:
+    # - custom-network
+    # ports:
+    # - "6443:6443" # kube server
+    # - "2379-2380:2379-2380" # etcd server
+    # - "10259:10259" # kube scheduler
+    # - "10257:10257" # kube controller manager
+    # - "10250:10250" # worker balancer
+    # - "8285:8285" # flannel
+    # - "8472:8472" # flannel vxlan
+    # - "51820:51820" # flannel wireguard
+    # {% if command == "server" %}
+    # - "30000-32767:30000-32767"
+    # {% endif %}
+  {% endif %}
+    privileged: true
+    restart: unless-stopped
+    command: >
+      --command={{command}}
+      {% if command == "server" %}
+      --port_range="30000-32767"
+      {% else %}
+      --server_ip={{pool_ip}}
+      --token={{pool_token}}
+      {% endif %}
+      {%if vpn %}
+      --flannel_iface={{flannel_iface}}
+      {% endif %}
+      {% if num_gpus and num_gpus > 0 %}
+      --gpu=on
+      {% else %}
+      --gpu=off
+      {% endif %}
+      {% if node_labels %}
+      --extra="{{node_labels}}"
+      {% endif %}
+
+    volumes:
+    - {{k3s_path}}:/var/lib/rancher/k3s # Persist data
+    - {{etc_path}}:/etc/rancher/k3s # Config files
+
+  {% if num_gpus and num_gpus > 0 %}
+    deploy:
+      resources:
+        reservations:
+          devices:
+          - driver: nvidia
+            count: {{num_gpus}}
+            capabilities: [gpu]
+  {% endif %}
 {% endif %}
 
 networks:
   custom-network:
-    driver: bridge
+    driver: bridge
+

kalavai_client/assets/nginx.conf

@@ -0,0 +1,12 @@
+events {}
+
+http {
+    server {
+        listen {{redirect_source_port}};
+        server_name localhost;
+
+        location / {
+                proxy_pass http://{{redirect_container}}:{{redirect_target_port}};
+        }
+    }
+}

kalavai_client/cli.py

@@ -31,6 +31,7 @@ from kalavai_client.utils import (
     safe_remove,
     leave_vpn,
     join_vpn,
+    get_vpn_details,
     load_server_info,
     user_login,
     user_logout,
@@ -81,15 +82,17 @@ STORAGE_ACCESS_MODE = ["ReadWriteOnce"]
 STORAGE_CLASS_LABEL = "kalavai.storage.enabled"
 DEFAULT_STORAGE_NAME = "pool-cache"
 DEFAULT_STORAGE_SIZE = 20
+DEFAULT_WATCHER_PORT = 30001
 USER_NODE_LABEL = "kalavai.cluster.user"
 KUBE_VERSION = os.getenv("KALAVAI_KUBE_VERSION", "v1.31.1+k3s1")
-DEFAULT_FLANNEL_IFACE = os.getenv("KALAVAI_FLANNEL_IFACE", "netmaker")
+DEFAULT_FLANNEL_IFACE = os.getenv("KALAVAI_FLANNEL_IFACE", "netmaker-1")
 FORBIDEDEN_IPS = ["127.0.0.1"]
 # kalavai templates
 HELM_APPS_FILE = resource_path("kalavai_client/assets/apps.yaml")
 HELM_APPS_VALUES = resource_path("kalavai_client/assets/apps_values.yaml")
 # user specific config files
-DEFAULT_CONTAINER_NAME = "kalavai-seed"
+DEFAULT_CONTAINER_NAME = "kalavai"
+DEFAULT_VPN_CONTAINER_NAME = "kalavai-vpn"
 CONTAINER_HOST_PATH = user_path("pool/", create_path=True)
 USER_COMPOSE_FILE = user_path("docker-compose-worker.yaml")
 USER_VPN_COMPOSE_FILE = user_path("docker-compose-vpn.yaml")
@@ -115,27 +118,6 @@ CLUSTER = dockerCluster(
 ######################
 ## HELPER FUNCTIONS ##
 ######################
-
-def check_vpn_compatibility():
-    """Check required packages to join VPN"""
-    logs = []
-    console.log("[white]Checking system requirements...")
-    # netclient
-    try:
-        run_cmd("sudo netclient version >/dev/null 2>&1")
-    except:
-        logs.append("[red]Netmaker not installed. Install instructions:\n")
-        logs.append("   Linux: https://docs.netmaker.io/docs/netclient#linux\n")
-        logs.append("   Windows: https://docs.netmaker.io/docs/netclient#windows\n")
-        logs.append("   MacOS: https://docs.netmaker.io/docs/netclient#mac\n")
-    
-    if len(logs) == 0:
-        console.log("[green]System is ready to join a pool")
-        return True
-    else:
-        for log in logs:
-            console.log(log)
-        return False
     
 def check_seed_compatibility():
     """Check required packages to start pools"""
@@ -179,16 +161,6 @@ def check_worker_compatibility():
 
 
 def cleanup_local():
-    # disconnect from private network
-    console.log("Disconnecting from VPN...")
-    try:
-        vpns = leave_vpn()
-        if vpns is not None:
-            for vpn in vpns:
-                console.log(f"You have left {vpn} VPN")
-    except:
-        # no vpn
-        pass
     console.log("Removing local cache files...")
     safe_remove(CONTAINER_HOST_PATH)
     safe_remove(USER_COMPOSE_FILE)
@@ -365,7 +337,7 @@ def select_token_type():
             break
     return {"admin": choice == 0, "user": choice == 1, "worker": choice == 2}
 
-def generate_compose_config(role, node_name, ip_address, node_labels, is_public, server=None, token=None):
+def generate_compose_config(role, node_name, is_public, node_labels=None, pool_ip=None, vpn_token=None, pool_token=None):
     num_gpus = 0
     try:
         has_gpus = check_gpu_drivers()
@@ -377,20 +349,24 @@ def generate_compose_config(role, node_name, ip_address, node_labels, is_public,
             ) 
     except:
         console.log(f"[red]WARNING: error when fetching NVIDIA GPU info. GPUs will not be used on this local machine")
+    if node_labels is not None:
+        node_labels = " ".join([f"--node-label {key}={value}" for key, value in node_labels.items()])
     compose_values = {
         "user_path": user_path(""),
         "service_name": DEFAULT_CONTAINER_NAME,
-        "pool_ip": server,
-        "token": token,
-        "hostname": node_name,
+        "vpn": is_public,
+        "vpn_name": DEFAULT_VPN_CONTAINER_NAME,
+        "pool_ip": pool_ip,
+        "pool_token": pool_token,
+        "vpn_token": vpn_token,
+        "node_name": node_name,
         "command": role,
         "storage_enabled": "True",
-        "ip_address": ip_address,
         "num_gpus": num_gpus,
         "k3s_path": f"{CONTAINER_HOST_PATH}/k3s",
         "etc_path": f"{CONTAINER_HOST_PATH}/etc",
-        "node_labels": " ".join([f"--node-label {key}={value}" for key, value in node_labels.items()]),
-        "flannel_iface": DEFAULT_FLANNEL_IFACE if is_public else None
+        "node_labels": node_labels,
+        "flannel_iface": DEFAULT_FLANNEL_IFACE if is_public else ""
     }
     # generate local config files
     compose_yaml = load_template(
@@ -585,35 +561,56 @@ def pool__start(cluster_name, *others,  only_registered_users: bool=False, ip_ad
         STORAGE_CLASS_LABEL: is_storage_compatible()
     }
     if location is not None:
-        console.log("Joining private network")
+        console.log("Fetching VPN credentials")
         try:
-            if not check_vpn_compatibility():
-                return
-            vpn = join_vpn(
+            vpn = get_vpn_details(
                 location=location,
                 user_cookie=USER_COOKIE)
             node_labels[USER_NODE_LABEL] = user["username"]
         except Exception as e:
             console.log(f"[red]Error when joining network: {str(e)}")
             return
-
-    if ip_address is None:
-        console.log(f"Scanning for valid IPs (subnet {vpn['subnet']})...")
-        ip_address = select_ip_address(subnet=vpn["subnet"])
+    
+    # Generate docker compose recipe
+    generate_compose_config(
+        role="server",
+        vpn_token=vpn["key"],
+        node_name=socket.gethostname(),
+        node_labels=node_labels,
+        is_public=location is not None
+    )
+    
+    # start server
+    console.log("Deploying seed...")
+    CLUSTER.start_seed_node()
+    
+    while not CLUSTER.is_agent_running():
+        console.log("Waiting for seed to start...")
+        time.sleep(10)
+    
+    # select IP address (for external discovery)
+    if ip_address is None and location is None:
+        # local IP
+        console.log(f"Scanning for valid IPs")
+        ip_address = select_ip_address()
+    else:
+        # load VPN ip
+        ip_address = CLUSTER.get_vpn_ip()
     console.log(f"Using {ip_address} address for server")
 
+    # populate local cred files
     auth_key = str(uuid.uuid4())
     write_auth_key = str(uuid.uuid4())
     readonly_auth_key = str(uuid.uuid4())
-    watcher_port = 30001
-    watcher_service = f"{ip_address}:{watcher_port}"
+    
+    watcher_service = f"{ip_address}:{DEFAULT_WATCHER_PORT}"
     values = {
         CLUSTER_NAME_KEY: cluster_name,
         CLUSTER_IP_KEY: ip_address,
         AUTH_KEY: auth_key,
         READONLY_AUTH_KEY: readonly_auth_key,
         WRITE_AUTH_KEY: write_auth_key,
-        WATCHER_PORT_KEY: watcher_port,
+        WATCHER_PORT_KEY: DEFAULT_WATCHER_PORT,
         WATCHER_SERVICE_KEY: watcher_service,
         USER_NODE_LABEL_KEY: USER_NODE_LABEL,
         ALLOW_UNREGISTERED_USER_KEY: not only_registered_users
@@ -630,15 +627,6 @@ def pool__start(cluster_name, *others,  only_registered_users: bool=False, ip_ad
         cluster_name=cluster_name,
         public_location=location,
         user_api_key=user["api_key"])
-
-    # 1. Generate docker compose recipe
-    compose_yaml = generate_compose_config(
-        role="server",
-        node_name=socket.gethostname(),
-        ip_address=ip_address,
-        node_labels=node_labels,
-        is_public=location is not None
-    )
     
     # Generate helmfile recipe
     helm_yaml = load_template(
@@ -650,14 +638,6 @@ def pool__start(cluster_name, *others,  only_registered_users: bool=False, ip_ad
         f.write(helm_yaml)
     
     console.log("[green]Config files have been generated in your local machine\n")
-
-    # # 1. start server
-    console.log("Deploying seed...")
-    CLUSTER.start_seed_node()
-    
-    while not CLUSTER.is_agent_running():
-        console.log("Waiting for seed to start...")
-        time.sleep(10)
     
     console.log("Setting pool dependencies...")
     # set template values in helmfile
@@ -691,7 +671,6 @@ def pool__start(cluster_name, *others,  only_registered_users: bool=False, ip_ad
 
     return None
 
-
 @arguably.command
 def pool__token(*others, admin=False, user=False, worker=False):
     """
@@ -773,6 +752,7 @@ def pool__join(token, *others, node_name=None, ip_address: str=None):
     if CLUSTER.is_agent_running():
         console.log(f"[white] You are already connected to {load_server_info(data_key=CLUSTER_NAME_KEY, file=USER_LOCAL_SERVER_FILE)}. Enter [yellow]kalavai pool stop[white] to exit and join another one.")
         return
+    
     # check that is not attached to another instance
     if os.path.exists(USER_LOCAL_SERVER_FILE):
         option = user_confirm(
@@ -810,20 +790,20 @@ def pool__join(token, *others, node_name=None, ip_address: str=None):
     }
     user = defaultdict(lambda: None)
     if public_location is not None:
-        console.log("Joining private network")
+        user = user_login(user_cookie=USER_COOKIE)
+        if user is None:
+            console.log("[red]Must be logged in to join public pools. Run [yellow]kalavai login[red] to authenticate")
+            exit()
+        console.log("Fetching VPN credentials")
         try:
-            if not check_vpn_compatibility():
-                return
-            vpn = join_vpn(
+            vpn = get_vpn_details(
                 location=public_location,
                 user_cookie=USER_COOKIE)
-            user = user_login(user_cookie=USER_COOKIE)
             node_labels[USER_NODE_LABEL] = user["username"]
         except Exception as e:
             console.log(f"[red]Error when joining network: {str(e)}")
             console.log("Are you authenticated? Try [yellow]kalavai login")
             return
-        # validate public seed
         try:
             validate_join_public_seed(
                 cluster_name=cluster_name,
@@ -832,31 +812,29 @@ def pool__join(token, *others, node_name=None, ip_address: str=None):
             )
         except Exception as e:
             console.log(f"[red]Error when joining network: {str(e)}")
-            leave_vpn(vpn_file=USER_VPN_COMPOSE_FILE)
             return
 
     # send note to server to let them know the node is coming online
-    if not pre_join_check(node_name=node_name, server_url=watcher_service, server_key=auth_key):
-        console.log(f"[red] Failed pre join checks. Server offline or node '{node_name}' may already exist. Please specify a different one with '--node-name'")
-        leave_vpn(vpn_file=USER_VPN_COMPOSE_FILE)
-        return
-
-    if ip_address is None:
-        console.log(f"Scanning for valid IPs (subnet {vpn['subnet']})...")
-        ip_address = select_ip_address(subnet=vpn["subnet"])
-    console.log(f"Using {ip_address} address for worker")
+    # TODO: won't be able to check for VPN pools...
+    # if not pre_join_check(node_name=node_name, server_url=watcher_service, server_key=auth_key):
+    #     console.log(f"[red] Failed pre join checks. Server offline or node '{node_name}' may already exist. Please specify a different one with '--node-name'")
+    #     leave_vpn(container_name=DEFAULT_VPN_CONTAINER_NAME)
+    #     return
         
     # local agent join
     # 1. Generate local cache files
     console.log("Generating config files...")
-    compose_yaml = generate_compose_config(
+    
+    # Generate docker compose recipe
+    generate_compose_config(
         role="agent",
-        server=f"https://{kalavai_seed_ip}:6443",
-        token=kalavai_token,
-        node_name=socket.gethostname(),
-        ip_address=ip_address,
+        pool_ip=f"https://{kalavai_seed_ip}:6443",
+        pool_token=kalavai_token,
+        vpn_token=vpn["key"],
+        node_name=node_name,
         node_labels=node_labels,
         is_public=public_location is not None)
+    
     store_server_info(
         server_ip=kalavai_seed_ip,
         auth_key=auth_key,
@@ -866,8 +844,6 @@ def pool__join(token, *others, node_name=None, ip_address: str=None):
         cluster_name=cluster_name,
         public_location=public_location,
         user_api_key=user["api_key"])
-    
-    init_user_workspace()
 
     option = user_confirm(
         question="Docker compose ready. Would you like Kalavai to deploy it?",
@@ -883,18 +859,23 @@ def pool__join(token, *others, node_name=None, ip_address: str=None):
         CLUSTER.start_worker_node()
     except Exception as e:
         console.log(f"[red] Error connecting to {cluster_name} @ {kalavai_seed_ip}. Check with the admin if the token is still valid.")
-        leave_vpn(vpn_file=USER_VPN_COMPOSE_FILE)
+        leave_vpn(container_name=DEFAULT_VPN_CONTAINER_NAME)
         exit()
 
-    while not CLUSTER.is_agent_running():
-        console.log("Waiting for worker to start...")
-        time.sleep(10)
+    # ensure we are connected
+    while True:
+        console.log("Waiting for core services to be ready, may take a few minutes...")
+        time.sleep(30)
+        if is_watcher_alive(server_creds=USER_LOCAL_SERVER_FILE, user_cookie=USER_COOKIE):
+            break
+    
+    init_user_workspace()
     
     # set status to schedulable
     console.log(f"[green] You are connected to {cluster_name}")
 
 @arguably.command
-def pool__stop(*others):
+def pool__stop(*others, skip_node_deletion=False):
     """
     Stop sharing your device and clean up. DO THIS ONLY IF YOU WANT TO REMOVE KALAVAI-CLIENT from your device.
 
@@ -903,7 +884,8 @@ def pool__stop(*others):
     """
     console.log("[white] Stopping kalavai app...")
     # delete local node from server
-    node__delete(load_server_info(data_key=NODE_NAME_KEY, file=USER_LOCAL_SERVER_FILE))
+    if not skip_node_deletion:
+        node__delete(load_server_info(data_key=NODE_NAME_KEY, file=USER_LOCAL_SERVER_FILE))
     # unpublish event (only if seed node)
     # TODO: no, this should be done via the platform!!!
     # try:
@@ -916,7 +898,20 @@ def pool__stop(*others):
     #     console.log(f"[red][WARNING]: (ignore if not a public pool) Error when unpublishing cluster. {str(e)}")
     # remove local node agent
     console.log("Removing agent and local cache")
+    
+    # disconnect from VPN first, then remove agent, then remove local files
+    console.log("Disconnecting from VPN...")
+    try:
+        vpns = leave_vpn(container_name=DEFAULT_VPN_CONTAINER_NAME)
+        if vpns is not None:
+            for vpn in vpns:
+                console.log(f"You have left {vpn} VPN")
+    except:
+        # no vpn
+        pass
+
     CLUSTER.remove_agent()
+
     # clean local files
     cleanup_local()
     console.log("[white] Kalavai has stopped sharing your resources. Use [yellow]kalavai pool start[white] or [yellow]kalavai pool join[white] to start again!")
@@ -1107,6 +1102,7 @@ def pool__attach(token, *others, node_name=None):
     """
     Set creds in token on the local instance
     """
+    # check that is not attached to another instance
     if os.path.exists(USER_LOCAL_SERVER_FILE):
         option = user_confirm(
             question="You seem to be connected to an instance already. Are you sure you want to join a new one?",
@@ -1115,34 +1111,39 @@ def pool__attach(token, *others, node_name=None):
         if option == 0:
             console.log("[green]Nothing happened.")
             return
+    
+    # check token
+    if not pool__check_token(token):
+        return
+
     try:
         data = decode_dict(token)
         kalavai_seed_ip = data[CLUSTER_IP_KEY]
-        kalavai_token = data[CLUSTER_TOKEN_KEY]
         cluster_name = data[CLUSTER_NAME_KEY]
         auth_key = data[AUTH_KEY]
         watcher_service = data[WATCHER_SERVICE_KEY]
         public_location = data[PUBLIC_LOCATION_KEY]
-    except:
-        console.log("[red]Error when parsing token. Invalid token")
+        vpn = defaultdict(lambda: None)
+    except Exception as e:
+        console.log(str(e))
+        console.log("[red] Invalid token")
         return
-
+    
     user = defaultdict(lambda: None)
     if public_location is not None:
-        console.log("Joining private network")
+        user = user_login(user_cookie=USER_COOKIE)
+        if user is None:
+            console.log("[red]Must be logged in to join public pools. Run [yellow]kalavai login[red] to authenticate")
+            exit()
+        console.log("Fetching VPN credentials")
         try:
-            if not check_vpn_compatibility():
-                return
-            vpn = join_vpn(
+            vpn = get_vpn_details(
                 location=public_location,
                 user_cookie=USER_COOKIE)
-            user = user_login(user_cookie=USER_COOKIE)
-            time.sleep(5)
         except Exception as e:
             console.log(f"[red]Error when joining network: {str(e)}")
             console.log("Are you authenticated? Try [yellow]kalavai login")
             return
-        # validate public seed
         try:
             validate_join_public_seed(
                 cluster_name=cluster_name,
@@ -1151,9 +1152,19 @@ def pool__attach(token, *others, node_name=None):
             )
         except Exception as e:
             console.log(f"[red]Error when joining network: {str(e)}")
-            leave_vpn(vpn_file=USER_VPN_COMPOSE_FILE)
             return
-
+        
+    # local agent join
+    # 1. Generate local cache files
+    console.log("Generating config files...")
+    
+    # Generate docker compose recipe
+    generate_compose_config(
+        role="",
+        vpn_token=vpn["key"],
+        node_name=node_name,
+        is_public=public_location is not None)
+    
     store_server_info(
         server_ip=kalavai_seed_ip,
         auth_key=auth_key,
@@ -1164,7 +1175,26 @@ def pool__attach(token, *others, node_name=None):
         public_location=public_location,
         user_api_key=user["api_key"])
 
-    console.log(f"[green]You are now connected to {cluster_name} @ {kalavai_seed_ip}")
+    option = user_confirm(
+        question="Docker compose ready. Would you like Kalavai to deploy it?",
+        options=["no", "yes"]
+    )
+    if option == 0:
+        console.log("Manually deploy the worker with the following command:\n")
+        print(f"docker compose -f {USER_COMPOSE_FILE} up -d")
+        return
+    
+    console.log(f"[white] Connecting to {cluster_name} @ {kalavai_seed_ip} (this may take a few minutes)...")
+    run_cmd(f"docker compose -f {USER_COMPOSE_FILE} up -d")
+    # ensure we are connected
+    while True:
+        console.log("Waiting for core services to be ready, may take a few minutes...")
+        time.sleep(30)
+        if is_watcher_alive(server_creds=USER_LOCAL_SERVER_FILE, user_cookie=USER_COOKIE):
+            break
+    
+    # set status to schedulable
+    console.log(f"[green] You are connected to {cluster_name}")
 
 
 @arguably.command

kalavai_client/cluster.py

@@ -7,7 +7,8 @@ from kalavai_client.utils import (
     run_cmd,
     check_gpu_drivers,
     validate_poolconfig,
-    user_path
+    user_path,
+    populate_template
 )
 
 
@@ -20,6 +21,9 @@ class Cluster(ABC):
     def start_worker_node(self, url, token, node_name, auth_key, watcher_service, ip_address, labels, flannel_iface):
         raise NotImplementedError()
 
+    @abstractmethod
+    def get_vpn_ip(self):
+        raise NotImplementedError()
 
     @abstractmethod
     def update_dependencies(self, dependencies_files):
@@ -79,11 +83,23 @@ class dockerCluster(Cluster):
     def start_seed_node(self):
         
         run_cmd(f"docker compose -f {self.compose_file} up -d")
-        time.sleep(5)
-        run_cmd(f"docker cp {self.container_name}:/etc/rancher/k3s/k3s.yaml {self.kubeconfig_file}")
+        # wait for container to be setup
+        while True:
+            try:
+                run_cmd(f"docker cp {self.container_name}:/etc/rancher/k3s/k3s.yaml {self.kubeconfig_file} >/dev/null 2>&1")
+                break
+            except:
+                pass
+            time.sleep(5)
 
     def start_worker_node(self):
         run_cmd(f"docker compose -f {self.compose_file} up -d")
+    
+    def get_vpn_ip(self):
+        command = populate_template(
+            template_str="docker exec -it {{container_name}} ifconfig {{iface_name}} | grep 'inet ' | awk '{gsub(/^addr:/, \"\", $2); print $2}'",
+            values_dict={"container_name": self.container_name, "iface_name": self.default_flannel_iface})
+        return run_cmd(command).decode().strip()
         
 
     def update_dependencies(self, dependencies_file=None, debug=False, retries=3):
@@ -122,8 +138,13 @@ class dockerCluster(Cluster):
     def is_seed_node(self):
         if not os.path.isfile(self.compose_file):
             return False
-        status = "server" in run_cmd(f"docker compose -f {self.compose_file} ps --services --status=running").decode()
-        return status
+        if not self.is_agent_running():
+            return False
+        try:
+            run_cmd(f"docker container exec {self.container_name} cat /var/lib/rancher/k3s/server/node-token >/dev/null 2>&1")
+            return True
+        except:
+            return False
     
     def is_cluster_init(self):
         if not os.path.isfile(self.compose_file):

kalavai_client/utils.py

@@ -241,12 +241,12 @@ def join_vpn(location, user_cookie):
     run_cmd(f"sudo netclient join -t {token} >/dev/null 2>&1")
     return vpn
 
-def leave_vpn():
+def leave_vpn(container_name):
     try:
-        vpns = json.loads(run_cmd("sudo netclient list").decode())
+        vpns = json.loads(run_cmd(f"docker exec {container_name} netclient list").decode())
         left_vpns = [vpn['network'] for vpn in vpns]
         for vpn in left_vpns:
-            run_cmd(f"sudo netclient leave {vpn}")
+            run_cmd(f"docker exec {container_name} netclient leave {vpn}")
         return left_vpns
     except:
         return None

{kalavai_client-0.5.7.dist-info → kalavai_client-0.5.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: kalavai-client
-Version: 0.5.7
+Version: 0.5.9
 Summary: Client app for kalavai platform
 License: Apache-2.0
 Keywords: LLM,platform
@@ -156,6 +156,16 @@ Make sure you also install python3-dev package. For ubuntu distros:
 sudo apt install python3-dev
 ```
 
+If you see:
+```bash
+AttributeError: install_layout. Did you mean: 'install_platlib'?
+      [end of output]
+```
+
+Upgrade your setuptools:
+```bash
+pip install -U setuptools
+```
 
 ### Install the client
 
@@ -270,10 +280,11 @@ Python version <= 3.12.
 ```bash
 sudo add-apt-repository ppa:deadsnakes/ppa
 sudo apt update
-sudo apt install python3.10 python3.10-dev
+sudo apt install python3.10 python3.10-dev python3-virtualenv
 virtualenv -p python3.10 env
 source env/bin/activate
 sudo apt install  python3.10-venv python3.10-dev -y
+pip install -U setuptools
 pip install -e .[dev]
 ```
 
@@ -291,4 +302,4 @@ To run the unit tests, use:
 python -m unittest
 ```
 
-
+docker run --rm --net=host -v   /root/.cache/kalavai/:/root/.cache/kalavai/  ghcr.io/helmfile/helmfile:v0.169.2 helmfile sync --file  /root/.cache/kalavai/apps.yaml --kubeconfig /root/.cache/kalavai/kubeconfig

kalavai_client-0.5.9.dist-info/RECORD

@@ -0,0 +1,20 @@
+kalavai_client/__init__.py,sha256=vItrdD645pc4NrXPAz6cc0ExWrMEfwvLkksqG8FHQaE,22
+kalavai_client/__main__.py,sha256=WQUfxvRsBJH5gsCJg8pLz95QnZIj7Ol8psTO77m0QE0,73
+kalavai_client/assets/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+kalavai_client/assets/apps.yaml,sha256=_8BgT9F611c8uZJvhTE_0CLbQqnLaUQosqxZjzOslXQ,5979
+kalavai_client/assets/apps_values.yaml,sha256=CjKVelPQHd-hm-DTMEuya92feKiphU9mh3HrosLYYPE,1676
+kalavai_client/assets/docker-compose-template.yaml,sha256=gJ0NkhcG2c-gZPmSd385dadrXkZrWruTJkiaxcaKkQ0,2725
+kalavai_client/assets/nginx.conf,sha256=drVVCg8GHucz7hmt_BI6giAhK92OV71257NTs3LthwM,225
+kalavai_client/assets/pool_config_template.yaml,sha256=fFz4w2-fMKD5KvyzFdfcWD_jSneRlmnjLc8hCctweX0,576
+kalavai_client/assets/pool_config_values.yaml,sha256=VrM3XHQfQo6QLZ68qvagooUptaYgl1pszniY_JUtemk,233
+kalavai_client/assets/user_workspace.yaml,sha256=wDvlMYknOPABAEo0dsQwU7bac8iubjAG9tdkFbJZ5Go,476
+kalavai_client/assets/user_workspace_values.yaml,sha256=G0HOzQUxrDMCwuW9kbWUZaKMzDDPVwDwzBHCL2Xi2ZM,542
+kalavai_client/auth.py,sha256=QsBh28L2LwjBBK6pTUE4Xu36lLDTyetyU1YfS1Hbb6g,1717
+kalavai_client/cli.py,sha256=4qTZuYNFhsdJbnER-MSBJHPNgJc_lWzbWR0Bj2YeQe0,68889
+kalavai_client/cluster.py,sha256=fULTAad4KXEGeWZmp4_VBoBwT5eED_HOBUsXIKmf0CU,12119
+kalavai_client/utils.py,sha256=NOORAsRs9A84w8yyvhGwXcvshyZyf5wj86HT4HPzOcI,12405
+kalavai_client-0.5.9.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+kalavai_client-0.5.9.dist-info/METADATA,sha256=LEBFwVRSWQ6GRLPBWhygefs4S_EAgpgoqaSXSmF3CwI,13013
+kalavai_client-0.5.9.dist-info/WHEEL,sha256=_GR1VF9XYsw6EE9ATkyto4e6vRiOUBshR1STiZ5m1gE,92
+kalavai_client-0.5.9.dist-info/entry_points.txt,sha256=9T6D45gxwzfVbglMm1r6XPdXuuZdHfy_7fCeu2jUphc,50
+kalavai_client-0.5.9.dist-info/RECORD,,

kalavai_client/assets/vpn-template.yaml

@@ -1,13 +0,0 @@
-# https://docs.netmaker.io/docs/netclient#docker
-services:
-    {{service_name}}:
-        network_mode: host
-        privileged: true
-        restart: always
-        environment:
-        - TOKEN={{vpn_token}}
-        - IFACE_NAME={{flannel_iface}}
-        volumes:
-        - '{{etc_path}}/netclient:/etc/netclient'
-        container_name: {{service_name}}
-        image: 'gravitl/netclient:latest'

kalavai_client-0.5.7.dist-info/RECORD

@@ -1,20 +0,0 @@
-kalavai_client/__init__.py,sha256=VsVCdw5n_XkxVQiDP5x4If6sIoowwW2gG3FlCDuzkaY,22
-kalavai_client/__main__.py,sha256=WQUfxvRsBJH5gsCJg8pLz95QnZIj7Ol8psTO77m0QE0,73
-kalavai_client/assets/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-kalavai_client/assets/apps.yaml,sha256=aW9wKyvZhZFMHzBkZOsgVq-kpntED6U8B9XoHkm5F9Y,5963
-kalavai_client/assets/apps_values.yaml,sha256=CjKVelPQHd-hm-DTMEuya92feKiphU9mh3HrosLYYPE,1676
-kalavai_client/assets/docker-compose-template.yaml,sha256=qDv0og338clLobDDPEJ-HiGlMcCMMx2NOi5R_hdhKvw,1442
-kalavai_client/assets/pool_config_template.yaml,sha256=fFz4w2-fMKD5KvyzFdfcWD_jSneRlmnjLc8hCctweX0,576
-kalavai_client/assets/pool_config_values.yaml,sha256=VrM3XHQfQo6QLZ68qvagooUptaYgl1pszniY_JUtemk,233
-kalavai_client/assets/user_workspace.yaml,sha256=wDvlMYknOPABAEo0dsQwU7bac8iubjAG9tdkFbJZ5Go,476
-kalavai_client/assets/user_workspace_values.yaml,sha256=G0HOzQUxrDMCwuW9kbWUZaKMzDDPVwDwzBHCL2Xi2ZM,542
-kalavai_client/assets/vpn-template.yaml,sha256=Hm7sevtrsakSxSKMJwl68hzOEWCaxwYytwkTgKhe_MM,397
-kalavai_client/auth.py,sha256=QsBh28L2LwjBBK6pTUE4Xu36lLDTyetyU1YfS1Hbb6g,1717
-kalavai_client/cli.py,sha256=qBOe4IULIX7pzKUhHoR1JppWrbUeV1aslxQLMWrRGpU,67913
-kalavai_client/cluster.py,sha256=C7uofrpSEj4PUhF_VLfwH4k9BhJMTkuKRWHxDYu1OD0,11345
-kalavai_client/utils.py,sha256=aHgNj5iVOhgTrI-UFoqgAvnesn4t6Qos3-WnNSxZ-fc,12342
-kalavai_client-0.5.7.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-kalavai_client-0.5.7.dist-info/METADATA,sha256=q7m04ZO0VMss4HEuJxBEY-587JwjHRmwZ8Cj8a7jyxw,12584
-kalavai_client-0.5.7.dist-info/WHEEL,sha256=_GR1VF9XYsw6EE9ATkyto4e6vRiOUBshR1STiZ5m1gE,92
-kalavai_client-0.5.7.dist-info/entry_points.txt,sha256=9T6D45gxwzfVbglMm1r6XPdXuuZdHfy_7fCeu2jUphc,50
-kalavai_client-0.5.7.dist-info/RECORD,,