konduktor-nightly 0.1.0.dev20250815104918__tar.gz → 0.1.0.dev20250816104612__tar.gz

{konduktor_nightly-0.1.0.dev20250815104918 → konduktor_nightly-0.1.0.dev20250816104612}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: konduktor-nightly
-Version: 0.1.0.dev20250815104918
+Version: 0.1.0.dev20250816104612
 Summary: GPU Cluster Health Management
 Author: Andrew Aikawa
 Author-email: asai@berkeley.edu

{konduktor_nightly-0.1.0.dev20250815104918 → konduktor_nightly-0.1.0.dev20250816104612}/konduktor/__init__.py

@@ -11,7 +11,7 @@ from konduktor.task import Task
 __all__ = ['launch', 'Resources', 'Task', 'Serving']
 
 # Replaced with the current commit when building the wheels.
-_KONDUKTOR_COMMIT_SHA = '9e2a8dc9b3415f811b7c8f1e7ed4c8d9e941ec61'
+_KONDUKTOR_COMMIT_SHA = '108d7fe47b1bd5db50d555510714d2e204fb7b6f'
 os.makedirs(os.path.expanduser('~/.konduktor'), exist_ok=True)
 
 
@@ -45,5 +45,5 @@ def _get_git_commit():
 
 
 __commit__ = _get_git_commit()
-__version__ = '1.0.0.dev0.1.0.dev20250815104918'
+__version__ = '1.0.0.dev0.1.0.dev20250816104612'
 __root_dir__ = os.path.dirname(os.path.abspath(__file__))

{konduktor_nightly-0.1.0.dev20250815104918 → konduktor_nightly-0.1.0.dev20250816104612}/konduktor/backends/jobset.py

@@ -176,7 +176,7 @@ class JobsetBackend(backend.Backend):
         context = kubernetes_utils.get_current_kube_config_context_name()
         namespace = kubernetes_utils.get_kube_config_context_namespace(context)
         # TODO(asaiacai): need to set env variables in pod
-        jobset_response = jobset_utils.create_jobset(
+        jobset_utils.create_jobset(
             namespace,
             task,
             pod_spec['kubernetes']['pod_config'],
@@ -192,10 +192,9 @@ class JobsetBackend(backend.Backend):
                 ):
                     _wait_for_jobset_start(namespace, task.name)
                 try:
-                    assert jobset_response is not None
                     log_thread = threading.Thread(
                         target=log_utils.tail_logs,
-                        args=(jobset_response,),
+                        args=(task.name,),
                         daemon=True,
                     )
                     logger.info('streaming logs...')

{konduktor_nightly-0.1.0.dev20250815104918 → konduktor_nightly-0.1.0.dev20250816104612}/konduktor/templates/pod.yaml.j2

@@ -7,6 +7,16 @@ kubernetes:
         {% if accelerator_type %}
         trainy.ai/accelerator: {{ accelerator_type }}
         {% endif %}
+        {% if konduktor_debug %}
+        konduktor.ai/debug: "true"
+        {% else %}
+        konduktor.ai/debug: "false"
+        {% endif %}
+        {% if tailscale_secret %}
+        konduktor.ai/tailscale: "true"
+        {% else %}
+        konduktor.ai/tailscale: "false"
+        {% endif %}
     spec:
       restartPolicy: "Never"
       # trigger this on GPU request
@@ -312,13 +322,18 @@ kubernetes:
                   if ! command -v tailscale >/dev/null 2>&1; then
                     $(prefix_cmd) curl -fsSL https://tailscale.com/install.sh | DEBIAN_FRONTEND=noninteractive $(prefix_cmd) sh > ~/.konduktor/tmp/tailscale-install.log 2>&1
                   fi
-                  $(prefix_cmd) tailscaled --tun=userspace-networking --state=mem: >/dev/null 2>&1 &
-                  $(prefix_cmd) timeout 5 tailscale up --auth-key=${TS_AUTHKEY} --ssh --hostname=${TS_HOSTNAME}
-                  $(prefix_cmd) sleep 10
+                  $(prefix_cmd) tailscaled --tun=userspace-networking --state=mem: >~/.konduktor/tmp/tailscaled.log 2>&1 &
+                  $(prefix_cmd) sleep 2
+                  $(prefix_cmd) timeout 5 tailscale up --auth-key=${TS_AUTHKEY} --ssh --hostname=${TS_HOSTNAME} || echo "tailscale up failed retrying"
+                  $(prefix_cmd) sleep 2
                 done
                 $(prefix_cmd) echo "Tailscale is up"
+                $(prefix_cmd) tailscale status
               }
               InstallTailscale | tee ~/.konduktor/tmp/tailscale-out.log
+              {% if konduktor_debug %}
+              $(prefix_cmd) cat ~/.konduktor/tmp/tailscale*.log
+              {% endif %}
               {% endif %}
               end_epoch=$(date +%s);
 
@@ -416,8 +431,8 @@ kubernetes:
           secretName: {{ git_ssh }}
           defaultMode: 384
       {% endif %}
-
         
+
       # TODO(asaiacai): should we add nodeSelectors here or leave to
       # kueue resource flavors. leaning towards defining
       # in kueue and just querying for the kueue resource flavor

{konduktor_nightly-0.1.0.dev20250815104918 → konduktor_nightly-0.1.0.dev20250816104612}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "konduktor-nightly"
-version = "0.1.0.dev20250815104918"
+version = "0.1.0.dev20250816104612"
 description = "GPU Cluster Health Management"
 packages = [
     {include = "konduktor"}