konduktor-nightly 0.1.0.dev20250815104918__py3-none-any.whl → 0.1.0.dev20250816104612__py3-none-any.whl

konduktor/__init__.py

@@ -11,7 +11,7 @@ from konduktor.task import Task
 __all__ = ['launch', 'Resources', 'Task', 'Serving']
 
 # Replaced with the current commit when building the wheels.
-_KONDUKTOR_COMMIT_SHA = '9e2a8dc9b3415f811b7c8f1e7ed4c8d9e941ec61'
+_KONDUKTOR_COMMIT_SHA = '108d7fe47b1bd5db50d555510714d2e204fb7b6f'
 os.makedirs(os.path.expanduser('~/.konduktor'), exist_ok=True)
 
 
@@ -45,5 +45,5 @@ def _get_git_commit():
 
 
 __commit__ = _get_git_commit()
-__version__ = '1.0.0.dev0.1.0.dev20250815104918'
+__version__ = '1.0.0.dev0.1.0.dev20250816104612'
 __root_dir__ = os.path.dirname(os.path.abspath(__file__))

konduktor/backends/jobset.py

@@ -176,7 +176,7 @@ class JobsetBackend(backend.Backend):
         context = kubernetes_utils.get_current_kube_config_context_name()
         namespace = kubernetes_utils.get_kube_config_context_namespace(context)
         # TODO(asaiacai): need to set env variables in pod
-        jobset_response = jobset_utils.create_jobset(
+        jobset_utils.create_jobset(
             namespace,
             task,
             pod_spec['kubernetes']['pod_config'],
@@ -192,10 +192,9 @@ class JobsetBackend(backend.Backend):
                 ):
                     _wait_for_jobset_start(namespace, task.name)
                 try:
-                    assert jobset_response is not None
                     log_thread = threading.Thread(
                         target=log_utils.tail_logs,
-                        args=(jobset_response,),
+                        args=(task.name,),
                         daemon=True,
                     )
                     logger.info('streaming logs...')

konduktor/templates/pod.yaml.j2

@@ -7,6 +7,16 @@ kubernetes:
         {% if accelerator_type %}
         trainy.ai/accelerator: {{ accelerator_type }}
         {% endif %}
+        {% if konduktor_debug %}
+        konduktor.ai/debug: "true"
+        {% else %}
+        konduktor.ai/debug: "false"
+        {% endif %}
+        {% if tailscale_secret %}
+        konduktor.ai/tailscale: "true"
+        {% else %}
+        konduktor.ai/tailscale: "false"
+        {% endif %}
     spec:
       restartPolicy: "Never"
       # trigger this on GPU request
@@ -312,13 +322,18 @@ kubernetes:
                   if ! command -v tailscale >/dev/null 2>&1; then
                     $(prefix_cmd) curl -fsSL https://tailscale.com/install.sh | DEBIAN_FRONTEND=noninteractive $(prefix_cmd) sh > ~/.konduktor/tmp/tailscale-install.log 2>&1
                   fi
-                  $(prefix_cmd) tailscaled --tun=userspace-networking --state=mem: >/dev/null 2>&1 &
-                  $(prefix_cmd) timeout 5 tailscale up --auth-key=${TS_AUTHKEY} --ssh --hostname=${TS_HOSTNAME}
-                  $(prefix_cmd) sleep 10
+                  $(prefix_cmd) tailscaled --tun=userspace-networking --state=mem: >~/.konduktor/tmp/tailscaled.log 2>&1 &
+                  $(prefix_cmd) sleep 2
+                  $(prefix_cmd) timeout 5 tailscale up --auth-key=${TS_AUTHKEY} --ssh --hostname=${TS_HOSTNAME} || echo "tailscale up failed retrying"
+                  $(prefix_cmd) sleep 2
                 done
                 $(prefix_cmd) echo "Tailscale is up"
+                $(prefix_cmd) tailscale status
               }
               InstallTailscale | tee ~/.konduktor/tmp/tailscale-out.log
+              {% if konduktor_debug %}
+              $(prefix_cmd) cat ~/.konduktor/tmp/tailscale*.log
+              {% endif %}
               {% endif %}
               end_epoch=$(date +%s);
 
@@ -416,8 +431,8 @@ kubernetes:
           secretName: {{ git_ssh }}
           defaultMode: 384
       {% endif %}
-
         
+
       # TODO(asaiacai): should we add nodeSelectors here or leave to
       # kueue resource flavors. leaning towards defining
       # in kueue and just querying for the kueue resource flavor

{konduktor_nightly-0.1.0.dev20250815104918.dist-info → konduktor_nightly-0.1.0.dev20250816104612.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: konduktor-nightly
-Version: 0.1.0.dev20250815104918
+Version: 0.1.0.dev20250816104612
 Summary: GPU Cluster Health Management
 Author: Andrew Aikawa
 Author-email: asai@berkeley.edu

{konduktor_nightly-0.1.0.dev20250815104918.dist-info → konduktor_nightly-0.1.0.dev20250816104612.dist-info}/RECORD

@@ -1,4 +1,4 @@
-konduktor/__init__.py,sha256=IJrSlRG-q24AZUQh3Sx5BuKRBdiiWfGk1J6UV0Od5KI,1574
+konduktor/__init__.py,sha256=yLJCPm4HBMfS-3YCs6f31atbH43xOS7u1CX36vWZaYk,1574
 konduktor/adaptors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 konduktor/adaptors/aws.py,sha256=s47Ra-GaqCQibzVfmD0pmwEWHif1EGO5opMbwkLxTCU,8244
 konduktor/adaptors/common.py,sha256=ZIqzjx77PIHUwpjfAQ1uX8B2aX78YMuGj4Bppd-MdyM,4183
@@ -9,7 +9,7 @@ konduktor/backends/backend.py,sha256=qh0bp94lzoTYZkzyQv2-CVrB5l91FkG2vclXg24UFC0
 konduktor/backends/constants.py,sha256=nt9G9AmFCOMwO4GuKgRQSzJJuKapOmaROp4_Y0tMF5A,732
 konduktor/backends/deployment.py,sha256=EHfB2uLeKFQ3maek9tx6XL4_sjQ-ax59DZA79Q3EkVs,5519
 konduktor/backends/deployment_utils.py,sha256=VGuL01rKe7p7PoVRI_cP4tiZRxHZ13nnTMG-bmDf7P0,28975
-konduktor/backends/jobset.py,sha256=E9THHmcpxTohsx6Goi9mKF4dy_mYpR2DHloSwGVr9jA,8509
+konduktor/backends/jobset.py,sha256=OwgDog9nH-FoUmNU_H--C3U5jx70reTKL1l849M1k5A,8430
 konduktor/backends/jobset_utils.py,sha256=7fB8X4b2Q5BKFCIGME72dyeCfi-EemoMeJVnwtzcjq4,25184
 konduktor/backends/pod_utils.py,sha256=Jfv_CY8suF0e7QEaeQiNRRxRnOueLgPR8SfLEO7lnwc,15260
 konduktor/check.py,sha256=JennyWoaqSKhdyfUldd266KwVXTPJpcYQa4EED4a_BA,7569
@@ -77,7 +77,7 @@ konduktor/serving.py,sha256=sh8TPAUXg23Bkt0ByatIMdxFFqzRm18HJTEkt3wHzdo,5147
 konduktor/task.py,sha256=97iLCo62qpN9wLGNPeFw64E8k1nch7AyySY3BUXHPWY,37496
 konduktor/templates/deployment.yaml.j2,sha256=uXFjDQaimbpFdAn2RJGaIvS_PzDY136cw_L3QMjz3ZA,3452
 konduktor/templates/jobset.yaml.j2,sha256=gURWl6uQv_OLni-LFy2E7ttjGOtuRDt5Vfs4ALH7fpI,1196
-konduktor/templates/pod.yaml.j2,sha256=TORLvkLthUc4KXTGccIJbNaE-y3uXwlZFHP6zebxhdI,18238
+konduktor/templates/pod.yaml.j2,sha256=gxBaHFwDfRE71nh3glYPsmirOr6Qn__-f6oHmQRP4QU,18809
 konduktor/usage/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 konduktor/usage/constants.py,sha256=gCL8afIHZhO0dcxbJGpESE9sCC1cBSbeRnQ8GwNOY4M,612
 konduktor/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -97,8 +97,8 @@ konduktor/utils/schemas.py,sha256=tBrKhnkfn9uKDYdlb4L2KgooW-muuhww7U8fu9zX-ms,18
 konduktor/utils/subprocess_utils.py,sha256=WoFkoFhGecPR8-rF8WJxbIe-YtV94LXz9UG64SDhCY4,9448
 konduktor/utils/ux_utils.py,sha256=7-Lt3QbDVvBQUli5_U9lOdXKeC-ip8rZBpO9gQ6vPJw,7955
 konduktor/utils/validator.py,sha256=5C1kE57Eyj1OPnAbvojqMNHHtf5fnl47FK_vEttd8aw,4331
-konduktor_nightly-0.1.0.dev20250815104918.dist-info/LICENSE,sha256=MuuqTZbHvmqXR_aNKAXzggdV45ANd3wQ5YI7tnpZhm0,6586
-konduktor_nightly-0.1.0.dev20250815104918.dist-info/METADATA,sha256=nK82xb6Qw94vR9UyKVmUma1MBA0geg6QUNky96zpsJU,4247
-konduktor_nightly-0.1.0.dev20250815104918.dist-info/WHEEL,sha256=XbeZDeTWKc1w7CSIyre5aMDU_-PohRwTQceYnisIYYY,88
-konduktor_nightly-0.1.0.dev20250815104918.dist-info/entry_points.txt,sha256=k3nG5wDFIJhNqsZWrHk4d0irIB2Ns9s47cjRWYsTCT8,48
-konduktor_nightly-0.1.0.dev20250815104918.dist-info/RECORD,,
+konduktor_nightly-0.1.0.dev20250816104612.dist-info/LICENSE,sha256=MuuqTZbHvmqXR_aNKAXzggdV45ANd3wQ5YI7tnpZhm0,6586
+konduktor_nightly-0.1.0.dev20250816104612.dist-info/METADATA,sha256=0a3HUMOHD-ecec5AHlJS9CRkSKvBb4EyWfcr-BqRwok,4247
+konduktor_nightly-0.1.0.dev20250816104612.dist-info/WHEEL,sha256=XbeZDeTWKc1w7CSIyre5aMDU_-PohRwTQceYnisIYYY,88
+konduktor_nightly-0.1.0.dev20250816104612.dist-info/entry_points.txt,sha256=k3nG5wDFIJhNqsZWrHk4d0irIB2Ns9s47cjRWYsTCT8,48
+konduktor_nightly-0.1.0.dev20250816104612.dist-info/RECORD,,