konduktor-nightly 0.1.0.dev20250819104842__py3-none-any.whl → 0.1.0.dev20250821104804__py3-none-any.whl

konduktor/__init__.py

@@ -11,7 +11,7 @@ from konduktor.task import Task
 __all__ = ['launch', 'Resources', 'Task', 'Serving']
 
 # Replaced with the current commit when building the wheels.
-_KONDUKTOR_COMMIT_SHA = '108d7fe47b1bd5db50d555510714d2e204fb7b6f'
+_KONDUKTOR_COMMIT_SHA = 'eee38d922bf4c7cb8a2e6e730092dde0ae372500'
 os.makedirs(os.path.expanduser('~/.konduktor'), exist_ok=True)
 
 
@@ -45,5 +45,5 @@ def _get_git_commit():
 
 
 __commit__ = _get_git_commit()
-__version__ = '1.0.0.dev0.1.0.dev20250819104842'
+__version__ = '1.0.0.dev0.1.0.dev20250821104804'
 __root_dir__ = os.path.dirname(os.path.abspath(__file__))

konduktor/backends/jobset_utils.py

@@ -464,6 +464,41 @@ def _get_end_time_from_conditions(job: Dict[str, Any]) -> str:
     return '-'
 
 
+def _get_time_delta(delta: 'timedelta') -> Tuple[str, 'timedelta']:
+    total_seconds = int(delta.total_seconds())
+
+    days, remainder = divmod(total_seconds, 86400)  # 86400 seconds in a day
+    hours, remainder = divmod(remainder, 3600)  # 3600 seconds in an hour
+    minutes, seconds = divmod(remainder, 60)  # 60 seconds in a minute
+
+    days_str = f'{days} day{"s" if days != 1 else ""}, ' if days > 0 else ''
+    hours_str = f'{hours} hr{"s" if hours != 1 else ""}, ' if hours > 0 else ''
+    minutes_str = (
+        f'{minutes} min{"s" if minutes != 1 else ""}'
+        if minutes > 0 and days == 0
+        else ''
+    )
+
+    seconds_str = (
+        f'{seconds} sec{"s" if seconds != 1 else ""}'
+        if seconds > 0 and days == 0 and hours == 0 and minutes == 0
+        else ''
+    )
+
+    result = f'{days_str}{hours_str}{minutes_str}{seconds_str}'
+    return result if result else '<1 minute', delta
+
+
+def _get_job_length(start_time: str, end_time: str) -> str:
+    if start_time == '-' or end_time == '-':
+        return '-'
+    else:
+        start = datetime.strptime(start_time, '%m/%d/%y %I:%M%p')
+        end = datetime.strptime(end_time, '%m/%d/%y %I:%M%p')
+        delta, _ = _get_time_delta(end - start)
+        return delta
+
+
 def show_status_table(
     namespace: str,
     all_users: bool,
@@ -531,32 +566,6 @@ def show_status_table(
                 f'{JobStatus.PENDING.name}{colorama.Style.RESET_ALL}'
             )
 
-    def _get_time_delta(timestamp: str) -> Tuple[str, 'timedelta']:
-        delta = datetime.now(timezone.utc) - datetime.strptime(
-            timestamp, '%Y-%m-%dT%H:%M:%SZ'
-        ).replace(tzinfo=timezone.utc)
-        total_seconds = int(delta.total_seconds())
-
-        days, remainder = divmod(total_seconds, 86400)  # 86400 seconds in a day
-        hours, remainder = divmod(remainder, 3600)  # 3600 seconds in an hour
-        minutes, seconds = divmod(remainder, 60)  # 60 seconds in a minute
-
-        days_str = f'{days} day{"s" if days != 1 else ""}, ' if days > 0 else ''
-        hours_str = f'{hours} hr{"s" if hours != 1 else ""}, ' if hours > 0 else ''
-        minutes_str = (
-            f'{minutes} min{"s" if minutes != 1 else ""}'
-            if minutes > 0 and days == 0
-            else ''
-        )
-        seconds_str = (
-            f'{seconds} sec{"s" if seconds != 1 else ""}'
-            if seconds > 0 and days == 0 and hours == 0 and minutes == 0
-            else ''
-        )
-
-        result = f'{days_str}{hours_str}{minutes_str}{seconds_str}'
-        return result if result else '<1 minute', delta
-
     def _get_resources(job: Dict[str, Any]) -> str:
         num_pods = int(
             job['spec']['replicatedJobs'][0]['template']['spec']['parallelism']
@@ -580,9 +589,18 @@ def show_status_table(
             'SUBMITTED',
             'START TIME',
             'END TIME',
+            'DURATION',
         ]
     else:
-        columns = ['NAME', 'STATUS', 'RESOURCES', 'SUBMITTED', 'START TIME', 'END TIME']
+        columns = [
+            'NAME',
+            'STATUS',
+            'RESOURCES',
+            'SUBMITTED',
+            'START TIME',
+            'END TIME',
+            'DURATION',
+        ]
     job_table = log_utils.create_table(columns)
     job_specs = list_jobset(namespace)
     assert job_specs is not None, 'Retrieving jobs failed'
@@ -621,13 +639,18 @@ def show_status_table(
             start_time = _format_timestamp(start_time)
 
         # Get submitted time (how long ago)
-        submitted_time, _ = _get_time_delta(job['metadata']['creationTimestamp'])
+        time_delta = datetime.now(timezone.utc) - datetime.strptime(
+            job['metadata']['creationTimestamp'], '%Y-%m-%dT%H:%M:%SZ'
+        ).replace(tzinfo=timezone.utc)
+        submitted_time, _ = _get_time_delta(time_delta)
 
         # Get end time (from JobSet conditions)
         end_time = _get_end_time_from_conditions(job)
         if end_time != '-':
             end_time = _format_timestamp(end_time)
 
+        job_length = _get_job_length(start_time, end_time)
+
         if all_users:
             rows.append(
                 [
@@ -638,6 +661,7 @@ def show_status_table(
                     submitted_time,
                     start_time,
                     end_time,
+                    job_length,
                     job['metadata']['creationTimestamp'],
                 ]
             )
@@ -654,6 +678,7 @@ def show_status_table(
                     submitted_time,
                     start_time,
                     end_time,
+                    job_length,
                     job['metadata']['creationTimestamp'],
                 ]
             )

konduktor/backends/pod_utils.py

@@ -1,6 +1,7 @@
 """Pod utils: handles pod spec creation and manipulation"""
 
 import base64
+import json
 import os
 import tempfile
 import typing
@@ -284,7 +285,7 @@ def create_pod_spec(task: 'konduktor.Task') -> Dict[str, Any]:
     pod_config['kubernetes']['pod_config']['spec']['containers'][0]['env'] = list(
         env_map.values()
     )
-    logger.debug(f'rendered pod spec: \n\t{pod_config}')
+    logger.debug(f'rendered pod spec: \n\t{json.dumps(pod_config, indent=2)}')
 
     # validate pod spec using json schema
     try:

konduktor/templates/pod.yaml.j2

@@ -161,6 +161,10 @@ kubernetes:
           - name: git-ssh-secret
             mountPath: /run/konduktor/git-ssh-secret
           {% endif %}
+          {% if tailscale_secret %}
+          - name: tailscale-state
+            mountPath: /var/lib/tailscale
+          {% endif %}
           command: ["bash", "-c"]
           args:
             - |
@@ -317,18 +321,19 @@ kubernetes:
               export TS_HOSTNAME=$(echo "$POD_NAME" | sed 's/-[^-]*$//')
               $(prefix_cmd) echo "TS_HOSTNAME=${TS_HOSTNAME}" >> /etc/environment
               function InstallTailscale {
-                while ! tailscale status >/dev/null 2>&1; do
+                if ! command -v tailscale >/dev/null 2>&1; then
+                  $(prefix_cmd) curl -fsSL https://tailscale.com/install.sh | DEBIAN_FRONTEND=noninteractive $(prefix_cmd) sh > ~/.konduktor/tmp/tailscale-install.log 2>&1
+                fi
+                if ! tailscale status >/dev/null 2>&1; then
                   $(prefix_cmd) mkdir -p /var/run/tailscale /var/cache/tailscale /var/lib/tailscale
-                  if ! command -v tailscale >/dev/null 2>&1; then
-                    $(prefix_cmd) curl -fsSL https://tailscale.com/install.sh | DEBIAN_FRONTEND=noninteractive $(prefix_cmd) sh > ~/.konduktor/tmp/tailscale-install.log 2>&1
-                  fi
-                  $(prefix_cmd) tailscaled --tun=userspace-networking --state=mem: >~/.konduktor/tmp/tailscaled.log 2>&1 &
-                  $(prefix_cmd) sleep 2
-                  $(prefix_cmd) timeout 5 tailscale up --auth-key=${TS_AUTHKEY} --ssh --hostname=${TS_HOSTNAME} || echo "tailscale up failed retrying"
-                  $(prefix_cmd) sleep 2
+                  $(prefix_cmd) nohup tailscaled --tun=userspace-networking >~/.konduktor/tmp/tailscaled.log 2>&1 &
+                fi
+                until tailscale status >/dev/null 2>&1; do
+                  $(prefix_cmd) tailscale up --auth-key=${TS_AUTHKEY} --ssh --hostname=${TS_HOSTNAME} --accept-dns=false || echo "tailscale up failed retrying"
                 done
                 $(prefix_cmd) echo "Tailscale is up"
                 $(prefix_cmd) tailscale status
+                $(prefix_cmd) tailscale netcheck
               }
               InstallTailscale | tee ~/.konduktor/tmp/tailscale-out.log
               {% if konduktor_debug %}
@@ -413,6 +418,10 @@ kubernetes:
         emptyDir:
           medium: "Memory"
           sizeLimit: 4Gi
+      {% if tailscale_secret %}
+      - name: tailscale-state
+        emptyDir: {}
+      {% endif %}
       - name: sync
         emptyDir: {}
       {% for secret_type, secret_name in mount_secrets.items() %}

konduktor/utils/log_utils.py

@@ -397,6 +397,8 @@ def tail_vicky_logs(
                 for line in response.iter_lines(decode_unicode=True):
                     if line:
                         payload = json.loads(line)
+                        if 'missing _msg field' in payload['_msg']:
+                            payload['_msg'] = ''
                         print(
                             f"{colorama.Fore.CYAN}{colorama.Style.BRIGHT} "
                             f"(job_name={job_name} worker_id={worker_id})"

{konduktor_nightly-0.1.0.dev20250819104842.dist-info → konduktor_nightly-0.1.0.dev20250821104804.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: konduktor-nightly
-Version: 0.1.0.dev20250819104842
+Version: 0.1.0.dev20250821104804
 Summary: GPU Cluster Health Management
 Author: Andrew Aikawa
 Author-email: asai@berkeley.edu

{konduktor_nightly-0.1.0.dev20250819104842.dist-info → konduktor_nightly-0.1.0.dev20250821104804.dist-info}/RECORD

@@ -1,4 +1,4 @@
-konduktor/__init__.py,sha256=JjXvsKhn2OfPRW0uQtLdrb4JnzbqDdv7tR9E6WsTax4,1574
+konduktor/__init__.py,sha256=vSTzrg0X_r9sPmZDztRtaQbXDhuOpoRTQWuv4z1wybA,1574
 konduktor/adaptors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 konduktor/adaptors/aws.py,sha256=s47Ra-GaqCQibzVfmD0pmwEWHif1EGO5opMbwkLxTCU,8244
 konduktor/adaptors/common.py,sha256=ZIqzjx77PIHUwpjfAQ1uX8B2aX78YMuGj4Bppd-MdyM,4183
@@ -10,8 +10,8 @@ konduktor/backends/constants.py,sha256=nt9G9AmFCOMwO4GuKgRQSzJJuKapOmaROp4_Y0tMF
 konduktor/backends/deployment.py,sha256=EHfB2uLeKFQ3maek9tx6XL4_sjQ-ax59DZA79Q3EkVs,5519
 konduktor/backends/deployment_utils.py,sha256=VGuL01rKe7p7PoVRI_cP4tiZRxHZ13nnTMG-bmDf7P0,28975
 konduktor/backends/jobset.py,sha256=OwgDog9nH-FoUmNU_H--C3U5jx70reTKL1l849M1k5A,8430
-konduktor/backends/jobset_utils.py,sha256=7fB8X4b2Q5BKFCIGME72dyeCfi-EemoMeJVnwtzcjq4,25184
-konduktor/backends/pod_utils.py,sha256=Jfv_CY8suF0e7QEaeQiNRRxRnOueLgPR8SfLEO7lnwc,15260
+konduktor/backends/jobset_utils.py,sha256=8YUIFg7mi33A5p_K9GhVdjllTdXRMj_GEg-iQ5Nj_iU,25708
+konduktor/backends/pod_utils.py,sha256=EkoyN4mMfnrwEG656J1kxPtNZQClo4X7JEyzV1F8oHg,15294
 konduktor/check.py,sha256=JennyWoaqSKhdyfUldd266KwVXTPJpcYQa4EED4a_BA,7569
 konduktor/cli.py,sha256=YW9vNlbayDIcHk5D8-qsVGwoOdRV1o9kFmF2jtBf3vE,56641
 konduktor/config.py,sha256=9upqgCCYvcu6fKw7tovEYC1MWTkAAir0_WHPdayylbI,15536
@@ -77,7 +77,7 @@ konduktor/serving.py,sha256=sh8TPAUXg23Bkt0ByatIMdxFFqzRm18HJTEkt3wHzdo,5147
 konduktor/task.py,sha256=97iLCo62qpN9wLGNPeFw64E8k1nch7AyySY3BUXHPWY,37496
 konduktor/templates/deployment.yaml.j2,sha256=uXFjDQaimbpFdAn2RJGaIvS_PzDY136cw_L3QMjz3ZA,3452
 konduktor/templates/jobset.yaml.j2,sha256=gURWl6uQv_OLni-LFy2E7ttjGOtuRDt5Vfs4ALH7fpI,1196
-konduktor/templates/pod.yaml.j2,sha256=gxBaHFwDfRE71nh3glYPsmirOr6Qn__-f6oHmQRP4QU,18809
+konduktor/templates/pod.yaml.j2,sha256=owid7Wpo-kmQ-Gvx1zVoyG_feOiKviGDsnWLTlReXPo,19086
 konduktor/usage/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 konduktor/usage/constants.py,sha256=gCL8afIHZhO0dcxbJGpESE9sCC1cBSbeRnQ8GwNOY4M,612
 konduktor/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -90,15 +90,15 @@ konduktor/utils/env_options.py,sha256=T41Slzf4Mzl-n45CGXXqdy2fCrYhPNZQ7RP5vmnN4x
 konduktor/utils/exceptions.py,sha256=5IFnN5bIUSBJv4KRRrCepk5jyY9EG5vWWQqbjCmP3NU,6682
 konduktor/utils/kubernetes_enums.py,sha256=SabUueF6Bpzbpa57gyH5VB65xla2N9l8CZmAeYTfGmM,176
 konduktor/utils/kubernetes_utils.py,sha256=7RThCOiyaALRqbwHZ40qMnBsbAgt669k0NHkxtfx7Bs,26205
-konduktor/utils/log_utils.py,sha256=jxm9ovPcJPOGfd2wOwPDEThRO25ETIV5a1DmAfLhqJc,16861
+konduktor/utils/log_utils.py,sha256=k43eGpSwIdGhNotC8w7_Hq-bbyQQTrHwMOAMct_gr9M,16978
 konduktor/utils/loki_utils.py,sha256=h2ZvZQr1nE_wXXsKsGMjhG2s2MXknNd4icydTR_ruKU,3539
 konduktor/utils/rich_utils.py,sha256=ycADW6Ij3wX3uT8ou7T8qxX519RxlkJivsLvUahQaJo,3583
 konduktor/utils/schemas.py,sha256=tBrKhnkfn9uKDYdlb4L2KgooW-muuhww7U8fu9zX-ms,18336
 konduktor/utils/subprocess_utils.py,sha256=WoFkoFhGecPR8-rF8WJxbIe-YtV94LXz9UG64SDhCY4,9448
 konduktor/utils/ux_utils.py,sha256=7-Lt3QbDVvBQUli5_U9lOdXKeC-ip8rZBpO9gQ6vPJw,7955
 konduktor/utils/validator.py,sha256=5C1kE57Eyj1OPnAbvojqMNHHtf5fnl47FK_vEttd8aw,4331
-konduktor_nightly-0.1.0.dev20250819104842.dist-info/LICENSE,sha256=MuuqTZbHvmqXR_aNKAXzggdV45ANd3wQ5YI7tnpZhm0,6586
-konduktor_nightly-0.1.0.dev20250819104842.dist-info/METADATA,sha256=YevrR3deaLynmnMl_mbF4nNSpMXXgL0PhHlnRS3Xlu8,4247
-konduktor_nightly-0.1.0.dev20250819104842.dist-info/WHEEL,sha256=XbeZDeTWKc1w7CSIyre5aMDU_-PohRwTQceYnisIYYY,88
-konduktor_nightly-0.1.0.dev20250819104842.dist-info/entry_points.txt,sha256=k3nG5wDFIJhNqsZWrHk4d0irIB2Ns9s47cjRWYsTCT8,48
-konduktor_nightly-0.1.0.dev20250819104842.dist-info/RECORD,,
+konduktor_nightly-0.1.0.dev20250821104804.dist-info/LICENSE,sha256=MuuqTZbHvmqXR_aNKAXzggdV45ANd3wQ5YI7tnpZhm0,6586
+konduktor_nightly-0.1.0.dev20250821104804.dist-info/METADATA,sha256=qQPp3F_j37Mx4Ios1of0b_961SL9jkOJd-5sZ-_mFwE,4247
+konduktor_nightly-0.1.0.dev20250821104804.dist-info/WHEEL,sha256=XbeZDeTWKc1w7CSIyre5aMDU_-PohRwTQceYnisIYYY,88
+konduktor_nightly-0.1.0.dev20250821104804.dist-info/entry_points.txt,sha256=k3nG5wDFIJhNqsZWrHk4d0irIB2Ns9s47cjRWYsTCT8,48
+konduktor_nightly-0.1.0.dev20250821104804.dist-info/RECORD,,