konduktor-nightly 0.1.0.dev20250324104653__py3-none-any.whl → 0.1.0.dev20250326104701__py3-none-any.whl

konduktor/__init__.py

@@ -14,7 +14,7 @@ __all__ = [
 ]
 
 # Replaced with the current commit when building the wheels.
-_KONDUKTOR_COMMIT_SHA = '1099bba7fab85c3954acf0b9de385239105f2047'
+_KONDUKTOR_COMMIT_SHA = 'c26d3750b0c2f6f2604b4747a767367294157533'
 
 
 def _get_git_commit():
@@ -47,5 +47,5 @@ def _get_git_commit():
 
 
 __commit__ = _get_git_commit()
-__version__ = '1.0.0.dev0.1.0.dev20250324104653'
+__version__ = '1.0.0.dev0.1.0.dev20250326104701'
 __root_dir__ = os.path.dirname(os.path.abspath(__file__))

konduktor/backends/jobset.py

@@ -58,6 +58,7 @@ def _raise_job_error(job):
 
 
 def _wait_for_jobset_start(namespace: str, job_name: str):
+    time.sleep(2)
     start = time.time()
     timeout = config.get_nested(
         ('kubernetes', 'provision_timeout'),

konduktor/utils/loki_utils.py

@@ -14,7 +14,7 @@ from konduktor import logging
 
 logger = logging.get_logger(__name__)
 
-LOKI_PORT = 3100
+LOKI_REMOTE_PORT = 3100
 WEBSOCKET_TIMEOUT = 10
 INFINITY = 999999
 
@@ -23,7 +23,7 @@ async def _read_loki_logs(loki_url: str, timeout: int, job_name: str, worker_id:
     ws = await asyncio.wait_for(websockets.connect(loki_url), timeout=WEBSOCKET_TIMEOUT)
     logger.info(
         f'{colorama.Fore.YELLOW}Tailing logs from Loki. '
-        f'Forwarding to port {LOKI_PORT}. Press Ctrl+C to stop. '
+        f'Forwarding from remote port {LOKI_REMOTE_PORT}. Press Ctrl+C to stop. '
         f'{colorama.Style.RESET_ALL}'
     )
     try:
@@ -44,7 +44,9 @@ async def _read_loki_logs(loki_url: str, timeout: int, job_name: str, worker_id:
                 logger.debug(f'Dropped log: {message}')
                 continue
     except asyncio.exceptions.TimeoutError:
-        logger.debug('Websocket timed-out, closing the connection!')
+        logger.debug('Websocket timed-out, closing log stream!')
+    except KeyboardInterrupt:
+        logger.debug('Keyboard interrupt, closing log stream!')
 
 
 def tail_loki_logs_ws(
@@ -57,23 +59,25 @@ def tail_loki_logs_ws(
         # until we reach the end of the log and then invoke tail again.
         # Also include checks that the job is running/ever ran.
         raise ValueError('num_logs must be less than or equal to 5000')
-    loki_url = f'ws://localhost:{LOKI_PORT}/loki/api/v1/tail'
-    params = {
-        'query': urllib.parse.quote(
-            f'{{k8s_job_name="{job_name}-workers-0"}} '
-            f' | batch_kubernetes_io_job_completion_index = `{worker_id}`'
-        ),
-        'limit': num_logs,
-        'delay': 5,
-        # TODO(asaiacai): need to auto-generate the start and end times.
-    }
-
-    query_string = '&'.join(f'{key}={value}' for key, value in params.items())
-    loki_url += f'?{query_string}'
-
     loki_svc = kr8s.objects.Service.get('loki', namespace='loki')
-    timeout = INFINITY if follow else WEBSOCKET_TIMEOUT
-    with kr8s.portforward.PortForward(loki_svc, LOKI_PORT):
+    with kr8s.portforward.PortForward(
+        loki_svc, LOKI_REMOTE_PORT, local_port='auto'
+    ) as port:
+        loki_url = f'ws://localhost:{port}/loki/api/v1/tail'
+        logger.debug(f'Loki URL: {loki_url}')
+        params = {
+            'query': urllib.parse.quote(
+                f'{{k8s_job_name="{job_name}-workers-0"}} '
+                f' | batch_kubernetes_io_job_completion_index = `{worker_id}`'
+            ),
+            'limit': num_logs,
+            'delay': 5,
+            # TODO(asaiacai): need to auto-generate the start and end times.
+        }
+
+        query_string = '&'.join(f'{key}={value}' for key, value in params.items())
+        loki_url += f'?{query_string}'
+        timeout = INFINITY if follow else WEBSOCKET_TIMEOUT
         asyncio.run(_read_loki_logs(loki_url, timeout, job_name, worker_id))
 
 
@@ -82,4 +86,4 @@ def tail_loki_logs_ws(
 
 # Run the WebSocket log tailing function
 if __name__ == '__main__':
-    tail_loki_logs_ws('tune-c3c8', worker_id=0, follow=False)
+    tail_loki_logs_ws('tune-bc43', worker_id=0, follow=True)

{konduktor_nightly-0.1.0.dev20250324104653.dist-info → konduktor_nightly-0.1.0.dev20250326104701.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: konduktor-nightly
-Version: 0.1.0.dev20250324104653
+Version: 0.1.0.dev20250326104701
 Summary: GPU Cluster Health Management
 Author: Andrew Aikawa
 Author-email: asai@berkeley.edu

{konduktor_nightly-0.1.0.dev20250324104653.dist-info → konduktor_nightly-0.1.0.dev20250326104701.dist-info}/RECORD

@@ -1,10 +1,10 @@
-konduktor/__init__.py,sha256=Mgfb4Jmm8gV-yHbe8YzLkWOSZwRg690OOkvZPcBsgZg,1477
+konduktor/__init__.py,sha256=S2aE6nXOYJ8l9jDoTBVddj92D9nF5jB65UiR5_Q1UjE,1477
 konduktor/adaptors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 konduktor/adaptors/common.py,sha256=mYb_6c3u5MghtiFfiW5OO-EH6t7cIR5npbkgUmz6FYE,3517
 konduktor/adaptors/gcp.py,sha256=liCm4_D_qSci0DZA2t5bckLIoGDkJ8qx31EO_hSBzo0,3751
 konduktor/backends/__init__.py,sha256=1Q6sqqdeMYarpTX_U-QVywJYf7idiUTRsyP-E4BQSOw,129
 konduktor/backends/backend.py,sha256=qh0bp94lzoTYZkzyQv2-CVrB5l91FkG2vclXg24UFC0,2910
-konduktor/backends/jobset.py,sha256=BN_2aPuaj_oA9Rd5px61Nc-Dgys7Kt8Oz2z4qdAfhm0,8233
+konduktor/backends/jobset.py,sha256=lh_PihQgM0tmVryCpjSsZjWug8hBnJr7ua9lqk0qEAM,8251
 konduktor/backends/jobset_utils.py,sha256=FR_IDoDU8noTE1qSG-L0KAIe52geeGtekzhInnmjgwc,16636
 konduktor/check.py,sha256=hIrxDMKaGX2eZP-Pj9TCymGUHQAp93m48Gj3XMiqadA,7833
 konduktor/cli.py,sha256=90bnh3nIobfBkzqS_SXgw9Z8Zqh4ouwpLDj0kx_6kL8,23562
@@ -81,14 +81,14 @@ konduktor/utils/exceptions.py,sha256=GBOFIkk9nikqWGR0FXGXOWVVImoH7nWnMl_L3Oux3fo
 konduktor/utils/kubernetes_enums.py,sha256=SabUueF6Bpzbpa57gyH5VB65xla2N9l8CZmAeYTfGmM,176
 konduktor/utils/kubernetes_utils.py,sha256=NGBredKPWpZC8VNlwTfWLhHnc-p68d5xlxT-0e92738,23556
 konduktor/utils/log_utils.py,sha256=lgHCq4OdtJNfbpso-uYGONUCVNsUrUkUWjROarsHt6s,9897
-konduktor/utils/loki_utils.py,sha256=SrRwTYHWGfiqqufY2XcKk4imgdxUCFBZ5oxyhCEJI0Y,3221
+konduktor/utils/loki_utils.py,sha256=ND1pbbbFhLhLKw3870j44LpR_9MB0EkDJSs5K7nWdY4,3473
 konduktor/utils/rich_utils.py,sha256=kdjNe6S2LlpOxyzhFHqMzCz7g4ROC4e7TPWgcbRsrQE,3577
 konduktor/utils/schemas.py,sha256=4Goihc-NpFQpiJ7RSiKirAIPNWqw_DV_TRqVwejqTDY,17479
 konduktor/utils/subprocess_utils.py,sha256=WoFkoFhGecPR8-rF8WJxbIe-YtV94LXz9UG64SDhCY4,9448
 konduktor/utils/ux_utils.py,sha256=NPNu3Igu2Z9Oq77ghJhy_fIxQZTXWr9BtKyxN3Wslzo,7164
 konduktor/utils/validator.py,sha256=tgBghVyedyzGx84-U2Qfoh_cJBE3oUk9gclMW90ORks,691
-konduktor_nightly-0.1.0.dev20250324104653.dist-info/LICENSE,sha256=MuuqTZbHvmqXR_aNKAXzggdV45ANd3wQ5YI7tnpZhm0,6586
-konduktor_nightly-0.1.0.dev20250324104653.dist-info/METADATA,sha256=_4iDoVPSzhlCx8f0DQaAEBnQ5RS4TiEW5lsvUGMGdsg,4070
-konduktor_nightly-0.1.0.dev20250324104653.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
-konduktor_nightly-0.1.0.dev20250324104653.dist-info/entry_points.txt,sha256=k3nG5wDFIJhNqsZWrHk4d0irIB2Ns9s47cjRWYsTCT8,48
-konduktor_nightly-0.1.0.dev20250324104653.dist-info/RECORD,,
+konduktor_nightly-0.1.0.dev20250326104701.dist-info/LICENSE,sha256=MuuqTZbHvmqXR_aNKAXzggdV45ANd3wQ5YI7tnpZhm0,6586
+konduktor_nightly-0.1.0.dev20250326104701.dist-info/METADATA,sha256=f-kTqSFtPMyPuMFZGHCM9FZuqZ76U6iWpblqvFiMbXA,4070
+konduktor_nightly-0.1.0.dev20250326104701.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
+konduktor_nightly-0.1.0.dev20250326104701.dist-info/entry_points.txt,sha256=k3nG5wDFIJhNqsZWrHk4d0irIB2Ns9s47cjRWYsTCT8,48
+konduktor_nightly-0.1.0.dev20250326104701.dist-info/RECORD,,