konduktor-nightly 0.1.0.dev20250603105033__py3-none-any.whl → 0.1.0.dev20250605105049__py3-none-any.whl

konduktor/__init__.py

@@ -14,7 +14,7 @@ __all__ = [
 ]
 
 # Replaced with the current commit when building the wheels.
-_KONDUKTOR_COMMIT_SHA = 'cf8484c6676903ded1f8aa2758f8cac533329c05'
+_KONDUKTOR_COMMIT_SHA = 'b5ac51935edc296ff721f11d112e64438b366608'
 os.makedirs(os.path.expanduser('~/.konduktor'), exist_ok=True)
 
 
@@ -48,5 +48,5 @@ def _get_git_commit():
 
 
 __commit__ = _get_git_commit()
-__version__ = '1.0.0.dev0.1.0.dev20250603105033'
+__version__ = '1.0.0.dev0.1.0.dev20250605105049'
 __root_dir__ = os.path.dirname(os.path.abspath(__file__))

konduktor/backends/jobset_utils.py

@@ -300,6 +300,7 @@ def create_jobset(
                 'accelerator_type': accelerator_type,
                 'num_accelerators': num_accelerators,
                 'completions': task.resources.get_completions(),
+                'max_restarts': task.resources.get_max_restarts(),
                 **_JOBSET_METADATA_LABELS,
             },
             temp.name,

konduktor/cli.py

@@ -102,6 +102,7 @@ def _make_task_with_overrides(
     memory: Optional[str] = None,
     instance_type: Optional[str] = None,
     num_nodes: Optional[int] = None,
+    max_restarts: Optional[int] = None,
     image_id: Optional[str] = None,
     disk_size: Optional[int] = None,
     env: Optional[List[Tuple[str, str]]] = None,
@@ -147,6 +148,10 @@ def _make_task_with_overrides(
 
     task.set_resources_override(override_params)
 
+    if max_restarts is not None:
+        assert task.resources is not None
+        task.resources.job_config['max_restarts'] = max_restarts
+
     if num_nodes is not None:
         task.num_nodes = num_nodes
     if name is not None:
@@ -185,6 +190,16 @@ _TASK_OPTIONS = [
             'supplied.'
         ),
     ),
+    click.option(
+        '--max-restarts',
+        required=False,
+        type=int,
+        help=(
+            'Maximum number of jobset restarts allowed. Overrides YAML.'
+            'Overrides the "max_restarts" config in the YAML if both are '
+            'supplied.'
+        ),
+    ),
     click.option(
         '--cpus',
         default=None,
@@ -631,6 +646,7 @@ def launch(
     cpus: Optional[str],
     memory: Optional[str],
     num_nodes: Optional[int],
+    max_restarts: Optional[int],
     image_id: Optional[str],
     env_file: Optional[Dict[str, str]],
     env: List[Tuple[str, str]],
@@ -654,6 +670,7 @@ def launch(
         cpus=cpus,
         memory=memory,
         num_nodes=num_nodes,
+        max_restarts=max_restarts,
         image_id=image_id,
         env=env,
         disk_size=disk_size,
@@ -962,7 +979,7 @@ def create(kind, from_file, from_directory, inline, name):
                 old_name = s.metadata.name
                 click.echo(f'Found existing git-ssh secret: {old_name}, deleting it.')
                 kubernetes_utils.delete_secret(
-                    secret_name=old_name, namespace=namespace, context=context
+                    name=old_name, namespace=namespace, context=context
                 )
                 break
 

konduktor/resource.py

@@ -124,7 +124,7 @@ class Resources:
         self._set_cpus(cpus)
         self._set_memory(memory)
         self._set_accelerators(accelerators)
-        self.job_config = job_config
+        self.job_config = job_config or {}
 
         # TODO: move these out of init to prevent repeated calls.
         self._try_validate_cpus_mem()
@@ -386,8 +386,17 @@ class Resources:
         return accel_str
 
     def get_completions(self) -> Optional[int]:
-        if self.job_config and self.job_config['completions']:
-            return int(self.job_config['completions'])
+        value = self.job_config.get('completions')
+        return int(value) if value is not None else None
+
+    def get_max_restarts(self) -> Optional[int]:
+        value = self.job_config.get('max_restarts')
+        if value is not None:
+            value = int(value)
+            if value < 0:
+                with ux_utils.print_exception_no_traceback():
+                    raise ValueError('max_restarts must be a non-negative integer')
+            return value
         return None
 
     def copy(self, **override) -> 'Resources':

konduktor/templates/jobset.yaml.j2

@@ -16,6 +16,10 @@ jobset:
     annotations: {}
   spec:
     ttlSecondsAfterFinished: 259200 # 3 days
+    {% if max_restarts %}
+    failurePolicy:
+      maxRestarts: {{ max_restarts }}
+    {% endif %}
     replicatedJobs:
     - name: workers
       template:

konduktor/templates/pod.yaml.j2

@@ -183,8 +183,8 @@ kubernetes:
 
                 # Set root password if SSHKEY is provided
                 # Enable root login in SSH configuration
-                $(prefix_cmd) sed -i '/^#PermitRootLogin/c\PermitRootLogin yes' /etc/ssh/sshd_config
-                $(prefix_cmd) sed -i '/^PermitRootLogin/c\PermitRootLogin yes' /etc/ssh/sshd_config
+                $(prefix_cmd) sed -i '/^#PermitRootLogin/c\PermitRootLogin without-password' /etc/ssh/sshd_config
+                $(prefix_cmd) sed -i '/^PermitRootLogin/c\PermitRootLogin without-password' /etc/ssh/sshd_config
                 $(prefix_cmd) echo "Root login is enabled."
 
                 # Create the .ssh directory and authorized_keys file if they don't exist

konduktor/utils/accelerator_registry.py

@@ -1,13 +1,6 @@
 """Accelerator registry."""
 
-_ACCELERATORS = [
-    'A100',
-    'A100-80GB',
-    'B200',
-    'H100',
-    'H200',
-    'L40S',
-]
+_ACCELERATORS = ['A100', 'A100-80GB', 'B200', 'H100', 'H200', 'L40S', 'T4']
 
 
 def canonicalize_accelerator_name(accelerator: str) -> str:

konduktor/utils/schemas.py

@@ -427,6 +427,9 @@ def get_job_schema():
             'completions': {
                 'type': 'number',
             },
+            'max_restarts': {
+                'type': 'integer',
+            },
         },
     }
 

{konduktor_nightly-0.1.0.dev20250603105033.dist-info → konduktor_nightly-0.1.0.dev20250605105049.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: konduktor-nightly
-Version: 0.1.0.dev20250603105033
+Version: 0.1.0.dev20250605105049
 Summary: GPU Cluster Health Management
 Author: Andrew Aikawa
 Author-email: asai@berkeley.edu

{konduktor_nightly-0.1.0.dev20250603105033.dist-info → konduktor_nightly-0.1.0.dev20250605105049.dist-info}/RECORD

@@ -1,4 +1,4 @@
-konduktor/__init__.py,sha256=ChNdvAf2A-WmTHORSS31GGfP_dFREGg45VqQjZov4bo,1540
+konduktor/__init__.py,sha256=6W1DQ2KPTCksSVesiBXKLOt4MwKcbZL8C1ot4Ov3dYg,1540
 konduktor/adaptors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 konduktor/adaptors/aws.py,sha256=s47Ra-GaqCQibzVfmD0pmwEWHif1EGO5opMbwkLxTCU,8244
 konduktor/adaptors/common.py,sha256=ZIqzjx77PIHUwpjfAQ1uX8B2aX78YMuGj4Bppd-MdyM,4183
@@ -7,9 +7,9 @@ konduktor/authentication.py,sha256=_mVy3eqoKohicHostFiGwG1-2ybxP-l7ouofQ0LRlCY,4
 konduktor/backends/__init__.py,sha256=1Q6sqqdeMYarpTX_U-QVywJYf7idiUTRsyP-E4BQSOw,129
 konduktor/backends/backend.py,sha256=qh0bp94lzoTYZkzyQv2-CVrB5l91FkG2vclXg24UFC0,2910
 konduktor/backends/jobset.py,sha256=UdhwAuZODLMbLY51Y2zOBsh6wg4Pb84oHVvUKzx3Z2w,8434
-konduktor/backends/jobset_utils.py,sha256=diGpy-qpsQeVMFZVQsMgG3HxJxl2huxqbRU6FMA0QHY,21363
+konduktor/backends/jobset_utils.py,sha256=DSdWdCUPdTh9EzFG0z_GoHVeCW49kLwTujaKjC1ko3I,21430
 konduktor/check.py,sha256=JennyWoaqSKhdyfUldd266KwVXTPJpcYQa4EED4a_BA,7569
-konduktor/cli.py,sha256=qiTFut28crvcXOoSvnN3NcTb-xPmtFB336zdt9Q2bxU,33370
+konduktor/cli.py,sha256=4eYDqSvwEn38rDNk6fKiSk4BwdmSna2XonUrI9-o7w0,33903
 konduktor/config.py,sha256=J50JxC6MsXMnlrJPXdDUMr38C89xvOO7mR8KJ6fyils,15520
 konduktor/constants.py,sha256=T3AeXXxuQHINW_bAWyztvDeS8r4g8kXBGIwIq13cys0,1814
 konduktor/controller/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -68,14 +68,14 @@ konduktor/manifests/controller_deployment.yaml,sha256=6p3oSLkEVONZsvKZGqVop0Dhn4
 konduktor/manifests/dashboard_deployment.yaml,sha256=xJLd4FbPMAosI0fIv5_8y7dV9bw0Vsf81l-w4MB_aU8,2837
 konduktor/manifests/dmesg_daemonset.yaml,sha256=pSWt7YOeTYjS0l0iki1fvHOs7MhY-sH-RQfVW6JJyno,1391
 konduktor/manifests/pod_cleanup_controller.yaml,sha256=hziL1Ka1kCAEL9R7Tjvpb80iw1vcq9_3gwHCu75Bi0A,3939
-konduktor/resource.py,sha256=w2PdIrmQaJWA-GLSmVBcg4lxwuxvPulz35_YSKa5o24,19254
+konduktor/resource.py,sha256=Fg4kon7jQ9xDo9Iz8Q0J8doIRmTkSwIhYXLH6jbtRO8,19610
 konduktor/task.py,sha256=ofwd8WIhfD6C3ThLcv6X3GUzQHyZ6ddjUagE-umF4K0,35207
-konduktor/templates/jobset.yaml.j2,sha256=onYiHtXAgk-XBtji994hPu_g0hxnLzvmfxwjbdKdeZc,960
-konduktor/templates/pod.yaml.j2,sha256=JvDruGpBbRHSklqNEeKSvPH0Y1uldvcylqLUaIcguuQ,16086
+konduktor/templates/jobset.yaml.j2,sha256=rdURknodtgLp4zoA2PX86Nn4wPpi3tr5l4IG55aWBRg,1059
+konduktor/templates/pod.yaml.j2,sha256=7KyUy4orakJ8nI7ee8AIGGY4HHfhs-6zwUlAzhekZHw,16112
 konduktor/usage/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 konduktor/usage/constants.py,sha256=gCL8afIHZhO0dcxbJGpESE9sCC1cBSbeRnQ8GwNOY4M,612
 konduktor/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-konduktor/utils/accelerator_registry.py,sha256=1tpVIaM0UZ3w1desPhVEwNCUruamhP-igDZrcfaoRWI,574
+konduktor/utils/accelerator_registry.py,sha256=LmhLPR-N9kxrk0UBYYcF1O6ADv9HHwsn_Pc3SOYFNzs,553
 konduktor/utils/annotations.py,sha256=oy2-BLydkFt3KWkXDuaGY84d6b7iISuy4eAT9uXk0Fc,2225
 konduktor/utils/base64_utils.py,sha256=mF-Tw98mFRG70YE4w6s9feuQSCYZHOb8YatBZwMugyI,3130
 konduktor/utils/common_utils.py,sha256=4yG5Kjvu1hu6x2nKNaaCUKQNrheUaG61Qe913MFPry8,15060
@@ -87,12 +87,12 @@ konduktor/utils/kubernetes_utils.py,sha256=VG7qatUFyWHY-PCQ8fYWh2kn2TMwfg84cn-Vk
 konduktor/utils/log_utils.py,sha256=oFCKkYKCS_e_GRw_-0F7WsiIZNqJL1RZ4cD5-zh59Q4,9765
 konduktor/utils/loki_utils.py,sha256=h2ZvZQr1nE_wXXsKsGMjhG2s2MXknNd4icydTR_ruKU,3539
 konduktor/utils/rich_utils.py,sha256=ycADW6Ij3wX3uT8ou7T8qxX519RxlkJivsLvUahQaJo,3583
-konduktor/utils/schemas.py,sha256=2fHsTi3t9q3LXqOPrcpkmPsMbaoJBnuJstd6ULmDiUo,16455
+konduktor/utils/schemas.py,sha256=VGPERAso2G4sVAznsJ80qT2Q-I_EFxXw6Rfcw-vkYgQ,16535
 konduktor/utils/subprocess_utils.py,sha256=WoFkoFhGecPR8-rF8WJxbIe-YtV94LXz9UG64SDhCY4,9448
 konduktor/utils/ux_utils.py,sha256=czCwiS1bDqgeKtzAJctczpLwFZzAse7WuozdvzEFYJ4,7437
 konduktor/utils/validator.py,sha256=tgBghVyedyzGx84-U2Qfoh_cJBE3oUk9gclMW90ORks,691
-konduktor_nightly-0.1.0.dev20250603105033.dist-info/LICENSE,sha256=MuuqTZbHvmqXR_aNKAXzggdV45ANd3wQ5YI7tnpZhm0,6586
-konduktor_nightly-0.1.0.dev20250603105033.dist-info/METADATA,sha256=oSHm7k_dhVrpXPt8R0bhBnRfGjMoyBep9nkpBMqP7Qo,4289
-konduktor_nightly-0.1.0.dev20250603105033.dist-info/WHEEL,sha256=XbeZDeTWKc1w7CSIyre5aMDU_-PohRwTQceYnisIYYY,88
-konduktor_nightly-0.1.0.dev20250603105033.dist-info/entry_points.txt,sha256=k3nG5wDFIJhNqsZWrHk4d0irIB2Ns9s47cjRWYsTCT8,48
-konduktor_nightly-0.1.0.dev20250603105033.dist-info/RECORD,,
+konduktor_nightly-0.1.0.dev20250605105049.dist-info/LICENSE,sha256=MuuqTZbHvmqXR_aNKAXzggdV45ANd3wQ5YI7tnpZhm0,6586
+konduktor_nightly-0.1.0.dev20250605105049.dist-info/METADATA,sha256=uZGkLvGqATI96wS6l8jrBSQo7B9x2VPBwaZw-PhDWa0,4289
+konduktor_nightly-0.1.0.dev20250605105049.dist-info/WHEEL,sha256=XbeZDeTWKc1w7CSIyre5aMDU_-PohRwTQceYnisIYYY,88
+konduktor_nightly-0.1.0.dev20250605105049.dist-info/entry_points.txt,sha256=k3nG5wDFIJhNqsZWrHk4d0irIB2Ns9s47cjRWYsTCT8,48
+konduktor_nightly-0.1.0.dev20250605105049.dist-info/RECORD,,