konduktor-nightly 0.1.0.dev20250530104807__py3-none-any.whl → 0.1.0.dev20250531104602__py3-none-any.whl

konduktor/__init__.py

@@ -14,7 +14,7 @@ __all__ = [
 ]
 
 # Replaced with the current commit when building the wheels.
-_KONDUKTOR_COMMIT_SHA = '3424c3df69fe44e339d2575f7090470185d5f549'
+_KONDUKTOR_COMMIT_SHA = 'cf8484c6676903ded1f8aa2758f8cac533329c05'
 os.makedirs(os.path.expanduser('~/.konduktor'), exist_ok=True)
 
 
@@ -48,5 +48,5 @@ def _get_git_commit():
 
 
 __commit__ = _get_git_commit()
-__version__ = '1.0.0.dev0.1.0.dev20250530104807'
+__version__ = '1.0.0.dev0.1.0.dev20250531104602'
 __root_dir__ = os.path.dirname(os.path.abspath(__file__))

konduktor/backends/jobset_utils.py

@@ -184,6 +184,32 @@ def create_pod_spec(task: 'konduktor.Task') -> Dict[str, Any]:
                     f'Failed to set k8s secret {secret_name}: \n{result}'
                 )
 
+    # Mount the user's secrets
+    git_ssh_secret_name = None
+    env_secret_envs = []
+
+    context = kubernetes_utils.get_current_kube_config_context_name()
+    namespace = kubernetes_utils.get_kube_config_context_namespace(context)
+    user_hash = common_utils.get_user_hash()
+    label_selector = f'konduktor/owner={user_hash}'
+    user_secrets = kubernetes_utils.list_secrets(
+        namespace, context, label_filter=label_selector
+    )
+
+    for secret in user_secrets:
+        kind = kubernetes_utils.get_secret_kind(secret)
+        if kind == 'git-ssh' and git_ssh_secret_name is None:
+            git_ssh_secret_name = secret.metadata.name
+        elif kind == 'env':
+            secret_name = secret.metadata.name
+            key = next(iter(secret.data))
+            env_secret_envs.append(
+                {
+                    'name': key,
+                    'valueFrom': {'secretKeyRef': {'name': secret_name, 'key': key}},
+                }
+            )
+
     with tempfile.NamedTemporaryFile() as temp:
         common_utils.fill_template(
             'pod.yaml.j2',
@@ -210,6 +236,9 @@ def create_pod_spec(task: 'konduktor.Task') -> Dict[str, Any]:
                 # SSH
                 'enable_ssh': enable_ssh,
                 'secret_name': secret_name,
+                # Kinds of Secrets
+                # --kind git-ssh
+                'git_ssh': git_ssh_secret_name,
             },
             temp.name,
         )
@@ -218,16 +247,24 @@ def create_pod_spec(task: 'konduktor.Task') -> Dict[str, Any]:
         kubernetes_utils.combine_pod_config_fields(temp.name, pod_config)
         pod_config = common_utils.read_yaml(temp.name)
 
-        for env_var in pod_config['kubernetes']['pod_config']['spec']['containers'][0][
-            'env'
-        ]:
-            if env_var['name'] in task.envs:
-                env_var['value'] = task.envs.pop(env_var['name'])
+    # Priority order: task.envs > secret envs > existing pod_config envs
+    existing_envs = pod_config['kubernetes']['pod_config']['spec']['containers'][0].get(
+        'env', []
+    )
+    env_map = {env['name']: env for env in existing_envs}
+
+    # Inject secret envs
+    for env in env_secret_envs:
+        env_map[env['name']] = env
 
-        for k, v in task.envs.items():
-            pod_config['kubernetes']['pod_config']['spec']['containers'][0][
-                'env'
-            ].append({'name': k, 'value': v})
+    # Inject task.envs
+    for k, v in task.envs.items():
+        env_map[k] = {'name': k, 'value': v}
+
+    # Replace the container's env section with the merged and prioritized map
+    pod_config['kubernetes']['pod_config']['spec']['containers'][0]['env'] = list(
+        env_map.values()
+    )
 
     # TODO(asaiacai): have some schema validations. see
     # https://github.com/skypilot-org/skypilot/pull/4466

konduktor/cli.py

@@ -35,7 +35,9 @@ each other.
 """
 
 import os
+import pathlib
 import shlex
+from base64 import b64encode
 from typing import Any, Dict, List, Optional, Tuple
 
 import click
@@ -72,7 +74,7 @@ def _parse_env_var(env_var: str) -> Tuple[str, str]:
     ret = tuple(env_var.split('=', 1))
     if len(ret) != 2:
         raise click.UsageError(
-            f'Invalid env var: {env_var}. Must be in the form of KEY=VAL ' 'or KEY.'
+            f'Invalid env var: {env_var}. Must be in the form of KEY=VALUE'
         )
     return ret[0], ret[1]
 
@@ -780,6 +782,287 @@ def check(clouds: Tuple[str]):
     konduktor_check.check(clouds=clouds_arg)
 
 
+class KeyValueType(click.ParamType):
+    name = 'key=value'
+
+    def convert(self, value, param, ctx):
+        if '=' not in value:
+            self.fail(f'{value!r} is not a valid key=value pair', param, ctx)
+        key, val = value.split('=', 1)
+        return key, val
+
+
+_SECRET_CREATE_OPTIONS = [
+    click.option(
+        '--inline',
+        type=KeyValueType(),
+        help='Key=value pair to store as an env secret (only valid with --kind env).',
+    ),
+    click.option(
+        '--from-file',
+        '--from_file',
+        type=click.Path(dir_okay=False),
+        help='Path to a single file to store as a secret.',
+    ),
+    click.option(
+        '--from-directory',
+        '--from_directory',
+        type=click.Path(file_okay=False),
+        help='Path to a directory to store as a multi-file secret.',
+    ),
+    click.option(
+        '--kind',
+        default='default',
+        type=click.Choice(['default', 'env', 'git-ssh']),
+        help='Type of secret being created. More kinds coming soon.',
+    ),
+]
+
+
+@cli.group(cls=_NaturalOrderGroup)
+def secret():
+    """Manage secrets used in Konduktor.
+
+    USAGE: konduktor secret COMMAND
+
+    \b
+    Use one of the following COMMANDS:
+      create [FLAGS] [NAME]
+      delete [NAME]
+      list [FLAGS]
+
+    \b
+    Examples:
+      konduktor secret create --kind git-ssh --from-file=~/.ssh/id_rsa my-ssh-name
+      konduktor secret create --kind env --inline FOO=bar my-env-name
+      konduktor delete my-ssh-name
+      konduktor list
+
+    \b
+    For details on COMMAND ARGS:
+      konduktor secret create -h
+      konduktor secret delete -h
+      konduktor secret list -h
+    """
+
+
+@_add_click_options(_SECRET_CREATE_OPTIONS)
+@secret.command()
+@click.argument('name', required=True)
+def create(kind, from_file, from_directory, inline, name):
+    """Create a new secret."""
+
+    if not kubernetes_utils.is_k8s_resource_name_valid(name):
+        raise click.BadParameter(
+            f'Invalid secret name: {name}. '
+            f'Name must consist of lower case alphanumeric characters or -, '
+            f'and must start and end with alphanumeric characters.',
+        )
+
+    basename = name
+    secret_name = f'{basename}-{common_utils.get_user_hash()}'
+
+    context = kubernetes_utils.get_current_kube_config_context_name()
+    namespace = kubernetes_utils.get_kube_config_context_namespace(context)
+
+    from_file = os.path.expanduser(from_file) if from_file else None
+    from_directory = os.path.expanduser(from_directory) if from_directory else None
+
+    sources = [bool(from_file), bool(from_directory), bool(inline)]
+
+    if sources.count(True) > 1:
+        raise click.UsageError(
+            'Only one of --from-file, --from-directory, or --inline can be used.\n'
+            'Examples:\n'
+            f'  {colorama.Style.BRIGHT}konduktor secret create --kind git-ssh '
+            f'--from-file=~/.ssh/id_rsa my-ssh-name\n{colorama.Style.RESET_ALL}'
+            f'  {colorama.Style.BRIGHT}konduktor secret create --kind env '
+            f'--inline FOO=bar my-env-name{colorama.Style.RESET_ALL}'
+        )
+
+    if sources.count(True) == 0:
+        raise click.UsageError(
+            'You must specify one of --from-file, --from-directory, or --inline.\n'
+            'Examples:\n'
+            f'  {colorama.Style.BRIGHT}konduktor secret create --kind git-ssh '
+            f'--from-file=~/.ssh/id_rsa my-ssh-name\n{colorama.Style.RESET_ALL}'
+            f'  {colorama.Style.BRIGHT}konduktor secret create --kind env '
+            f'--inline FOO=bar my-env-name{colorama.Style.RESET_ALL}'
+        )
+
+    if from_file and not os.path.isfile(from_file):
+        raise click.BadParameter(
+            f'--from-file {from_file} does not exist or is not a file'
+        )
+    if from_directory and not os.path.isdir(from_directory):
+        raise click.BadParameter(
+            f'--from-directory {from_directory} does not exist or is not a directory'
+        )
+
+    if kind == 'git-ssh' and not from_file:
+        raise click.UsageError(
+            '--kind git-ssh requires --from-file (not --from-directory or --inline). \n'
+            'Example:\n'
+            f'  {colorama.Style.BRIGHT}konduktor secret create --kind git-ssh '
+            f'--from-file=~/.ssh/id_rsa my-ssh-name{colorama.Style.RESET_ALL}'
+        )
+    if kind == 'env' and not inline:
+        raise click.UsageError(
+            '--kind env requires --inline (not --from-file or --from-directory). \n'
+            'Example:\n'
+            f'  {colorama.Style.BRIGHT}konduktor secret create --kind env '
+            f'--inline FOO=bar my-env-name{colorama.Style.RESET_ALL}'
+        )
+
+    data = {}
+    if from_directory:
+        click.echo(f'Creating secret from directory: {from_directory}')
+        base_path = pathlib.Path(from_directory)
+        for path in base_path.rglob('*'):
+            if path.is_file():
+                rel_path = path.relative_to(base_path)
+                with open(path, 'rb') as f:
+                    data[str(rel_path)] = b64encode(f.read()).decode()
+    elif from_file:
+        click.echo(f'Creating secret from file: {from_file}')
+        key = os.path.basename(from_file)
+        if kind == 'git-ssh':
+            key = 'gitkey'
+        try:
+            with open(from_file, 'rb') as f:
+                data[key] = b64encode(f.read()).decode()
+        except OSError as e:
+            raise click.ClickException(f'Failed to read {kind} file {from_file}: {e}')
+    else:
+        click.echo('Creating secret from inline key=value pair')
+        key, value = inline
+        data = {key: b64encode(value.encode()).decode()}
+
+    secret_metadata = {
+        'name': secret_name,
+        'labels': {
+            'parent': 'konduktor',
+            'konduktor/owner': common_utils.get_user_hash(),
+            'konduktor/basename': basename,
+            'konduktor/secret-kind': kind or None,
+        },
+    }
+
+    # Limit --kind git-ssh secret to 1 max per user
+    # Overwrites if user trying to create more than 1
+    if kind == 'git-ssh':
+        user_hash = common_utils.get_user_hash()
+        label_selector = f'konduktor/owner={user_hash}'
+        existing = kubernetes_utils.list_secrets(
+            namespace, context, label_filter=label_selector
+        )
+        for s in existing:
+            labels = s.metadata.labels or {}
+            if labels.get('konduktor/secret-kind') == 'git-ssh':
+                old_name = s.metadata.name
+                click.echo(f'Found existing git-ssh secret: {old_name}, deleting it.')
+                kubernetes_utils.delete_secret(
+                    secret_name=old_name, namespace=namespace, context=context
+                )
+                break
+
+    ok, err = kubernetes_utils.set_secret(
+        secret_name=secret_name,
+        namespace=namespace,
+        context=context,
+        data=data,
+        secret_metadata=secret_metadata,
+    )
+    if not ok:
+        raise click.ClickException(f'Failed to create secret: {err}')
+    click.secho(f'Secret {basename} created in namespace {namespace}.', fg='green')
+
+
+@secret.command()
+@click.argument('name', required=True)
+def delete(name):
+    """Delete a secret by name."""
+
+    context = kubernetes_utils.get_current_kube_config_context_name()
+    namespace = kubernetes_utils.get_kube_config_context_namespace(context)
+    user_hash = common_utils.get_user_hash()
+
+    label_selector = f'konduktor/owner={user_hash}'
+    secrets = kubernetes_utils.list_secrets(
+        namespace, context, label_filter=label_selector
+    )
+
+    matches = [
+        s
+        for s in secrets
+        if s.metadata.labels and s.metadata.labels.get('konduktor/basename') == name
+    ]
+
+    if not matches:
+        raise click.ClickException(
+            f'No secret named "{name}" owned by you found in namespace {namespace}.'
+        )
+    elif len(matches) > 1:
+        raise click.ClickException(f'Multiple secrets with basename "{name}" found.')
+
+    full_name = matches[0].metadata.name
+
+    ok, err = kubernetes_utils.delete_secret(full_name, namespace, context)
+    if not ok:
+        raise click.ClickException(f'Failed to delete secret: {err}')
+    click.secho(f'Secret {name} deleted from namespace {namespace}.', fg='yellow')
+
+
+@secret.command(name='list')
+@click.option(
+    '--all-users',
+    '--all_users',
+    '-u',
+    is_flag=True,
+    default=False,
+    help='Show all secrets, including those not owned by the current user.',
+)
+def list_secrets(all_users: bool):
+    """List secrets in the namespace.
+    Defaults to only your secrets unless --all-users is set."""
+
+    context = kubernetes_utils.get_current_kube_config_context_name()
+    namespace = kubernetes_utils.get_kube_config_context_namespace(context)
+
+    if not all_users:
+        user_hash = common_utils.get_user_hash()
+        username = common_utils.get_cleaned_username()
+        label_selector = f'konduktor/owner={user_hash}'
+        secrets = kubernetes_utils.list_secrets(
+            namespace, context, label_filter=label_selector
+        )
+    else:
+        secrets = kubernetes_utils.list_secrets(namespace, context)
+
+    if not secrets:
+        if all_users:
+            click.secho(f'No secrets found in {namespace}.', fg='yellow')
+        else:
+            click.secho(f'No secrets found for {username} in {namespace}.', fg='yellow')
+        return
+
+    if all_users:
+        click.secho(f'All secrets in {namespace} namespace:\n', bold=True)
+    else:
+        click.secho(f'Secrets in {namespace} namespace owned by you:\n', bold=True)
+
+    for s in secrets:
+        labels = s.metadata.labels or {}
+        basename = labels.get('konduktor/basename', s.metadata.name)
+        kind = labels.get('konduktor/secret-kind', '(none)')
+        owner = labels.get('konduktor/owner', '(none)')
+
+        if all_users:
+            click.echo(f'{basename:30}   kind={kind:10}   owner={owner}')
+        else:
+            click.echo(f'{basename:30}   kind={kind:10}')
+
+
 def main():
     return cli()
 

konduktor/data/aws/s3.py

@@ -1033,6 +1033,13 @@ class S3Store(storage_utils.AbstractStore):
             os.path.expanduser(os.path.join(credentials_dir, f))
             for f in _CREDENTIAL_FILES
         ]
+
+        secret_metadata = {
+            'labels': {
+                'konduktor/secret-kind': 'S3',
+            },
+        }
+
         ok, result = kubernetes_utils.set_secret(
             secret_name=cls._AWS_SECRET_NAME,
             namespace=namespace,
@@ -1042,6 +1049,7 @@ class S3Store(storage_utils.AbstractStore):
                     credentials_files
                 )
             },
+            secret_metadata=secret_metadata,
         )
         if not ok:
             logger.error(f'Failed to set AWS credentials in k8s secret: \n{result}')

konduktor/data/gcp/gcs.py

@@ -887,6 +887,13 @@ class GcsStore(storage_utils.AbstractStore):
             os.path.expanduser(os.path.join(credentials_dir, f))
             for f in _CREDENTIAL_FILES
         ]
+
+        secret_metadata = {
+            'labels': {
+                'konduktor/secret-kind': 'GCS',
+            },
+        }
+
         ok, result = kubernetes_utils.set_secret(
             secret_name=cls._GCP_SECRET_NAME,
             namespace=namespace,
@@ -896,6 +903,7 @@ class GcsStore(storage_utils.AbstractStore):
                     credentials_files
                 )
             },
+            secret_metadata=secret_metadata,
         )
         if not ok:
             logger.error(f'Failed to set GCP credentials in k8s secret: \n{result}')

konduktor/templates/pod.yaml.j2

@@ -72,6 +72,10 @@ kubernetes:
                 name: {{ secret_name }}
                 key: PRIVKEY
           {% endif %}
+          {% if git_ssh %}
+          - name: GIT_SSH_COMMAND
+            value: "ssh -i /run/konduktor/git-ssh-secret/gitkey -o StrictHostKeyChecking=no"
+          {% endif %}
           # these are for compatibility with skypilot
           - name: SKYPILOT_NODE_IPS
             value: "{{ node_hostnames }}"
@@ -92,6 +96,10 @@ kubernetes:
           - name: {{ secret_type }}-secret
             mountPath: /run/konduktor/{{ secret_type }}-secret
           {% endfor %}
+          {% if git_ssh %}
+          - name: git-ssh-secret
+            mountPath: /run/konduktor/git-ssh-secret
+          {% endif %}
           command: ["bash", "-c"]
           args:
             - |
@@ -275,6 +283,9 @@ kubernetes:
               $(prefix_cmd) unzip /run/konduktor/s3-secret/awscredentials -d ~/.aws
               {% endif %}
               {% endfor %}
+              {% if git_ssh %}
+              $(prefix_cmd) echo "Unpacking GIT-SSH secret"
+              {% endif %}
               end_epoch=$(date +%s);
               $(prefix_cmd) echo "===== KONDUKTOR: Unpacking secrets credentials took $((end_epoch - start_epoch)) seconds ====="
 
@@ -337,6 +348,13 @@ kubernetes:
         secret:
           secretName: {{ secret_name }}
       {% endfor %}
+      {% if git_ssh %}
+      - name: git-ssh-secret
+        secret:
+          secretName: {{ git_ssh }}
+          defaultMode: 384
+      {% endif %}
+
         
       # TODO(asaiacai): should we add nodeSelectors here or leave to
       # kueue resource flavors. leaning towards defining

konduktor/utils/common_utils.py

@@ -193,6 +193,10 @@ def get_user_hash(force_fresh_hash: bool = False) -> str:
             local client.
     """
 
+    override = os.environ.get('KONDUKTOR_TEST_USER_HASH')
+    if override:
+        return override
+
     def _is_valid_user_hash(user_hash: Optional[str]) -> bool:
         if user_hash is None:
             return False

konduktor/utils/kubernetes_utils.py

@@ -33,6 +33,8 @@ DEFAULT_NAMESPACE = 'default'
 
 DEFAULT_SERVICE_ACCOUNT_NAME = 'konduktor-service-account'
 
+DNS_SUBDOMAIN_REGEX = r'^[a-z0-9]([-a-z0-9]*[a-z0-9])?$'
+
 MEMORY_SIZE_UNITS = {
     'B': 1,
     'K': 2**10,
@@ -579,6 +581,7 @@ def set_secret(
     namespace: str,
     context: Optional[str],
     data: Dict[str, str],
+    secret_metadata: Optional[Dict[str, Any]] = None,
 ) -> Tuple[bool, Optional[str]]:
     """
     Create/update a secret in a namespace. Values are encoded to base64.
@@ -588,26 +591,46 @@ def set_secret(
     ```
     """
     with _K8s_CLIENT_LOCK:
-        secret_exists, response = check_secret_exists(
-            secret_name=secret_name,
-            namespace=namespace,
-            context=context,
+        user_hash = common_utils.get_user_hash()
+
+        full_name = (
+            secret_metadata.get('name')
+            if secret_metadata and 'name' in secret_metadata
+            else secret_name
         )
+        assert isinstance(full_name, str), 'Secret name must be a string'
+
+        metadata: Dict[str, Any] = {
+            'name': full_name,
+            'labels': {
+                'parent': 'konduktor',
+                'konduktor/owner': user_hash,
+                'konduktor/basename': secret_name,
+            },
+        }
+
+        if secret_metadata:
+            metadata['labels'].update(secret_metadata.get('labels', {}))
 
-        secret_metadata = {'name': secret_name, 'labels': {'parent': 'konduktor'}}
         custom_metadata = config.get_nested(('kubernetes', 'custom_metadata'), {})
-        config.merge_k8s_configs(secret_metadata, custom_metadata)
+        config.merge_k8s_configs(metadata, custom_metadata)
 
         secret = kubernetes.client.V1Secret(
-            metadata=kubernetes.client.V1ObjectMeta(**secret_metadata),
+            metadata=kubernetes.client.V1ObjectMeta(**metadata),
             type='Opaque',
             data=data,
         )
 
+        secret_exists, _ = check_secret_exists(
+            secret_name=full_name,
+            namespace=namespace,
+            context=context,
+        )
+
         try:
             if secret_exists:
                 kube_client.core_api(context).patch_namespaced_secret(
-                    secret_name, namespace, secret
+                    full_name, namespace, secret
                 )
             else:
                 kube_client.core_api(context).create_namespaced_secret(
@@ -617,12 +640,50 @@ def set_secret(
             return False, str(e)
         else:
             logger.debug(
-                f'Secret {secret_name} in namespace {namespace} '
+                f'Secret {full_name} in namespace {namespace} '
                 f'in context {context} created/updated'
             )
             return True, None
 
 
+def list_secrets(
+    namespace: str,
+    context: Optional[str],
+    label_filter: Optional[str] = None,
+) -> List[kubernetes.client.V1Secret]:
+    """List all secrets in a namespace, optionally filtering by label."""
+    secrets = kube_client.core_api(context).list_namespaced_secret(namespace).items
+    if label_filter:
+        key, val = label_filter.split('=', 1)
+        return [
+            s
+            for s in secrets
+            if s.metadata.labels and s.metadata.labels.get(key) == val
+        ]
+    return secrets
+
+
+def delete_secret(
+    name: str,
+    namespace: str,
+    context: Optional[str],
+) -> Tuple[bool, Optional[str]]:
+    """Deletes a secret by name in the given namespace/context."""
+    try:
+        kube_client.core_api(context).delete_namespaced_secret(name, namespace)
+        logger.debug(f'Secret {name} deleted from namespace {namespace}')
+        return True, None
+    except kube_client.api_exception() as e:
+        return False, str(e)
+
+
+def get_secret_kind(secret: kubernetes.client.V1Secret) -> Optional[str]:
+    """Get the konduktor-specific kind of a secret, if labeled."""
+    if secret.metadata.labels:
+        return secret.metadata.labels.get('konduktor/secret-kind')
+    return None
+
+
 def get_autoscaler_type() -> Optional[kubernetes_enums.KubernetesAutoscalerType]:
     """Returns the autoscaler type by reading from config"""
     autoscaler_type = config.get_nested(('kubernetes', 'autoscaler'), None)
@@ -661,3 +722,10 @@ def is_label_valid(label_key: str, label_value: str) -> Tuple[bool, Optional[str
     if not key_valid or not value_valid:
         return False, error_msg
     return True, None
+
+
+def is_k8s_resource_name_valid(name):
+    """Returns whether or not a k8s name is valid (must consist of
+    lower case alphanumeric characters or -, and must start and end
+    with alphanumeric characters)"""
+    return re.match(DNS_SUBDOMAIN_REGEX, name)

{konduktor_nightly-0.1.0.dev20250530104807.dist-info → konduktor_nightly-0.1.0.dev20250531104602.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: konduktor-nightly
-Version: 0.1.0.dev20250530104807
+Version: 0.1.0.dev20250531104602
 Summary: GPU Cluster Health Management
 Author: Andrew Aikawa
 Author-email: asai@berkeley.edu

{konduktor_nightly-0.1.0.dev20250530104807.dist-info → konduktor_nightly-0.1.0.dev20250531104602.dist-info}/RECORD

@@ -1,4 +1,4 @@
-konduktor/__init__.py,sha256=xvEeJ9mWysi91Zif_yvb9nLV-KoVK8jtnTCnsswcSsA,1540
+konduktor/__init__.py,sha256=J0xYwCSHyRmCY7OMSztuG9RmIeNVp0_DIGVQd6MZgMU,1540
 konduktor/adaptors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 konduktor/adaptors/aws.py,sha256=s47Ra-GaqCQibzVfmD0pmwEWHif1EGO5opMbwkLxTCU,8244
 konduktor/adaptors/common.py,sha256=ZIqzjx77PIHUwpjfAQ1uX8B2aX78YMuGj4Bppd-MdyM,4183
@@ -7,9 +7,9 @@ konduktor/authentication.py,sha256=_mVy3eqoKohicHostFiGwG1-2ybxP-l7ouofQ0LRlCY,4
 konduktor/backends/__init__.py,sha256=1Q6sqqdeMYarpTX_U-QVywJYf7idiUTRsyP-E4BQSOw,129
 konduktor/backends/backend.py,sha256=qh0bp94lzoTYZkzyQv2-CVrB5l91FkG2vclXg24UFC0,2910
 konduktor/backends/jobset.py,sha256=UdhwAuZODLMbLY51Y2zOBsh6wg4Pb84oHVvUKzx3Z2w,8434
-konduktor/backends/jobset_utils.py,sha256=4vMYOhTENfBL9khzFuj69-Vy4g0sBkUpXX-1bfPnVys,20054
+konduktor/backends/jobset_utils.py,sha256=diGpy-qpsQeVMFZVQsMgG3HxJxl2huxqbRU6FMA0QHY,21363
 konduktor/check.py,sha256=JennyWoaqSKhdyfUldd266KwVXTPJpcYQa4EED4a_BA,7569
-konduktor/cli.py,sha256=Fl1dwNB5T-kDQAlAoOJetzl6RYt9FYUlowKjbNhVjkQ,23412
+konduktor/cli.py,sha256=qiTFut28crvcXOoSvnN3NcTb-xPmtFB336zdt9Q2bxU,33370
 konduktor/config.py,sha256=J50JxC6MsXMnlrJPXdDUMr38C89xvOO7mR8KJ6fyils,15520
 konduktor/constants.py,sha256=T3AeXXxuQHINW_bAWyztvDeS8r4g8kXBGIwIq13cys0,1814
 konduktor/controller/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -51,12 +51,12 @@ konduktor/dashboard/frontend/server.js,sha256=jcp6_Ww9YJD3uKY07jR3KMlAM6n1QZdxZn
 konduktor/dashboard/frontend/tailwind.config.js,sha256=fCnc48wvioIDOe5ldQ_6RE7F76cP7aU7pDrxBPJx-Fk,366
 konduktor/data/__init__.py,sha256=KMR2i3E9YcIpiIuCxtRdS7BQ1w2vUAbbve7agziJrLo,213
 konduktor/data/aws/__init__.py,sha256=_6zWfNNAK1QGgyKqg_yPYWcXlnffchyvIMErYa6tw_U,331
-konduktor/data/aws/s3.py,sha256=T4FnCxilNp35bsgmE7j5O3j15FVbgWRdUH8YFXCiwSw,48335
+konduktor/data/aws/s3.py,sha256=lNgI02wacyXudyrIfXPKrscH4o153Wa_o5qpQR-jjLQ,48506
 konduktor/data/constants.py,sha256=yXVEoTI2we1xOjVSU-bjRCQCLpVvpEvJ0GedXvSwEfw,127
 konduktor/data/data_utils.py,sha256=IG1jgb_La997wi90xCvxYYsHQRlmm8Aooq04ZSf8EDI,9670
 konduktor/data/gcp/__init__.py,sha256=rlQxACBC_Vu36mdgPyJgUy4mGc_6Nt_a96JAuaPz2pQ,489
 konduktor/data/gcp/constants.py,sha256=dMfOiFccM8O6rUi9kClJcbvw1K1VnS1JzzQk3apq8ho,1483
-konduktor/data/gcp/gcs.py,sha256=nqhCvQuGpHFPoxT5SKgxL25KtZuSg377Nh1bICiQwlc,42057
+konduktor/data/gcp/gcs.py,sha256=Zc1LXrjoeNU9EDK229evrKxjVqsKIUicbtYlugA_TiY,42229
 konduktor/data/gcp/utils.py,sha256=FJQcMXZqtMIzjZ98b3lTTc0UbdPUKTDLsOsfJaaH5-s,214
 konduktor/data/registry.py,sha256=CUbMsN_Q17Pf4wRHkqZrycErEjTP7cLEdgcfwVGcEpc,696
 konduktor/data/storage.py,sha256=o2So-bY9glvgbGdoN7AQNYmNnvGf1AUDPpImtadRL90,35213
@@ -71,19 +71,19 @@ konduktor/manifests/pod_cleanup_controller.yaml,sha256=hziL1Ka1kCAEL9R7Tjvpb80iw
 konduktor/resource.py,sha256=w2PdIrmQaJWA-GLSmVBcg4lxwuxvPulz35_YSKa5o24,19254
 konduktor/task.py,sha256=ofwd8WIhfD6C3ThLcv6X3GUzQHyZ6ddjUagE-umF4K0,35207
 konduktor/templates/jobset.yaml.j2,sha256=onYiHtXAgk-XBtji994hPu_g0hxnLzvmfxwjbdKdeZc,960
-konduktor/templates/pod.yaml.j2,sha256=XPG87LYieHlMxJSDVlWlU2c1eroKYYP5m3bz3daJDgY,15508
+konduktor/templates/pod.yaml.j2,sha256=JvDruGpBbRHSklqNEeKSvPH0Y1uldvcylqLUaIcguuQ,16086
 konduktor/usage/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 konduktor/usage/constants.py,sha256=gCL8afIHZhO0dcxbJGpESE9sCC1cBSbeRnQ8GwNOY4M,612
 konduktor/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 konduktor/utils/accelerator_registry.py,sha256=1tpVIaM0UZ3w1desPhVEwNCUruamhP-igDZrcfaoRWI,574
 konduktor/utils/annotations.py,sha256=oy2-BLydkFt3KWkXDuaGY84d6b7iISuy4eAT9uXk0Fc,2225
 konduktor/utils/base64_utils.py,sha256=mF-Tw98mFRG70YE4w6s9feuQSCYZHOb8YatBZwMugyI,3130
-konduktor/utils/common_utils.py,sha256=F5x7k4AdBB44u8PYRkaugORnZKnK3JLqGn1jHOKgUYo,14960
+konduktor/utils/common_utils.py,sha256=4yG5Kjvu1hu6x2nKNaaCUKQNrheUaG61Qe913MFPry8,15060
 konduktor/utils/constants.py,sha256=1DneiTR21lvKUcWdBGwC4I4fD4uPjbjLUilEnJS7rzA,216
 konduktor/utils/env_options.py,sha256=T41Slzf4Mzl-n45CGXXqdy2fCrYhPNZQ7RP5vmnN4xc,2258
 konduktor/utils/exceptions.py,sha256=5IFnN5bIUSBJv4KRRrCepk5jyY9EG5vWWQqbjCmP3NU,6682
 konduktor/utils/kubernetes_enums.py,sha256=SabUueF6Bpzbpa57gyH5VB65xla2N9l8CZmAeYTfGmM,176
-konduktor/utils/kubernetes_utils.py,sha256=1MZHwU4vy-exA4TA5_oTiV-zm1A2ayfeA0T_75DMFM8,23937
+konduktor/utils/kubernetes_utils.py,sha256=VG7qatUFyWHY-PCQ8fYWh2kn2TMwfg84cn-VkXdCwI8,26077
 konduktor/utils/log_utils.py,sha256=oFCKkYKCS_e_GRw_-0F7WsiIZNqJL1RZ4cD5-zh59Q4,9765
 konduktor/utils/loki_utils.py,sha256=h2ZvZQr1nE_wXXsKsGMjhG2s2MXknNd4icydTR_ruKU,3539
 konduktor/utils/rich_utils.py,sha256=ycADW6Ij3wX3uT8ou7T8qxX519RxlkJivsLvUahQaJo,3583
@@ -91,8 +91,8 @@ konduktor/utils/schemas.py,sha256=2fHsTi3t9q3LXqOPrcpkmPsMbaoJBnuJstd6ULmDiUo,16
 konduktor/utils/subprocess_utils.py,sha256=WoFkoFhGecPR8-rF8WJxbIe-YtV94LXz9UG64SDhCY4,9448
 konduktor/utils/ux_utils.py,sha256=czCwiS1bDqgeKtzAJctczpLwFZzAse7WuozdvzEFYJ4,7437
 konduktor/utils/validator.py,sha256=tgBghVyedyzGx84-U2Qfoh_cJBE3oUk9gclMW90ORks,691
-konduktor_nightly-0.1.0.dev20250530104807.dist-info/LICENSE,sha256=MuuqTZbHvmqXR_aNKAXzggdV45ANd3wQ5YI7tnpZhm0,6586
-konduktor_nightly-0.1.0.dev20250530104807.dist-info/METADATA,sha256=rJvh5iMrpeeqvyDT7ZfGFVmgNCBzg8Aczb2f4W4LQXA,4289
-konduktor_nightly-0.1.0.dev20250530104807.dist-info/WHEEL,sha256=XbeZDeTWKc1w7CSIyre5aMDU_-PohRwTQceYnisIYYY,88
-konduktor_nightly-0.1.0.dev20250530104807.dist-info/entry_points.txt,sha256=k3nG5wDFIJhNqsZWrHk4d0irIB2Ns9s47cjRWYsTCT8,48
-konduktor_nightly-0.1.0.dev20250530104807.dist-info/RECORD,,
+konduktor_nightly-0.1.0.dev20250531104602.dist-info/LICENSE,sha256=MuuqTZbHvmqXR_aNKAXzggdV45ANd3wQ5YI7tnpZhm0,6586
+konduktor_nightly-0.1.0.dev20250531104602.dist-info/METADATA,sha256=cN-qh6GPr5-9ZKsVP52lvNC6HqaLK-cW1XlWbiG9TpQ,4289
+konduktor_nightly-0.1.0.dev20250531104602.dist-info/WHEEL,sha256=XbeZDeTWKc1w7CSIyre5aMDU_-PohRwTQceYnisIYYY,88
+konduktor_nightly-0.1.0.dev20250531104602.dist-info/entry_points.txt,sha256=k3nG5wDFIJhNqsZWrHk4d0irIB2Ns9s47cjRWYsTCT8,48
+konduktor_nightly-0.1.0.dev20250531104602.dist-info/RECORD,,