konduktor-nightly 0.1.0.dev20250430104745__py3-none-any.whl → 0.1.0.dev20250502104736__py3-none-any.whl

konduktor/__init__.py

@@ -14,7 +14,7 @@ __all__ = [
 ]
 
 # Replaced with the current commit when building the wheels.
-_KONDUKTOR_COMMIT_SHA = 'd1d19dd0b1d1e1440aad10115f235e2b6ea95dd7'
+_KONDUKTOR_COMMIT_SHA = '8035a53e2bd3595665be51149e8287dc721d5edb'
 os.makedirs(os.path.expanduser('~/.konduktor'), exist_ok=True)
 
 
@@ -48,5 +48,5 @@ def _get_git_commit():
 
 
 __commit__ = _get_git_commit()
-__version__ = '1.0.0.dev0.1.0.dev20250430104745'
+__version__ = '1.0.0.dev0.1.0.dev20250502104736'
 __root_dir__ = os.path.dirname(os.path.abspath(__file__))

konduktor/backends/jobset_utils.py

@@ -15,9 +15,15 @@ if typing.TYPE_CHECKING:
     from datetime import timedelta
 
 import konduktor
-from konduktor import constants, kube_client, logging
+from konduktor import config, constants, kube_client, logging
 from konduktor.data import registry
-from konduktor.utils import common_utils, kubernetes_utils, log_utils
+from konduktor.utils import (
+    common_utils,
+    exceptions,
+    kubernetes_utils,
+    log_utils,
+    ux_utils,
+)
 
 if typing.TYPE_CHECKING:
     pass
@@ -42,7 +48,7 @@ _JOBSET_METADATA_LABELS = {
     'jobset_accelerator_label': JOBSET_ACCELERATOR_LABEL,
     'jobset_num_accelerators_label': JOBSET_NUM_ACCELERATORS_LABEL,
 }
-_RUN_DURATION_ANNOTATION = 'maxRunDurationSeconds'
+
 _RUN_DURATION_ANNOTATION_KEY = 'kueue.x-k8s.io/maxRunDurationSeconds'
 
 
@@ -130,6 +136,20 @@ def create_pod_spec(task: 'konduktor.Task') -> Dict[str, Any]:
         storage_secrets[store_scheme] = cloud_store._STORE.get_k8s_credential_name()
         sync_commands.append(cloud_store.make_sync_file_command(src, dst))
 
+    tailscale_secret = config.get_nested(('tailscale', 'secret_name'), None)
+    if tailscale_secret:
+        context = kubernetes_utils.get_current_kube_config_context_name()
+        namespace = kubernetes_utils.get_kube_config_context_namespace(context)
+        secret_exist, err = kubernetes_utils.check_secret_exists(
+            tailscale_secret, namespace, context
+        )
+        if not secret_exist:
+            with ux_utils.print_exception_no_traceback():
+                raise exceptions.MissingSecretError(
+                    f'No tailscale auth-key secret `{tailscale_secret}` found even '
+                    f'though specified by `tailscale.secret_name`: {err}'
+                )
+
     assert task.resources is not None, 'Task resources are required'
     assert task.resources.cpus is not None, 'Task resources cpus are required'
     assert task.resources.memory is not None, 'Task resources memory are required'
@@ -154,6 +174,8 @@ def create_pod_spec(task: 'konduktor.Task') -> Dict[str, Any]:
                 'mount_secrets': storage_secrets,
                 'remote_workdir': constants.KONDUKTOR_REMOTE_WORKDIR,
                 'user': common_utils.get_cleaned_username(),
+                # Tailscale credentials
+                'tailscale_secret': tailscale_secret,
             },
             temp.name,
         )

konduktor/templates/pod.yaml.j2

@@ -16,8 +16,7 @@ kubernetes:
         operator: "Exists"
       {% endif %}
       containers:
-        # TODO(asaiacai): should decide here whether we add the fabric interfaces/containers init etc.
-        # TODO(asaiacai): add ulimits
+        # TODO(asaiacai): should decide here whether we add the fabric interfaces/containers init etc.        
         - name: konduktor-container
           image: {{ image_id }}
           # this is set during jobset definition since we need to know the jobset
@@ -39,6 +38,24 @@ kubernetes:
             value: "{{ num_nodes }}"
           - name: NUM_GPUS_PER_NODE
             value: "{{ num_gpus }}"
+          {% if tailscale_secret %}
+          - name: TS_USERSPACE
+            value: "true"
+          - name: TS_AUTHKEY
+            valueFrom:
+              secretKeyRef:
+                name: {{ tailscale_secret }}
+                key: TS_AUTHKEY
+                optional: true
+          - name: POD_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.name
+          - name: POD_UID
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.uid
+          {% endif %}
           # these are for compatibility with skypilot
           - name: SKYPILOT_NODE_IPS
             value: "{{ node_hostnames }}"
@@ -58,7 +75,6 @@ kubernetes:
             mountPath: /run/konduktor/{{ secret_type }}-secret
           {% endfor %}
           command: ["bash", "-c"]
-          # TODO(asaiacai): should we just mount this as a configmap instead? - Edit: probably not
           args:
             - |
               # TODO(asaiacai): add debug environment variable for printing the apt-update, apt-install, sync-files output
@@ -76,7 +92,7 @@ kubernetes:
               {% if 'rsync' in run_cmd %}
               PACKAGES="$PACKAGES rsync";
               {% endif %}
-              {% if 'curl' in run_cmd %}
+              {% if 'curl' in run_cmd or tailscale_secret %}
               PACKAGES="$PACKAGES curl";
               {% endif %}
               {% if 'gs' in mount_secrets or 's3' in mount_secrets %}
@@ -117,6 +133,17 @@ kubernetes:
                 DEBIAN_FRONTEND=noninteractive $(prefix_cmd) apt-get install -y $MISSING_PACKAGES >> ~/.konduktor/tmp/apt-install.log;
               fi;
               end_epoch=$(date +%s);
+
+              {% if tailscale_secret %}
+              if ! command -v tailscale >/dev/null 2>&1; then
+                export TS_HOSTNAME=$(echo "$POD_NAME" | sed 's/-[^-]*$//')
+                $(prefix_cmd) curl -fsSL https://tailscale.com/install.sh | DEBIAN_FRONTEND=noninteractive $(prefix_cmd) sh >> ~/.konduktor/tmp/tailscale-install.log
+                $(prefix_cmd) tailscaled --tun=userspace-networking >/dev/null 2>&1 &
+                $(prefix_cmd) tailscale up --auth-key=${TS_AUTHKEY} --ssh --hostname=${TS_HOSTNAME} >/dev/null 2>&1
+              fi
+              {% endif %}
+              end_epoch=$(date +%s);
+
               $(prefix_cmd) echo "===== KONDUKTOR: Installing packages took $((end_epoch - start_epoch)) seconds ====="
 
               # unpack secrets credentials
@@ -153,6 +180,7 @@ kubernetes:
               {% endif %}
               end_epoch=$(date +%s);
               end_setup_time=$((end_epoch - start_setup));
+              ulimit -Sc 0 && ulimit -Hc 0
               $(prefix_cmd) echo "===== KONDUKTOR: Initialization took $end_setup_time seconds ====="
               # run task  
               $(prefix_cmd) cd {{ remote_workdir }}
@@ -180,7 +208,8 @@ kubernetes:
               {% endif %}
           securityContext:
             capabilities:
-              add: ["IPC_LOCK"]  # May be needed for memlock
+              add: 
+              - "IPC_LOCK"  # May be needed for memlock
 
       volumes:
       - name: shared-memory

konduktor/utils/exceptions.py

@@ -130,6 +130,10 @@ class CommandError(Exception):
     pass
 
 
+class MissingSecretError(Exception):
+    pass
+
+
 class NotSupportedError(Exception):
     """Raised when a feature is not supported."""
 

konduktor/utils/schemas.py

@@ -563,6 +563,17 @@ def get_config_schema():
         },
     }
 
+    tailscale_configs = {
+        'type': 'object',
+        'required': [],
+        'additionalProperties': False,
+        'properties': {
+            'secret_name': {
+                'type': 'string',
+            },
+        },
+    }
+
     for cloud, config in cloud_configs.items():
         if cloud == 'kubernetes':
             config['properties'].update(_REMOTE_IDENTITY_SCHEMA_KUBERNETES)
@@ -577,6 +588,7 @@ def get_config_schema():
             'admin_policy': admin_policy_schema,
             'nvidia_gpus': gpu_configs,
             'allowed_clouds': allowed_clouds,
+            'tailscale': tailscale_configs,
             **cloud_configs,
         },
     }

konduktor/utils/ux_utils.py

@@ -11,6 +11,7 @@ from typing import Callable, Optional, Union
 import colorama
 import rich.console as rich_console
 
+from konduktor import config
 from konduktor import logging as konduktor_logging
 
 if typing.TYPE_CHECKING:
@@ -211,6 +212,11 @@ def command_hint_messages(hint_type: CommandHintType, job_id: str) -> str:
         )
         hint_str = '\n📋 Useful Commands'
         hint_str += f'{job_hint_str}'
+        if config.get_nested(('tailscale', 'secret_name'), None) is not None:
+            hint_str += (
+                f'\n{INDENT_SYMBOL}To tailscale ssh:\t\t'
+                f'{BOLD}ssh root@{job_id}-workers-0-0 {RESET_BOLD}'
+            )
         return hint_str
     else:
         raise ValueError(f'Invalid hint type: {hint_type}')

{konduktor_nightly-0.1.0.dev20250430104745.dist-info → konduktor_nightly-0.1.0.dev20250502104736.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: konduktor-nightly
-Version: 0.1.0.dev20250430104745
+Version: 0.1.0.dev20250502104736
 Summary: GPU Cluster Health Management
 Author: Andrew Aikawa
 Author-email: asai@berkeley.edu

{konduktor_nightly-0.1.0.dev20250430104745.dist-info → konduktor_nightly-0.1.0.dev20250502104736.dist-info}/RECORD

@@ -1,4 +1,4 @@
-konduktor/__init__.py,sha256=Fur_FhjkiRaTC8mYuUo1rf0_RXIiVRM9U9oUYdDL5m8,1540
+konduktor/__init__.py,sha256=jMTLcTV0GSBv8eb57oCZaR9dTjrcnVas0dkuTq_7HAg,1540
 konduktor/adaptors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 konduktor/adaptors/aws.py,sha256=s47Ra-GaqCQibzVfmD0pmwEWHif1EGO5opMbwkLxTCU,8244
 konduktor/adaptors/common.py,sha256=uTdpKvgBSwYMmynx9wR5kiZQyTrdaw9ZI4KH6Z2E5Hw,4296
@@ -6,7 +6,7 @@ konduktor/adaptors/gcp.py,sha256=ierTF4z7vwpJ9BsC7LSiwv4uLcjGXscwZOwQrddr2vM,410
 konduktor/backends/__init__.py,sha256=1Q6sqqdeMYarpTX_U-QVywJYf7idiUTRsyP-E4BQSOw,129
 konduktor/backends/backend.py,sha256=qh0bp94lzoTYZkzyQv2-CVrB5l91FkG2vclXg24UFC0,2910
 konduktor/backends/jobset.py,sha256=veptYGXtk-ugWxBsBV5SnqI4rGKOlGfm_N3wApvNhSQ,8326
-konduktor/backends/jobset_utils.py,sha256=UJkDu6Y8u4N2AaNSJTOSgbGLyY25bzaP-I6esJ11jms,17578
+konduktor/backends/jobset_utils.py,sha256=B0N0sx-pWF9_CDeuSXAU4nm3ZIwroyVcq6aUAlNZZRs,18376
 konduktor/check.py,sha256=JennyWoaqSKhdyfUldd266KwVXTPJpcYQa4EED4a_BA,7569
 konduktor/cli.py,sha256=Ii9-2mrc-1f2ksLasA-xRb-JnEi_9ZeCXZ3lJ1GG8H8,23515
 konduktor/config.py,sha256=J50JxC6MsXMnlrJPXdDUMr38C89xvOO7mR8KJ6fyils,15520
@@ -70,7 +70,7 @@ konduktor/manifests/pod_cleanup_controller.yaml,sha256=hziL1Ka1kCAEL9R7Tjvpb80iw
 konduktor/resource.py,sha256=68z8gC8Ivqktwv0R6ylMn9ZNocgkcRT0yIRGGKOdwcM,18491
 konduktor/task.py,sha256=Vu1TzYtLvSBz-HyHY2gsM2cMcUhMNQu44L3CWmYRXKE,35232
 konduktor/templates/jobset.yaml.j2,sha256=onYiHtXAgk-XBtji994hPu_g0hxnLzvmfxwjbdKdeZc,960
-konduktor/templates/pod.yaml.j2,sha256=q_kFyuYP-RlHocQUV8Vsnh-hYurWO7Y_pwl0gytGMl4,8728
+konduktor/templates/pod.yaml.j2,sha256=s3eECjLevUWR-zvyeI8WjQWxQYJh_AMk1tdQVGNXpEM,9835
 konduktor/usage/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 konduktor/usage/constants.py,sha256=gCL8afIHZhO0dcxbJGpESE9sCC1cBSbeRnQ8GwNOY4M,612
 konduktor/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -80,18 +80,18 @@ konduktor/utils/base64_utils.py,sha256=mF-Tw98mFRG70YE4w6s9feuQSCYZHOb8YatBZwMug
 konduktor/utils/common_utils.py,sha256=F5x7k4AdBB44u8PYRkaugORnZKnK3JLqGn1jHOKgUYo,14960
 konduktor/utils/constants.py,sha256=1DneiTR21lvKUcWdBGwC4I4fD4uPjbjLUilEnJS7rzA,216
 konduktor/utils/env_options.py,sha256=T41Slzf4Mzl-n45CGXXqdy2fCrYhPNZQ7RP5vmnN4xc,2258
-konduktor/utils/exceptions.py,sha256=GBOFIkk9nikqWGR0FXGXOWVVImoH7nWnMl_L3Oux3fo,6581
+konduktor/utils/exceptions.py,sha256=IHyaP5ERZpPvWZeKWV3MVTyKsxo2Fq-13nhI0PRNQzk,6629
 konduktor/utils/kubernetes_enums.py,sha256=SabUueF6Bpzbpa57gyH5VB65xla2N9l8CZmAeYTfGmM,176
 konduktor/utils/kubernetes_utils.py,sha256=ivFVh90Gez19_JD5U4bgCO5zNtQUflF0hJsM5nZLj8A,23864
 konduktor/utils/log_utils.py,sha256=lgHCq4OdtJNfbpso-uYGONUCVNsUrUkUWjROarsHt6s,9897
 konduktor/utils/loki_utils.py,sha256=ND1pbbbFhLhLKw3870j44LpR_9MB0EkDJSs5K7nWdY4,3473
 konduktor/utils/rich_utils.py,sha256=kdjNe6S2LlpOxyzhFHqMzCz7g4ROC4e7TPWgcbRsrQE,3577
-konduktor/utils/schemas.py,sha256=Gv7SEhFpv-eO5izqRz8d-eQ9z-lVmY05akm6HEXIIdc,17478
+konduktor/utils/schemas.py,sha256=_VCWnsSgyP3u5cpACEmJeuqcy5mzu_fr0McHyZdiXd8,17757
 konduktor/utils/subprocess_utils.py,sha256=WoFkoFhGecPR8-rF8WJxbIe-YtV94LXz9UG64SDhCY4,9448
-konduktor/utils/ux_utils.py,sha256=NPNu3Igu2Z9Oq77ghJhy_fIxQZTXWr9BtKyxN3Wslzo,7164
+konduktor/utils/ux_utils.py,sha256=czCwiS1bDqgeKtzAJctczpLwFZzAse7WuozdvzEFYJ4,7437
 konduktor/utils/validator.py,sha256=tgBghVyedyzGx84-U2Qfoh_cJBE3oUk9gclMW90ORks,691
-konduktor_nightly-0.1.0.dev20250430104745.dist-info/LICENSE,sha256=MuuqTZbHvmqXR_aNKAXzggdV45ANd3wQ5YI7tnpZhm0,6586
-konduktor_nightly-0.1.0.dev20250430104745.dist-info/METADATA,sha256=A9ws8bq2H4K0J6wyXSPrZWjkbMGgE0cGHDJA4R3W7To,4366
-konduktor_nightly-0.1.0.dev20250430104745.dist-info/WHEEL,sha256=XbeZDeTWKc1w7CSIyre5aMDU_-PohRwTQceYnisIYYY,88
-konduktor_nightly-0.1.0.dev20250430104745.dist-info/entry_points.txt,sha256=k3nG5wDFIJhNqsZWrHk4d0irIB2Ns9s47cjRWYsTCT8,48
-konduktor_nightly-0.1.0.dev20250430104745.dist-info/RECORD,,
+konduktor_nightly-0.1.0.dev20250502104736.dist-info/LICENSE,sha256=MuuqTZbHvmqXR_aNKAXzggdV45ANd3wQ5YI7tnpZhm0,6586
+konduktor_nightly-0.1.0.dev20250502104736.dist-info/METADATA,sha256=smj8TlY9k4hFEtZNDLlnl5W-b8UjtFepi-iTqGiuI9o,4366
+konduktor_nightly-0.1.0.dev20250502104736.dist-info/WHEEL,sha256=XbeZDeTWKc1w7CSIyre5aMDU_-PohRwTQceYnisIYYY,88
+konduktor_nightly-0.1.0.dev20250502104736.dist-info/entry_points.txt,sha256=k3nG5wDFIJhNqsZWrHk4d0irIB2Ns9s47cjRWYsTCT8,48
+konduktor_nightly-0.1.0.dev20250502104736.dist-info/RECORD,,