konduktor-nightly 0.1.0.dev20250430104745__tar.gz → 0.1.0.dev20250501104750__tar.gz

{konduktor_nightly-0.1.0.dev20250430104745 → konduktor_nightly-0.1.0.dev20250501104750}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: konduktor-nightly
-Version: 0.1.0.dev20250430104745
+Version: 0.1.0.dev20250501104750
 Summary: GPU Cluster Health Management
 Author: Andrew Aikawa
 Author-email: asai@berkeley.edu

{konduktor_nightly-0.1.0.dev20250430104745 → konduktor_nightly-0.1.0.dev20250501104750}/konduktor/__init__.py

@@ -14,7 +14,7 @@ __all__ = [
 ]
 
 # Replaced with the current commit when building the wheels.
-_KONDUKTOR_COMMIT_SHA = 'd1d19dd0b1d1e1440aad10115f235e2b6ea95dd7'
+_KONDUKTOR_COMMIT_SHA = 'abd70e24a6d56a34a471a73cbc8ac970f33717c0'
 os.makedirs(os.path.expanduser('~/.konduktor'), exist_ok=True)
 
 
@@ -48,5 +48,5 @@ def _get_git_commit():
 
 
 __commit__ = _get_git_commit()
-__version__ = '1.0.0.dev0.1.0.dev20250430104745'
+__version__ = '1.0.0.dev0.1.0.dev20250501104750'
 __root_dir__ = os.path.dirname(os.path.abspath(__file__))

{konduktor_nightly-0.1.0.dev20250430104745 → konduktor_nightly-0.1.0.dev20250501104750}/konduktor/backends/jobset_utils.py

@@ -15,9 +15,15 @@ if typing.TYPE_CHECKING:
     from datetime import timedelta
 
 import konduktor
-from konduktor import constants, kube_client, logging
+from konduktor import config, constants, kube_client, logging
 from konduktor.data import registry
-from konduktor.utils import common_utils, kubernetes_utils, log_utils
+from konduktor.utils import (
+    common_utils,
+    exceptions,
+    kubernetes_utils,
+    log_utils,
+    ux_utils,
+)
 
 if typing.TYPE_CHECKING:
     pass
@@ -42,7 +48,7 @@ _JOBSET_METADATA_LABELS = {
     'jobset_accelerator_label': JOBSET_ACCELERATOR_LABEL,
     'jobset_num_accelerators_label': JOBSET_NUM_ACCELERATORS_LABEL,
 }
-_RUN_DURATION_ANNOTATION = 'maxRunDurationSeconds'
+
 _RUN_DURATION_ANNOTATION_KEY = 'kueue.x-k8s.io/maxRunDurationSeconds'
 
 
@@ -130,6 +136,20 @@ def create_pod_spec(task: 'konduktor.Task') -> Dict[str, Any]:
         storage_secrets[store_scheme] = cloud_store._STORE.get_k8s_credential_name()
         sync_commands.append(cloud_store.make_sync_file_command(src, dst))
 
+    tailscale_secret = config.get_nested(('tailscale', 'secret_name'), None)
+    if tailscale_secret:
+        context = kubernetes_utils.get_current_kube_config_context_name()
+        namespace = kubernetes_utils.get_kube_config_context_namespace(context)
+        secret_exist, err = kubernetes_utils.check_secret_exists(
+            tailscale_secret, namespace, context
+        )
+        if not secret_exist:
+            with ux_utils.print_exception_no_traceback():
+                raise exceptions.MissingSecretError(
+                    f'No tailscale auth-key secret `{tailscale_secret}` found even '
+                    f'though specified by `tailscale.secret_name`: {err}'
+                )
+
     assert task.resources is not None, 'Task resources are required'
     assert task.resources.cpus is not None, 'Task resources cpus are required'
     assert task.resources.memory is not None, 'Task resources memory are required'
@@ -154,6 +174,8 @@ def create_pod_spec(task: 'konduktor.Task') -> Dict[str, Any]:
                 'mount_secrets': storage_secrets,
                 'remote_workdir': constants.KONDUKTOR_REMOTE_WORKDIR,
                 'user': common_utils.get_cleaned_username(),
+                # Tailscale credentials
+                'tailscale_secret': tailscale_secret,
             },
             temp.name,
         )

{konduktor_nightly-0.1.0.dev20250430104745 → konduktor_nightly-0.1.0.dev20250501104750}/konduktor/templates/pod.yaml.j2

@@ -16,8 +16,7 @@ kubernetes:
         operator: "Exists"
       {% endif %}
       containers:
-        # TODO(asaiacai): should decide here whether we add the fabric interfaces/containers init etc.
-        # TODO(asaiacai): add ulimits
+        # TODO(asaiacai): should decide here whether we add the fabric interfaces/containers init etc.        
         - name: konduktor-container
           image: {{ image_id }}
           # this is set during jobset definition since we need to know the jobset
@@ -39,6 +38,24 @@ kubernetes:
             value: "{{ num_nodes }}"
           - name: NUM_GPUS_PER_NODE
             value: "{{ num_gpus }}"
+          {% if tailscale_secret %}
+          - name: TS_USERSPACE
+            value: "true"
+          - name: TS_AUTHKEY
+            valueFrom:
+              secretKeyRef:
+                name: {{ tailscale_secret }}
+                key: TS_AUTHKEY
+                optional: true
+          - name: POD_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.name
+          - name: POD_UID
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.uid
+          {% endif %}
           # these are for compatibility with skypilot
           - name: SKYPILOT_NODE_IPS
             value: "{{ node_hostnames }}"
@@ -58,7 +75,6 @@ kubernetes:
             mountPath: /run/konduktor/{{ secret_type }}-secret
           {% endfor %}
           command: ["bash", "-c"]
-          # TODO(asaiacai): should we just mount this as a configmap instead? - Edit: probably not
           args:
             - |
               # TODO(asaiacai): add debug environment variable for printing the apt-update, apt-install, sync-files output
@@ -76,7 +92,7 @@ kubernetes:
               {% if 'rsync' in run_cmd %}
               PACKAGES="$PACKAGES rsync";
               {% endif %}
-              {% if 'curl' in run_cmd %}
+              {% if 'curl' in run_cmd or tailscale_secret %}
               PACKAGES="$PACKAGES curl";
               {% endif %}
               {% if 'gs' in mount_secrets or 's3' in mount_secrets %}
@@ -117,6 +133,17 @@ kubernetes:
                 DEBIAN_FRONTEND=noninteractive $(prefix_cmd) apt-get install -y $MISSING_PACKAGES >> ~/.konduktor/tmp/apt-install.log;
               fi;
               end_epoch=$(date +%s);
+
+              {% if tailscale_secret %}
+              if ! command -v tailscale >/dev/null 2>&1; then
+                export TS_HOSTNAME=$(echo "$POD_NAME" | sed 's/-[^-]*$//')
+                $(prefix_cmd) curl -fsSL https://tailscale.com/install.sh | DEBIAN_FRONTEND=noninteractive $(prefix_cmd) sh >> ~/.konduktor/tmp/tailscale-install.log
+                $(prefix_cmd) tailscaled --tun=userspace-networking >/dev/null 2>&1 &
+                $(prefix_cmd) tailscale up --auth-key=${TS_AUTHKEY} --ssh --hostname=${TS_HOSTNAME} >/dev/null 2>&1
+              fi
+              {% endif %}
+              end_epoch=$(date +%s);
+
               $(prefix_cmd) echo "===== KONDUKTOR: Installing packages took $((end_epoch - start_epoch)) seconds ====="
 
               # unpack secrets credentials
@@ -153,6 +180,7 @@ kubernetes:
               {% endif %}
               end_epoch=$(date +%s);
               end_setup_time=$((end_epoch - start_setup));
+              ulimit -Sc 0 && ulimit -Hc 0
               $(prefix_cmd) echo "===== KONDUKTOR: Initialization took $end_setup_time seconds ====="
               # run task  
               $(prefix_cmd) cd {{ remote_workdir }}
@@ -180,7 +208,8 @@ kubernetes:
               {% endif %}
           securityContext:
             capabilities:
-              add: ["IPC_LOCK"]  # May be needed for memlock
+              add: 
+              - "IPC_LOCK"  # May be needed for memlock
 
       volumes:
       - name: shared-memory

{konduktor_nightly-0.1.0.dev20250430104745 → konduktor_nightly-0.1.0.dev20250501104750}/konduktor/utils/exceptions.py

@@ -130,6 +130,10 @@ class CommandError(Exception):
     pass
 
 
+class MissingSecretError(Exception):
+    pass
+
+
 class NotSupportedError(Exception):
     """Raised when a feature is not supported."""
 

{konduktor_nightly-0.1.0.dev20250430104745 → konduktor_nightly-0.1.0.dev20250501104750}/konduktor/utils/schemas.py

@@ -563,6 +563,17 @@ def get_config_schema():
         },
     }
 
+    tailscale_configs = {
+        'type': 'object',
+        'required': [],
+        'additionalProperties': False,
+        'properties': {
+            'secret_name': {
+                'type': 'string',
+            },
+        },
+    }
+
     for cloud, config in cloud_configs.items():
         if cloud == 'kubernetes':
             config['properties'].update(_REMOTE_IDENTITY_SCHEMA_KUBERNETES)
@@ -577,6 +588,7 @@ def get_config_schema():
             'admin_policy': admin_policy_schema,
             'nvidia_gpus': gpu_configs,
             'allowed_clouds': allowed_clouds,
+            'tailscale': tailscale_configs,
             **cloud_configs,
         },
     }

{konduktor_nightly-0.1.0.dev20250430104745 → konduktor_nightly-0.1.0.dev20250501104750}/konduktor/utils/ux_utils.py

@@ -11,6 +11,7 @@ from typing import Callable, Optional, Union
 import colorama
 import rich.console as rich_console
 
+from konduktor import config
 from konduktor import logging as konduktor_logging
 
 if typing.TYPE_CHECKING:
@@ -211,6 +212,11 @@ def command_hint_messages(hint_type: CommandHintType, job_id: str) -> str:
         )
         hint_str = '\n📋 Useful Commands'
         hint_str += f'{job_hint_str}'
+        if config.get_nested(('tailscale', 'secret_name'), None) is not None:
+            hint_str += (
+                f'\n{INDENT_SYMBOL}To tailscale ssh:\t\t'
+                f'{BOLD}ssh root@{job_id}-workers-0-0 {RESET_BOLD}'
+            )
         return hint_str
     else:
         raise ValueError(f'Invalid hint type: {hint_type}')

{konduktor_nightly-0.1.0.dev20250430104745 → konduktor_nightly-0.1.0.dev20250501104750}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "konduktor-nightly"
-version = "0.1.0.dev20250430104745"
+version = "0.1.0.dev20250501104750"
 description = "GPU Cluster Health Management"
 packages = [
     {include = "konduktor"}