konduktor-nightly 0.1.0.dev20250715105209__py3-none-any.whl → 0.1.0.dev20250717105242__py3-none-any.whl

konduktor/__init__.py

@@ -14,7 +14,7 @@ __all__ = [
 ]
 
 # Replaced with the current commit when building the wheels.
-_KONDUKTOR_COMMIT_SHA = '5760f2c18ed487270e9244d22c7209eee12821c1'
+_KONDUKTOR_COMMIT_SHA = '84818710a16e0a0515fbbd7878395fca37cf94f7'
 os.makedirs(os.path.expanduser('~/.konduktor'), exist_ok=True)
 
 
@@ -48,5 +48,5 @@ def _get_git_commit():
 
 
 __commit__ = _get_git_commit()
-__version__ = '1.0.0.dev0.1.0.dev20250715105209'
+__version__ = '1.0.0.dev0.1.0.dev20250717105242'
 __root_dir__ = os.path.dirname(os.path.abspath(__file__))

konduktor/backends/constants.py

@@ -0,0 +1 @@
+KONDUKTOR_SSH_PORT = 2222

konduktor/backends/jobset_utils.py

@@ -18,6 +18,7 @@ if typing.TYPE_CHECKING:
 
 import konduktor
 from konduktor import authentication, config, constants, kube_client, logging
+from konduktor.backends import constants as backend_constants
 from konduktor.data import registry
 from konduktor.utils import (
     common_utils,
@@ -79,6 +80,8 @@ def create_pod_spec(task: 'konduktor.Task') -> Dict[str, Any]:
     Returns:
         Dict[str, Any]: k8s pod spec
     """
+    context = kubernetes_utils.get_current_kube_config_context_name()
+    namespace = kubernetes_utils.get_kube_config_context_namespace(context)
 
     # fill out the templating variables
     assert task.resources is not None, 'Task resources are required'
@@ -107,7 +110,6 @@ def create_pod_spec(task: 'konduktor.Task') -> Dict[str, Any]:
     sync_commands = []
     mkdir_commands = []
     storage_secrets = {}
-
     # first do storage_mount sync
     for dst, store in task.storage_mounts.items():
         # TODO(asaiacai) idk why but theres an extra storage mount for the
@@ -122,6 +124,13 @@ def create_pod_spec(task: 'konduktor.Task') -> Dict[str, Any]:
         # should impelement a method here instead of raw dog dict access
         cloud_store = registry._REGISTRY[store_scheme]
         storage_secrets[store_scheme] = cloud_store._STORE.get_k8s_credential_name()
+        exists, _ = kubernetes_utils.check_secret_exists(
+            storage_secrets[store_scheme], namespace=namespace, context=context
+        )
+        assert exists, (
+            f"secret {storage_secrets[store_scheme]} doesn't "
+            f'exist in namespace {namespace}'
+        )
         mkdir_commands.append(
             f'cd {constants.KONDUKTOR_REMOTE_WORKDIR};' f'mkdir -p {dst}'
         )
@@ -142,10 +151,15 @@ def create_pod_spec(task: 'konduktor.Task') -> Dict[str, Any]:
             f'mkdir -p {os.path.dirname(dst)}'
         )
         storage_secrets[store_scheme] = cloud_store._STORE.get_k8s_credential_name()
+        exists, reason = kubernetes_utils.check_secret_exists(
+            storage_secrets[store_scheme], namespace=namespace, context=context
+        )
+        assert exists, (
+            f'secret {storage_secrets[store_scheme]} '
+            f"doesn't exist in namespace {namespace}"
+        )
         sync_commands.append(cloud_store.make_sync_file_command(src, dst))
 
-    context = kubernetes_utils.get_current_kube_config_context_name()
-    namespace = kubernetes_utils.get_kube_config_context_namespace(context)
     tailscale_secret = config.get_nested(('tailscale', 'secret_name'), None)
     if tailscale_secret:
         secret_exist, err = kubernetes_utils.check_secret_exists(
@@ -158,7 +172,7 @@ def create_pod_spec(task: 'konduktor.Task') -> Dict[str, Any]:
                     f'though specified by `tailscale.secret_name`: {err}'
                 )
 
-    enable_ssh = config.get_nested(('ssh', 'enable'), False)
+    enable_ssh = config.get_nested(('ssh', 'enable'), False) or tailscale_secret
     secret_name = None
     if enable_ssh:
         private_key_path, public_key_path = authentication.get_or_generate_keys()
@@ -188,8 +202,6 @@ def create_pod_spec(task: 'konduktor.Task') -> Dict[str, Any]:
     env_secret_envs = []
     default_secrets = []
 
-    context = kubernetes_utils.get_current_kube_config_context_name()
-    namespace = kubernetes_utils.get_kube_config_context_namespace(context)
     user_hash = common_utils.get_user_hash()
     label_selector = f'konduktor/owner={user_hash}'
     user_secrets = kubernetes_utils.list_secrets(
@@ -246,11 +258,14 @@ def create_pod_spec(task: 'konduktor.Task') -> Dict[str, Any]:
                 # SSH
                 'enable_ssh': enable_ssh,
                 'secret_name': secret_name,
+                'konduktor_ssh_port': backend_constants.KONDUKTOR_SSH_PORT,
                 # Kinds of Secrets
                 # --kind git-ssh
                 'git_ssh': git_ssh_secret_name,
                 # --kind default
                 'default_secrets': default_secrets,
+                # KONDUKTOR_DEBUG
+                'konduktor_debug': os.getenv('KONDUKTOR_DEBUG', 0),
             },
             temp.name,
         )
@@ -277,6 +292,7 @@ def create_pod_spec(task: 'konduktor.Task') -> Dict[str, Any]:
     pod_config['kubernetes']['pod_config']['spec']['containers'][0]['env'] = list(
         env_map.values()
     )
+    logger.debug(f'rendered pod spec: \n\t{pod_config}')
 
     # validate pod spec using json schema
     try:

konduktor/cli.py

@@ -713,6 +713,13 @@ def launch(
     required=False,
 )
 @click.option('--all', '-a', default=None, is_flag=True, help='Tear down all jobs.')
+@click.option(
+    '--all-users',
+    '--all_users',
+    default=False,
+    is_flag=True,
+    help='Include other users for teardown',
+)
 @click.option(
     '--yes',
     '-y',
@@ -723,7 +730,8 @@ def launch(
 )
 def down(
     jobs: List[str],
-    all: Optional[bool],  # pylint: disable=redefined-builtin
+    all: Optional[bool],
+    all_users: Optional[bool],
     yes: bool,
 ):
     # NOTE(dev): Keep the docstring consistent between the Python API and CLI.
@@ -753,26 +761,39 @@ def down(
       # Tear down all jobs matching a pattern.
       konduktor down "test-*"
       \b
-      # Tear down all existing jobs.
+      # Tear down all of this users jobs.
       konduktor down -a
+      konduktor down --all
+
+      # Tear down all jobs across all users
+      konduktor down --all --all-users
 
     """
 
     context = kubernetes_utils.get_current_kube_config_context_name()
     namespace = kubernetes_utils.get_kube_config_context_namespace(context)
+    jobs_response = jobset_utils.list_jobset(namespace)
+    assert jobs_response
+    jobs_specs = [
+        job
+        for job in jobs_response['items']
+        if (
+            job['metadata']['labels'][jobset_utils.JOBSET_USERID_LABEL]
+            == common_utils.user_and_hostname_hash()
+            and not all_users
+        )
+    ]
 
     if all:
-        jobs_specs = jobset_utils.list_jobset(namespace)
         assert jobs_specs is not None, f'No jobs found in namespace {namespace}'
         assert len(jobs_specs) > 0, f'No jobs found in namespace {namespace}'
-        jobs = [job['metadata']['name'] for job in jobs_specs['items']]
+        jobs = [job['metadata']['name'] for job in jobs_specs]
     elif jobs:
         # Get all available jobs to match against patterns
-        jobs_specs = jobset_utils.list_jobset(namespace)
-        if jobs_specs is None or len(jobs_specs.get('items', [])) == 0:
+        if len(jobs_specs) == 0:
             raise click.ClickException(f'No jobs found in namespace {namespace}')
 
-        all_job_names = [job['metadata']['name'] for job in jobs_specs['items']]
+        all_job_names = [job['metadata']['name'] for job in jobs_specs]
         matched_jobs = []
 
         for job_pattern in jobs:

konduktor/data/gcp/gcs.py

@@ -162,8 +162,6 @@ class GcsStore(storage_utils.AbstractStore):
         # This will generate
         # ~/.config/gcloud/application_default_credentials.json.
         f'{_INDENT_PREFIX}  $ gcloud auth application-default login\n'
-        f'{_INDENT_PREFIX}For more info: '
-        'https://konduktor.readthedocs.io/en/latest/getting-started/installation.html#google-cloud-platform-gcp'  # noqa: E501
     )
     _APPLICATION_CREDENTIAL_HINT = (
         'Run the following commands:\n'
@@ -171,8 +169,6 @@ class GcsStore(storage_utils.AbstractStore):
         f'{_INDENT_PREFIX}Or set the environment variable '
         'GOOGLE_APPLICATION_CREDENTIALS '
         'to the path of your service account key file.\n'
-        f'{_INDENT_PREFIX}For more info: '
-        'https://konduktor.readthedocs.ioo/en/latest/getting-started/installation.html#google-cloud-platform-gcp'  # noqa: E501
     )
 
     _REPR = 'GcsStore'

konduktor/templates/pod.yaml.j2

@@ -20,7 +20,8 @@ kubernetes:
         - name: konduktor-container
           {% if enable_ssh %}
           ports:
-            - containerPort: 2222
+            - name: ssh
+              containerPort: {{ konduktor_ssh_port }}
           {% endif %}
           image: {{ image_id }}
           # this is set during jobset definition since we need to know the jobset
@@ -38,6 +39,10 @@ kubernetes:
             valueFrom:
               fieldRef:
                 fieldPath: metadata.annotations['batch.kubernetes.io/job-completion-index']
+          - name: LOCAL_ADDR
+            valueFrom:
+              fieldRef:
+                fieldPath: status.podIP
           - name: NUM_NODES
             value: "{{ num_nodes }}"
           - name: NUM_GPUS_PER_NODE
@@ -71,6 +76,8 @@ kubernetes:
               secretKeyRef:
                 name: {{ secret_name }}
                 key: PRIVKEY
+          - name: KONDUKTOR_SSH_PORT
+            value: "{{ konduktor_ssh_port }}"
           {% endif %}
           {% if git_ssh %}
           - name: GIT_SSH_COMMAND
@@ -113,7 +120,11 @@ kubernetes:
             - |
               # TODO(asaiacai): add debug environment variable for printing the apt-update, apt-install, sync-files output
               # Helper function to conditionally use sudo
+              export RDZV_CONF=is_host=$(if [ "$RANK" == "0" ]; then echo "true"; else echo "false"; fi)
               set -eo pipefail
+              {% if konduktor_debug %}
+              set -x
+              {% endif %}
               mkdir -p ~/.konduktor/tmp
               start_epoch=$(date +%s);
               start_setup=$(date +%s);
@@ -167,7 +178,7 @@ kubernetes:
                 DEBIAN_FRONTEND=noninteractive $(prefix_cmd) apt-get install -y $MISSING_PACKAGES 2>&1 | tee -a ~/.konduktor/tmp/apt-install.log;
               fi;
               end_epoch=$(date +%s);
-
+              
               echo "Exposing ENV variables"
               $(prefix_cmd) env -0 | awk -v RS='\0' '
                 {
@@ -185,6 +196,7 @@ kubernetes:
               function InstallSSH {
                 export DEBIAN_FRONTEND=noninteractive
                 export TZ=Etc/UTC
+                set -u
                 if service sshd status > /dev/null 2>&1; then
                   $(prefix_cmd) echo "OpenSSH server is already started."
                   return
@@ -242,14 +254,15 @@ kubernetes:
                 # turn off PAM to fix sshd login issue
                 $(prefix_cmd) sed -i 's/UsePAM yes/UsePAM no/' /etc/ssh/sshd_config
 
-                # set default port to 2222
-                $(prefix_cmd) sed -i 's/#Port 22/Port 2222/' /etc/ssh/sshd_config
+                # set default port to 22
+                $(prefix_cmd) sed -i 's/#Port 22/Port {{ konduktor_ssh_port }}/' /etc/ssh/sshd_config
 
                 $(prefix_cmd) mkdir /run/sshd
                 $(prefix_cmd) chmod 0755 /run/sshd
 
                 $(prefix_cmd) service ssh start
                 $(prefix_cmd) echo "sshd service started"
+                set +u
               }
 
               InstallSSH
@@ -263,7 +276,7 @@ kubernetes:
                   if ! command -v tailscale >/dev/null 2>&1; then
                     $(prefix_cmd) curl -fsSL https://tailscale.com/install.sh | DEBIAN_FRONTEND=noninteractive $(prefix_cmd) sh > ~/.konduktor/tmp/tailscale-install.log 2>&1
                   fi
-                  $(prefix_cmd) tailscaled --tun=userspace-networking >/dev/null 2>&1 &
+                  $(prefix_cmd) tailscaled --tun=userspace-networking --state=mem: >/dev/null 2>&1 &
                   $(prefix_cmd) timeout 5 tailscale up --auth-key=${TS_AUTHKEY} --ssh --hostname=${TS_HOSTNAME}
                   $(prefix_cmd) sleep 10
                 done

{konduktor_nightly-0.1.0.dev20250715105209.dist-info → konduktor_nightly-0.1.0.dev20250717105242.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: konduktor-nightly
-Version: 0.1.0.dev20250715105209
+Version: 0.1.0.dev20250717105242
 Summary: GPU Cluster Health Management
 Author: Andrew Aikawa
 Author-email: asai@berkeley.edu

{konduktor_nightly-0.1.0.dev20250715105209.dist-info → konduktor_nightly-0.1.0.dev20250717105242.dist-info}/RECORD

@@ -1,4 +1,4 @@
-konduktor/__init__.py,sha256=v-n0dnLTzwfu7Y3lEvTmtk4Cn7-hyB8aYcyeb_AcCgI,1540
+konduktor/__init__.py,sha256=No4XStM7tVXkkg8Ut05Ttw2ZkuKgSK71G639a6FYooM,1540
 konduktor/adaptors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 konduktor/adaptors/aws.py,sha256=s47Ra-GaqCQibzVfmD0pmwEWHif1EGO5opMbwkLxTCU,8244
 konduktor/adaptors/common.py,sha256=ZIqzjx77PIHUwpjfAQ1uX8B2aX78YMuGj4Bppd-MdyM,4183
@@ -6,10 +6,11 @@ konduktor/adaptors/gcp.py,sha256=ierTF4z7vwpJ9BsC7LSiwv4uLcjGXscwZOwQrddr2vM,410
 konduktor/authentication.py,sha256=_mVy3eqoKohicHostFiGwG1-2ybxP-l7ouofQ0LRlCY,4570
 konduktor/backends/__init__.py,sha256=1Q6sqqdeMYarpTX_U-QVywJYf7idiUTRsyP-E4BQSOw,129
 konduktor/backends/backend.py,sha256=qh0bp94lzoTYZkzyQv2-CVrB5l91FkG2vclXg24UFC0,2910
+konduktor/backends/constants.py,sha256=nU_cd4x8V2GwP9-oGlcIwjt5snnyhmOlxXbXRZ8d6Fc,26
 konduktor/backends/jobset.py,sha256=UdhwAuZODLMbLY51Y2zOBsh6wg4Pb84oHVvUKzx3Z2w,8434
-konduktor/backends/jobset_utils.py,sha256=u5Z3SYv7rmkS1M0l1sHR5QEnJ-mIMk5hrM9WoPjJCoE,22283
+konduktor/backends/jobset_utils.py,sha256=zOxXikz5fRsciTIdEgK3mQNxJfsRVEiW1K6FQI5ZD3Y,23042
 konduktor/check.py,sha256=JennyWoaqSKhdyfUldd266KwVXTPJpcYQa4EED4a_BA,7569
-konduktor/cli.py,sha256=qtktD8N17IRC5MYEdaE0o3pv8EI36cvyyQkYUFi5_nQ,35590
+konduktor/cli.py,sha256=GXmm4DGLHuvfnqDG_3PsWXK7mOI3XWvlC3VeI6JDoqI,36004
 konduktor/config.py,sha256=J50JxC6MsXMnlrJPXdDUMr38C89xvOO7mR8KJ6fyils,15520
 konduktor/constants.py,sha256=T3AeXXxuQHINW_bAWyztvDeS8r4g8kXBGIwIq13cys0,1814
 konduktor/controller/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -56,7 +57,7 @@ konduktor/data/constants.py,sha256=yXVEoTI2we1xOjVSU-bjRCQCLpVvpEvJ0GedXvSwEfw,1
 konduktor/data/data_utils.py,sha256=IG1jgb_La997wi90xCvxYYsHQRlmm8Aooq04ZSf8EDI,9670
 konduktor/data/gcp/__init__.py,sha256=rlQxACBC_Vu36mdgPyJgUy4mGc_6Nt_a96JAuaPz2pQ,489
 konduktor/data/gcp/constants.py,sha256=dMfOiFccM8O6rUi9kClJcbvw1K1VnS1JzzQk3apq8ho,1483
-konduktor/data/gcp/gcs.py,sha256=Zc1LXrjoeNU9EDK229evrKxjVqsKIUicbtYlugA_TiY,42229
+konduktor/data/gcp/gcs.py,sha256=fFaQydgFj0zJbZrVCrOq7goXE6gT19i3f1NQpe_Hdq4,41888
 konduktor/data/gcp/utils.py,sha256=FJQcMXZqtMIzjZ98b3lTTc0UbdPUKTDLsOsfJaaH5-s,214
 konduktor/data/registry.py,sha256=CUbMsN_Q17Pf4wRHkqZrycErEjTP7cLEdgcfwVGcEpc,696
 konduktor/data/storage.py,sha256=o2So-bY9glvgbGdoN7AQNYmNnvGf1AUDPpImtadRL90,35213
@@ -71,7 +72,7 @@ konduktor/manifests/pod_cleanup_controller.yaml,sha256=hziL1Ka1kCAEL9R7Tjvpb80iw
 konduktor/resource.py,sha256=nHgPWXCbWj5sWyslNngrFypMN1K0Dksb0yHbJqWaei8,19612
 konduktor/task.py,sha256=ofwd8WIhfD6C3ThLcv6X3GUzQHyZ6ddjUagE-umF4K0,35207
 konduktor/templates/jobset.yaml.j2,sha256=rdURknodtgLp4zoA2PX86Nn4wPpi3tr5l4IG55aWBRg,1059
-konduktor/templates/pod.yaml.j2,sha256=7512UVJHv_8mQu0WO9yuOR3HGjWYtHOLe2IsatKeDH0,16635
+konduktor/templates/pod.yaml.j2,sha256=GoBNypXgPkFfmD-a4t0WVCo9sXbgSedvl-YNygso-Fc,17160
 konduktor/usage/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 konduktor/usage/constants.py,sha256=gCL8afIHZhO0dcxbJGpESE9sCC1cBSbeRnQ8GwNOY4M,612
 konduktor/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -91,8 +92,8 @@ konduktor/utils/schemas.py,sha256=VGPERAso2G4sVAznsJ80qT2Q-I_EFxXw6Rfcw-vkYgQ,16
 konduktor/utils/subprocess_utils.py,sha256=WoFkoFhGecPR8-rF8WJxbIe-YtV94LXz9UG64SDhCY4,9448
 konduktor/utils/ux_utils.py,sha256=czCwiS1bDqgeKtzAJctczpLwFZzAse7WuozdvzEFYJ4,7437
 konduktor/utils/validator.py,sha256=uCRlScO1NYxsbTNKY9dkoqvlO8S0ISIIB8XmX2ItcO8,2793
-konduktor_nightly-0.1.0.dev20250715105209.dist-info/LICENSE,sha256=MuuqTZbHvmqXR_aNKAXzggdV45ANd3wQ5YI7tnpZhm0,6586
-konduktor_nightly-0.1.0.dev20250715105209.dist-info/METADATA,sha256=wJo0dIzmQqEuSfL-9zkBgRtl9f1xVEtcHmtf2o9cdj0,4247
-konduktor_nightly-0.1.0.dev20250715105209.dist-info/WHEEL,sha256=XbeZDeTWKc1w7CSIyre5aMDU_-PohRwTQceYnisIYYY,88
-konduktor_nightly-0.1.0.dev20250715105209.dist-info/entry_points.txt,sha256=k3nG5wDFIJhNqsZWrHk4d0irIB2Ns9s47cjRWYsTCT8,48
-konduktor_nightly-0.1.0.dev20250715105209.dist-info/RECORD,,
+konduktor_nightly-0.1.0.dev20250717105242.dist-info/LICENSE,sha256=MuuqTZbHvmqXR_aNKAXzggdV45ANd3wQ5YI7tnpZhm0,6586
+konduktor_nightly-0.1.0.dev20250717105242.dist-info/METADATA,sha256=Ia8KlWcaZd7H2K18mcrXni62pY2E9mUj9mvxAFG5yf8,4247
+konduktor_nightly-0.1.0.dev20250717105242.dist-info/WHEEL,sha256=XbeZDeTWKc1w7CSIyre5aMDU_-PohRwTQceYnisIYYY,88
+konduktor_nightly-0.1.0.dev20250717105242.dist-info/entry_points.txt,sha256=k3nG5wDFIJhNqsZWrHk4d0irIB2Ns9s47cjRWYsTCT8,48
+konduktor_nightly-0.1.0.dev20250717105242.dist-info/RECORD,,