konduktor-nightly 0.1.0.dev20250715105209__py3-none-any.whl → 0.1.0.dev20250716105229__py3-none-any.whl

konduktor/__init__.py

@@ -14,7 +14,7 @@ __all__ = [
 ]
 
 # Replaced with the current commit when building the wheels.
-_KONDUKTOR_COMMIT_SHA = '5760f2c18ed487270e9244d22c7209eee12821c1'
+_KONDUKTOR_COMMIT_SHA = 'd7075fa88a909a1b4ccf58487d0de364c2a74e7d'
 os.makedirs(os.path.expanduser('~/.konduktor'), exist_ok=True)
 
 
@@ -48,5 +48,5 @@ def _get_git_commit():
 
 
 __commit__ = _get_git_commit()
-__version__ = '1.0.0.dev0.1.0.dev20250715105209'
+__version__ = '1.0.0.dev0.1.0.dev20250716105229'
 __root_dir__ = os.path.dirname(os.path.abspath(__file__))

konduktor/backends/jobset_utils.py

@@ -79,6 +79,8 @@ def create_pod_spec(task: 'konduktor.Task') -> Dict[str, Any]:
     Returns:
         Dict[str, Any]: k8s pod spec
     """
+    context = kubernetes_utils.get_current_kube_config_context_name()
+    namespace = kubernetes_utils.get_kube_config_context_namespace(context)
 
     # fill out the templating variables
     assert task.resources is not None, 'Task resources are required'
@@ -107,7 +109,6 @@ def create_pod_spec(task: 'konduktor.Task') -> Dict[str, Any]:
     sync_commands = []
     mkdir_commands = []
     storage_secrets = {}
-
     # first do storage_mount sync
     for dst, store in task.storage_mounts.items():
         # TODO(asaiacai) idk why but theres an extra storage mount for the
@@ -122,6 +123,13 @@ def create_pod_spec(task: 'konduktor.Task') -> Dict[str, Any]:
         # should impelement a method here instead of raw dog dict access
         cloud_store = registry._REGISTRY[store_scheme]
         storage_secrets[store_scheme] = cloud_store._STORE.get_k8s_credential_name()
+        exists, _ = kubernetes_utils.check_secret_exists(
+            storage_secrets[store_scheme], namespace=namespace, context=context
+        )
+        assert exists, (
+            f"secret {storage_secrets[store_scheme]} doesn't "
+            f'exist in namespace {namespace}'
+        )
         mkdir_commands.append(
             f'cd {constants.KONDUKTOR_REMOTE_WORKDIR};' f'mkdir -p {dst}'
         )
@@ -142,10 +150,15 @@ def create_pod_spec(task: 'konduktor.Task') -> Dict[str, Any]:
             f'mkdir -p {os.path.dirname(dst)}'
         )
         storage_secrets[store_scheme] = cloud_store._STORE.get_k8s_credential_name()
+        exists, reason = kubernetes_utils.check_secret_exists(
+            storage_secrets[store_scheme], namespace=namespace, context=context
+        )
+        assert exists, (
+            f'secret {storage_secrets[store_scheme]} '
+            f"doesn't exist in namespace {namespace}"
+        )
         sync_commands.append(cloud_store.make_sync_file_command(src, dst))
 
-    context = kubernetes_utils.get_current_kube_config_context_name()
-    namespace = kubernetes_utils.get_kube_config_context_namespace(context)
     tailscale_secret = config.get_nested(('tailscale', 'secret_name'), None)
     if tailscale_secret:
         secret_exist, err = kubernetes_utils.check_secret_exists(
@@ -188,8 +201,6 @@ def create_pod_spec(task: 'konduktor.Task') -> Dict[str, Any]:
     env_secret_envs = []
     default_secrets = []
 
-    context = kubernetes_utils.get_current_kube_config_context_name()
-    namespace = kubernetes_utils.get_kube_config_context_namespace(context)
     user_hash = common_utils.get_user_hash()
     label_selector = f'konduktor/owner={user_hash}'
     user_secrets = kubernetes_utils.list_secrets(
@@ -251,6 +262,8 @@ def create_pod_spec(task: 'konduktor.Task') -> Dict[str, Any]:
                 'git_ssh': git_ssh_secret_name,
                 # --kind default
                 'default_secrets': default_secrets,
+                # KONDUKTOR_DEBUG
+                'konduktor_debug': os.getenv('KONDUKTOR_DEBUG', 0),
             },
             temp.name,
         )

konduktor/cli.py

@@ -713,6 +713,13 @@ def launch(
     required=False,
 )
 @click.option('--all', '-a', default=None, is_flag=True, help='Tear down all jobs.')
+@click.option(
+    '--all-users',
+    '--all_users',
+    default=False,
+    is_flag=True,
+    help='Include other users for teardown',
+)
 @click.option(
     '--yes',
     '-y',
@@ -723,7 +730,8 @@ def launch(
 )
 def down(
     jobs: List[str],
-    all: Optional[bool],  # pylint: disable=redefined-builtin
+    all: Optional[bool],
+    all_users: Optional[bool],
     yes: bool,
 ):
     # NOTE(dev): Keep the docstring consistent between the Python API and CLI.
@@ -753,26 +761,39 @@ def down(
       # Tear down all jobs matching a pattern.
       konduktor down "test-*"
       \b
-      # Tear down all existing jobs.
+      # Tear down all of this users jobs.
       konduktor down -a
+      konduktor down --all
+
+      # Tear down all jobs across all users
+      konduktor down --all --all-users
 
     """
 
     context = kubernetes_utils.get_current_kube_config_context_name()
     namespace = kubernetes_utils.get_kube_config_context_namespace(context)
+    jobs_response = jobset_utils.list_jobset(namespace)
+    assert jobs_response
+    jobs_specs = [
+        job
+        for job in jobs_response['items']
+        if (
+            job['metadata']['labels'][jobset_utils.JOBSET_USERID_LABEL]
+            == common_utils.user_and_hostname_hash()
+            and not all_users
+        )
+    ]
 
     if all:
-        jobs_specs = jobset_utils.list_jobset(namespace)
         assert jobs_specs is not None, f'No jobs found in namespace {namespace}'
         assert len(jobs_specs) > 0, f'No jobs found in namespace {namespace}'
-        jobs = [job['metadata']['name'] for job in jobs_specs['items']]
+        jobs = [job['metadata']['name'] for job in jobs_specs]
     elif jobs:
         # Get all available jobs to match against patterns
-        jobs_specs = jobset_utils.list_jobset(namespace)
-        if jobs_specs is None or len(jobs_specs.get('items', [])) == 0:
+        if len(jobs_specs) == 0:
             raise click.ClickException(f'No jobs found in namespace {namespace}')
 
-        all_job_names = [job['metadata']['name'] for job in jobs_specs['items']]
+        all_job_names = [job['metadata']['name'] for job in jobs_specs]
         matched_jobs = []
 
         for job_pattern in jobs:

konduktor/data/gcp/gcs.py

@@ -162,8 +162,6 @@ class GcsStore(storage_utils.AbstractStore):
         # This will generate
         # ~/.config/gcloud/application_default_credentials.json.
         f'{_INDENT_PREFIX}  $ gcloud auth application-default login\n'
-        f'{_INDENT_PREFIX}For more info: '
-        'https://konduktor.readthedocs.io/en/latest/getting-started/installation.html#google-cloud-platform-gcp'  # noqa: E501
     )
     _APPLICATION_CREDENTIAL_HINT = (
         'Run the following commands:\n'
@@ -171,8 +169,6 @@ class GcsStore(storage_utils.AbstractStore):
         f'{_INDENT_PREFIX}Or set the environment variable '
         'GOOGLE_APPLICATION_CREDENTIALS '
         'to the path of your service account key file.\n'
-        f'{_INDENT_PREFIX}For more info: '
-        'https://konduktor.readthedocs.ioo/en/latest/getting-started/installation.html#google-cloud-platform-gcp'  # noqa: E501
     )
 
     _REPR = 'GcsStore'

konduktor/templates/pod.yaml.j2

@@ -113,7 +113,10 @@ kubernetes:
             - |
               # TODO(asaiacai): add debug environment variable for printing the apt-update, apt-install, sync-files output
               # Helper function to conditionally use sudo
-              set -eo pipefail
+              # set -eo pipefail
+              {% if konduktor_debug %}
+              set -x
+              {% endif %}
               mkdir -p ~/.konduktor/tmp
               start_epoch=$(date +%s);
               start_setup=$(date +%s);

{konduktor_nightly-0.1.0.dev20250715105209.dist-info → konduktor_nightly-0.1.0.dev20250716105229.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: konduktor-nightly
-Version: 0.1.0.dev20250715105209
+Version: 0.1.0.dev20250716105229
 Summary: GPU Cluster Health Management
 Author: Andrew Aikawa
 Author-email: asai@berkeley.edu

{konduktor_nightly-0.1.0.dev20250715105209.dist-info → konduktor_nightly-0.1.0.dev20250716105229.dist-info}/RECORD

@@ -1,4 +1,4 @@
-konduktor/__init__.py,sha256=v-n0dnLTzwfu7Y3lEvTmtk4Cn7-hyB8aYcyeb_AcCgI,1540
+konduktor/__init__.py,sha256=zYoBNEW7JjgSjqIRFfaHeNgMTT2K8KGedqy-SUCRYa8,1540
 konduktor/adaptors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 konduktor/adaptors/aws.py,sha256=s47Ra-GaqCQibzVfmD0pmwEWHif1EGO5opMbwkLxTCU,8244
 konduktor/adaptors/common.py,sha256=ZIqzjx77PIHUwpjfAQ1uX8B2aX78YMuGj4Bppd-MdyM,4183
@@ -7,9 +7,9 @@ konduktor/authentication.py,sha256=_mVy3eqoKohicHostFiGwG1-2ybxP-l7ouofQ0LRlCY,4
 konduktor/backends/__init__.py,sha256=1Q6sqqdeMYarpTX_U-QVywJYf7idiUTRsyP-E4BQSOw,129
 konduktor/backends/backend.py,sha256=qh0bp94lzoTYZkzyQv2-CVrB5l91FkG2vclXg24UFC0,2910
 konduktor/backends/jobset.py,sha256=UdhwAuZODLMbLY51Y2zOBsh6wg4Pb84oHVvUKzx3Z2w,8434
-konduktor/backends/jobset_utils.py,sha256=u5Z3SYv7rmkS1M0l1sHR5QEnJ-mIMk5hrM9WoPjJCoE,22283
+konduktor/backends/jobset_utils.py,sha256=sE3USQ15rcXeTCmkysN5YYugiYcaqhWS65vdJRZvlJA,22827
 konduktor/check.py,sha256=JennyWoaqSKhdyfUldd266KwVXTPJpcYQa4EED4a_BA,7569
-konduktor/cli.py,sha256=qtktD8N17IRC5MYEdaE0o3pv8EI36cvyyQkYUFi5_nQ,35590
+konduktor/cli.py,sha256=GXmm4DGLHuvfnqDG_3PsWXK7mOI3XWvlC3VeI6JDoqI,36004
 konduktor/config.py,sha256=J50JxC6MsXMnlrJPXdDUMr38C89xvOO7mR8KJ6fyils,15520
 konduktor/constants.py,sha256=T3AeXXxuQHINW_bAWyztvDeS8r4g8kXBGIwIq13cys0,1814
 konduktor/controller/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -56,7 +56,7 @@ konduktor/data/constants.py,sha256=yXVEoTI2we1xOjVSU-bjRCQCLpVvpEvJ0GedXvSwEfw,1
 konduktor/data/data_utils.py,sha256=IG1jgb_La997wi90xCvxYYsHQRlmm8Aooq04ZSf8EDI,9670
 konduktor/data/gcp/__init__.py,sha256=rlQxACBC_Vu36mdgPyJgUy4mGc_6Nt_a96JAuaPz2pQ,489
 konduktor/data/gcp/constants.py,sha256=dMfOiFccM8O6rUi9kClJcbvw1K1VnS1JzzQk3apq8ho,1483
-konduktor/data/gcp/gcs.py,sha256=Zc1LXrjoeNU9EDK229evrKxjVqsKIUicbtYlugA_TiY,42229
+konduktor/data/gcp/gcs.py,sha256=fFaQydgFj0zJbZrVCrOq7goXE6gT19i3f1NQpe_Hdq4,41888
 konduktor/data/gcp/utils.py,sha256=FJQcMXZqtMIzjZ98b3lTTc0UbdPUKTDLsOsfJaaH5-s,214
 konduktor/data/registry.py,sha256=CUbMsN_Q17Pf4wRHkqZrycErEjTP7cLEdgcfwVGcEpc,696
 konduktor/data/storage.py,sha256=o2So-bY9glvgbGdoN7AQNYmNnvGf1AUDPpImtadRL90,35213
@@ -71,7 +71,7 @@ konduktor/manifests/pod_cleanup_controller.yaml,sha256=hziL1Ka1kCAEL9R7Tjvpb80iw
 konduktor/resource.py,sha256=nHgPWXCbWj5sWyslNngrFypMN1K0Dksb0yHbJqWaei8,19612
 konduktor/task.py,sha256=ofwd8WIhfD6C3ThLcv6X3GUzQHyZ6ddjUagE-umF4K0,35207
 konduktor/templates/jobset.yaml.j2,sha256=rdURknodtgLp4zoA2PX86Nn4wPpi3tr5l4IG55aWBRg,1059
-konduktor/templates/pod.yaml.j2,sha256=7512UVJHv_8mQu0WO9yuOR3HGjWYtHOLe2IsatKeDH0,16635
+konduktor/templates/pod.yaml.j2,sha256=PagIrQnON2L32m6A1tdFnO2ieF2Lzggm8AwISWQB-Kk,16723
 konduktor/usage/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 konduktor/usage/constants.py,sha256=gCL8afIHZhO0dcxbJGpESE9sCC1cBSbeRnQ8GwNOY4M,612
 konduktor/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -91,8 +91,8 @@ konduktor/utils/schemas.py,sha256=VGPERAso2G4sVAznsJ80qT2Q-I_EFxXw6Rfcw-vkYgQ,16
 konduktor/utils/subprocess_utils.py,sha256=WoFkoFhGecPR8-rF8WJxbIe-YtV94LXz9UG64SDhCY4,9448
 konduktor/utils/ux_utils.py,sha256=czCwiS1bDqgeKtzAJctczpLwFZzAse7WuozdvzEFYJ4,7437
 konduktor/utils/validator.py,sha256=uCRlScO1NYxsbTNKY9dkoqvlO8S0ISIIB8XmX2ItcO8,2793
-konduktor_nightly-0.1.0.dev20250715105209.dist-info/LICENSE,sha256=MuuqTZbHvmqXR_aNKAXzggdV45ANd3wQ5YI7tnpZhm0,6586
-konduktor_nightly-0.1.0.dev20250715105209.dist-info/METADATA,sha256=wJo0dIzmQqEuSfL-9zkBgRtl9f1xVEtcHmtf2o9cdj0,4247
-konduktor_nightly-0.1.0.dev20250715105209.dist-info/WHEEL,sha256=XbeZDeTWKc1w7CSIyre5aMDU_-PohRwTQceYnisIYYY,88
-konduktor_nightly-0.1.0.dev20250715105209.dist-info/entry_points.txt,sha256=k3nG5wDFIJhNqsZWrHk4d0irIB2Ns9s47cjRWYsTCT8,48
-konduktor_nightly-0.1.0.dev20250715105209.dist-info/RECORD,,
+konduktor_nightly-0.1.0.dev20250716105229.dist-info/LICENSE,sha256=MuuqTZbHvmqXR_aNKAXzggdV45ANd3wQ5YI7tnpZhm0,6586
+konduktor_nightly-0.1.0.dev20250716105229.dist-info/METADATA,sha256=b7CQSOl3ZWKLIni_LBzurM0ff7pjhOXpMH8KVRxNOzE,4247
+konduktor_nightly-0.1.0.dev20250716105229.dist-info/WHEEL,sha256=XbeZDeTWKc1w7CSIyre5aMDU_-PohRwTQceYnisIYYY,88
+konduktor_nightly-0.1.0.dev20250716105229.dist-info/entry_points.txt,sha256=k3nG5wDFIJhNqsZWrHk4d0irIB2Ns9s47cjRWYsTCT8,48
+konduktor_nightly-0.1.0.dev20250716105229.dist-info/RECORD,,