konduktor-nightly 0.1.0.dev20250512104920__py3-none-any.whl → 0.1.0.dev20250514104854__py3-none-any.whl

konduktor/__init__.py

@@ -14,7 +14,7 @@ __all__ = [
 ]
 
 # Replaced with the current commit when building the wheels.
-_KONDUKTOR_COMMIT_SHA = '2b0d682b6fc8ff0d4e5ea417c4e324090f3c5f9b'
+_KONDUKTOR_COMMIT_SHA = '05c7d9e243ae23c6e9abb0a4a034bfc0815fd587'
 os.makedirs(os.path.expanduser('~/.konduktor'), exist_ok=True)
 
 
@@ -48,5 +48,5 @@ def _get_git_commit():
 
 
 __commit__ = _get_git_commit()
-__version__ = '1.0.0.dev0.1.0.dev20250512104920'
+__version__ = '1.0.0.dev0.1.0.dev20250514104854'
 __root_dir__ = os.path.dirname(os.path.abspath(__file__))

konduktor/authentication.py

@@ -0,0 +1,124 @@
+# Proprietary Changes made for Trainy under the Trainy Software License
+# Original source: skypilot: https://github.com/skypilot-org/skypilot
+# which is Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+The local machine's public key should not be uploaded to the remote VM, because
+it will cause private/public key pair mismatch when the user tries to launch new
+VM from that remote VM using SkyPilot, e.g., the node is used as a jobs
+controller. (Lambda cloud is an exception, due to the limitation of the cloud
+provider. See the comments in setup_lambda_authentication)
+"""
+
+import functools
+import os
+from typing import Tuple
+
+import filelock
+
+from konduktor import logging
+from konduktor.utils import common_utils
+
+logger = logging.get_logger(__name__)
+
+_SSH_KEY_PATH_PREFIX = '~/.konduktor/clients/{user_hash}/ssh'
+
+MAX_TRIALS = 64
+
+
+def get_ssh_key_and_lock_path() -> Tuple[str, str, str]:
+    user_hash = common_utils.get_user_hash()
+    user_ssh_key_prefix = _SSH_KEY_PATH_PREFIX.format(user_hash=user_hash)
+    os.makedirs(os.path.expanduser(user_ssh_key_prefix), exist_ok=True, mode=0o700)
+    private_key_path = os.path.join(user_ssh_key_prefix, 'konduktor-key')
+    public_key_path = os.path.join(user_ssh_key_prefix, 'konduktor-key.pub')
+    lock_path = os.path.join(user_ssh_key_prefix, '.__internal-konduktor-key.lock')
+    return private_key_path, public_key_path, lock_path
+
+
+def _generate_rsa_key_pair() -> Tuple[str, str]:
+    # Keep the import of the cryptography local to avoid expensive
+    # third-party imports when not needed.
+    # pylint: disable=import-outside-toplevel
+    from cryptography.hazmat.backends import default_backend
+    from cryptography.hazmat.primitives import serialization
+    from cryptography.hazmat.primitives.asymmetric import rsa
+
+    key = rsa.generate_private_key(
+        backend=default_backend(), public_exponent=65537, key_size=2048
+    )
+
+    private_key = (
+        key.private_bytes(
+            encoding=serialization.Encoding.PEM,
+            format=serialization.PrivateFormat.TraditionalOpenSSL,
+            encryption_algorithm=serialization.NoEncryption(),
+        )
+        .decode('utf-8')
+        .strip()
+    )
+
+    public_key = (
+        key.public_key()
+        .public_bytes(
+            serialization.Encoding.OpenSSH, serialization.PublicFormat.OpenSSH
+        )
+        .decode('utf-8')
+        .strip()
+    )
+
+    return public_key, private_key
+
+
+def _save_key_pair(
+    private_key_path: str, public_key_path: str, private_key: str, public_key: str
+) -> None:
+    key_dir = os.path.dirname(private_key_path)
+    os.makedirs(key_dir, exist_ok=True, mode=0o700)
+
+    with open(
+        private_key_path,
+        'w',
+        encoding='utf-8',
+        opener=functools.partial(os.open, mode=0o600),
+    ) as f:
+        f.write(private_key)
+
+    with open(
+        public_key_path,
+        'w',
+        encoding='utf-8',
+        opener=functools.partial(os.open, mode=0o644),
+    ) as f:
+        f.write(public_key)
+
+
+def get_or_generate_keys() -> Tuple[str, str]:
+    """Returns the aboslute private and public key paths."""
+    private_key_path, public_key_path, lock_path = get_ssh_key_and_lock_path()
+    private_key_path = os.path.expanduser(private_key_path)
+    public_key_path = os.path.expanduser(public_key_path)
+    lock_path = os.path.expanduser(lock_path)
+
+    lock_dir = os.path.dirname(lock_path)
+    # We should have the folder ~/.sky/generated/ssh to have 0o700 permission,
+    # as the ssh configs will be written to this folder as well in
+    # backend_utils.SSHConfigHelper
+    os.makedirs(lock_dir, exist_ok=True, mode=0o700)
+    with filelock.FileLock(lock_path, timeout=10):
+        if not os.path.exists(private_key_path):
+            public_key, private_key = _generate_rsa_key_pair()
+            _save_key_pair(private_key_path, public_key_path, private_key, public_key)
+    assert os.path.exists(public_key_path), (
+        'Private key found, but associated public key '
+        f'{public_key_path} does not exist.'
+    )
+    return private_key_path, public_key_path

konduktor/backends/jobset.py

@@ -70,25 +70,26 @@ def _wait_for_jobset_start(namespace: str, job_name: str):
         assert jobsets is not None, (
             f'Jobset {job_name} ' f'not found in namespace {namespace}'
         )
-        if jobsets['status']['replicatedJobsStatus'][0]['ready']:
-            logger.info(
-                f'task '
-                f'{colorama.Fore.CYAN}{colorama.Style.BRIGHT}{job_name}'
-                f'{colorama.Style.RESET_ALL} ready'
-            )
-            break
-        elif jobsets['status']['replicatedJobsStatus'][0]['succeeded']:
-            return
-        elif jobsets['status']['replicatedJobsStatus'][0]['failed']:
-            logger.info(
-                f'job '
-                f'{colorama.Fore.CYAN}{colorama.Style.BRIGHT}{job_name}'
-                f'{colorama.Style.RESET_ALL} '
-                f'{colorama.Fore.RED}{colorama.Style.BRIGHT}failed{colorama.Style.RESET_ALL}'
-            )
-            job = jobset_utils.get_job(namespace, job_name)
-            _raise_job_error(job)
-            return
+        if 'status' in jobsets:
+            if jobsets['status']['replicatedJobsStatus'][0]['ready']:
+                logger.info(
+                    f'task '
+                    f'{colorama.Fore.CYAN}{colorama.Style.BRIGHT}{job_name}'
+                    f'{colorama.Style.RESET_ALL} ready'
+                )
+                break
+            elif jobsets['status']['replicatedJobsStatus'][0]['succeeded']:
+                return
+            elif jobsets['status']['replicatedJobsStatus'][0]['failed']:
+                logger.info(
+                    f'job '
+                    f'{colorama.Fore.CYAN}{colorama.Style.BRIGHT}{job_name}'
+                    f'{colorama.Style.RESET_ALL} '
+                    f'{colorama.Fore.RED}{colorama.Style.BRIGHT}failed{colorama.Style.RESET_ALL}'
+                )
+                job = jobset_utils.get_job(namespace, job_name)
+                _raise_job_error(job)
+                return
         if timeout != -1 and time.time() - start > timeout:
             logger.error(
                 f'{colorama.Style.BRIGHT}'

konduktor/backends/jobset_utils.py

@@ -1,5 +1,6 @@
 """Jobset utils: wraps CRUD operations for jobsets"""
 
+import base64
 import enum
 import json
 import os
@@ -15,7 +16,7 @@ if typing.TYPE_CHECKING:
     from datetime import timedelta
 
 import konduktor
-from konduktor import config, constants, kube_client, logging
+from konduktor import authentication, config, constants, kube_client, logging
 from konduktor.data import registry
 from konduktor.utils import (
     common_utils,
@@ -93,6 +94,10 @@ def create_pod_spec(task: 'konduktor.Task') -> Dict[str, Any]:
     else:
         accelerator_type = None
 
+    assert task.resources.cpus is not None, 'Task resources cpus are required'
+    assert task.resources.memory is not None, 'Task resources memory are required'
+    assert task.resources.image_id is not None, 'Task resources image_id are required'
+
     # template the commands to run on the container for syncing files. At this point
     # task.stores is Dict[str, storage_utils.Storage] which is (dst, storage_obj_src)
     # first we iterate through storage_mounts and then file_mounts.
@@ -150,10 +155,35 @@ def create_pod_spec(task: 'konduktor.Task') -> Dict[str, Any]:
                     f'though specified by `tailscale.secret_name`: {err}'
                 )
 
-    assert task.resources is not None, 'Task resources are required'
-    assert task.resources.cpus is not None, 'Task resources cpus are required'
-    assert task.resources.memory is not None, 'Task resources memory are required'
-    assert task.resources.image_id is not None, 'Task resources image_id are required'
+    enable_ssh = config.get_nested(('ssh', 'enable'), False)
+    secret_name = None
+    if enable_ssh:
+        private_key_path, public_key_path = authentication.get_or_generate_keys()
+        with (
+            open(private_key_path, 'rb') as private_key_file,
+            open(public_key_path, 'rb') as public_key_file,
+        ):
+            private_key, public_key = private_key_file.read(), public_key_file.read()
+            user_hash = common_utils.get_user_hash()
+            context = kubernetes_utils.get_current_kube_config_context_name()
+            namespace = kubernetes_utils.get_kube_config_context_namespace(
+                context_name=context
+            )
+            secret_name = f'konduktor-ssh-keys-{user_hash}'
+            ok, result = kubernetes_utils.set_secret(
+                secret_name=secret_name,
+                namespace=namespace,
+                context=context,
+                data={
+                    'PUBKEY': base64.b64encode(public_key).decode(),
+                    'PRIVKEY': base64.b64encode(private_key).decode(),
+                },
+            )
+            if not ok:
+                raise exceptions.CreateSecretError(
+                    f'Failed to set k8s secret {secret_name}: \n{result}'
+                )
+
     with tempfile.NamedTemporaryFile() as temp:
         common_utils.fill_template(
             'pod.yaml.j2',
@@ -166,6 +196,7 @@ def create_pod_spec(task: 'konduktor.Task') -> Dict[str, Any]:
                 'master_addr': master_addr,
                 'num_nodes': task.num_nodes,
                 'job_name': task.name,  # append timestamp and user id here?
+                'setup_cmd': task.setup or '',
                 'run_cmd': task.run,
                 'node_hostnames': node_hostnames,
                 'accelerator_type': accelerator_type,
@@ -176,6 +207,9 @@ def create_pod_spec(task: 'konduktor.Task') -> Dict[str, Any]:
                 'user': common_utils.get_cleaned_username(),
                 # Tailscale credentials
                 'tailscale_secret': tailscale_secret,
+                # SSH
+                'enable_ssh': enable_ssh,
+                'secret_name': secret_name,
             },
             temp.name,
         )
@@ -183,6 +217,13 @@ def create_pod_spec(task: 'konduktor.Task') -> Dict[str, Any]:
         # merge with `~/.konduktor/config.yaml``
         kubernetes_utils.combine_pod_config_fields(temp.name, pod_config)
         pod_config = common_utils.read_yaml(temp.name)
+
+        for env_var in pod_config['kubernetes']['pod_config']['spec']['containers'][0][
+            'env'
+        ]:
+            if env_var['name'] in task.envs:
+                env_var['value'] = task.envs.pop(env_var['name'])
+
         for k, v in task.envs.items():
             pod_config['kubernetes']['pod_config']['spec']['containers'][0][
                 'env'
@@ -221,6 +262,7 @@ def create_jobset(
                 'user': common_utils.get_cleaned_username(),
                 'accelerator_type': accelerator_type,
                 'num_accelerators': num_accelerators,
+                'completions': task.resources.get_completions(),
                 **_JOBSET_METADATA_LABELS,
             },
             temp.name,

konduktor/data/aws/s3.py

@@ -1037,8 +1037,11 @@ class S3Store(storage_utils.AbstractStore):
             secret_name=cls._AWS_SECRET_NAME,
             namespace=namespace,
             context=context,
-            secret_key=cls._AWS_CREDENTIALS_KEY,
-            secret_value=base64_utils.zip_base64encode(credentials_files),
+            data={
+                cls._AWS_CREDENTIALS_KEY: base64_utils.zip_base64encode(
+                    credentials_files
+                )
+            },
         )
         if not ok:
             logger.error(f'Failed to set AWS credentials in k8s secret: \n{result}')

konduktor/data/gcp/gcs.py

@@ -891,8 +891,11 @@ class GcsStore(storage_utils.AbstractStore):
             secret_name=cls._GCP_SECRET_NAME,
             namespace=namespace,
             context=context,
-            secret_key=cls._GCP_CREDENTIALS_KEY,
-            secret_value=base64_utils.zip_base64encode(credentials_files),
+            data={
+                cls._GCP_CREDENTIALS_KEY: base64_utils.zip_base64encode(
+                    credentials_files
+                )
+            },
         )
         if not ok:
             logger.error(f'Failed to set GCP credentials in k8s secret: \n{result}')

konduktor/resource.py

@@ -49,6 +49,7 @@ class Resources:
         image_id: Union[str, None] = None,
         disk_size: Optional[int] = None,
         labels: Optional[Dict[str, str]] = None,
+        job_config: Optional[Dict[str, Union[int, str]]] = None,
         # Internal use only.
         # pylint: disable=invalid-name
         _cluster_config_overrides: Optional[Dict[str, Any]] = None,
@@ -91,6 +92,7 @@ class Resources:
             instance tags. On GCP, labels map to instance labels. On
             Kubernetes, labels map to pod labels. On other clouds, labels are
             not supported and will be ignored.
+          job_config: the configuration of the job spec
         Raises:
             ValueError: if some attributes are invalid.
             exceptions.NoCloudAccessError: if no public cloud is enabled.
@@ -122,6 +124,7 @@ class Resources:
         self._set_cpus(cpus)
         self._set_memory(memory)
         self._set_accelerators(accelerators)
+        self.job_config = job_config
 
         # TODO: move these out of init to prevent repeated calls.
         self._try_validate_cpus_mem()
@@ -382,6 +385,11 @@ class Resources:
             accel_str = f'{accel_name}:{accel_count}'
         return accel_str
 
+    def get_completions(self) -> Optional[int]:
+        if self.job_config and self.job_config['completions']:
+            return int(self.job_config['completions'])
+        return None
+
     def copy(self, **override) -> 'Resources':
         """Returns a copy of the given Resources."""
         resources = Resources(
@@ -392,6 +400,7 @@ class Resources:
             disk_size=override.pop('disk_size', self.disk_size),
             image_id=override.pop('image_id', self.image_id),
             labels=override.pop('labels', self.labels),
+            job_config=override.pop('job_config', self.job_config),
         )
         assert len(override) == 0
         return resources
@@ -404,6 +413,13 @@ class Resources:
             config, schemas.get_resources_schema(), 'Invalid resources YAML: '
         )
 
+        if config.get('job_config', None):
+            common_utils.validate_schema(
+                config['job_config'],
+                schemas.get_job_schema(),
+                'Invalid job config YAML',
+            )
+
         def _override_resources(
             base_resource_config: Dict[str, Any], override_configs: List[Dict[str, Any]]
         ) -> List[Resources]:
@@ -446,6 +462,7 @@ class Resources:
         resources_fields['disk_size'] = config.pop('disk_size', None)
         resources_fields['image_id'] = config.pop('image_id', None)
         resources_fields['labels'] = config.pop('labels', None)
+        resources_fields['job_config'] = config.pop('job_config', None)
 
         if resources_fields['cpus'] is not None:
             resources_fields['cpus'] = str(resources_fields['cpus'])
@@ -475,4 +492,5 @@ class Resources:
         add_if_not_none('disk_size', self.disk_size)
         add_if_not_none('image_id', self.image_id)
         add_if_not_none('labels', self.labels)
+        add_if_not_none('job_config', self.job_config)
         return config

konduktor/task.py

@@ -181,8 +181,7 @@ class Task:
         """
         assert name is not None, 'Task name is required'
         self.name = name
-        if setup is not None:
-            raise ValueError('`setup` is being deprecated and not supported')
+        self.setup = setup
         self.run = run
         self.storage_mounts: Dict[str, storage_lib.Storage] = {}
         self.storage_plans: Dict[storage_lib.Storage, storage_lib.StoreType] = {}
@@ -320,6 +319,7 @@ class Task:
 
         task = Task(
             config.pop('name', None),
+            setup=config.pop('setup', None),
             run=config.pop('run', None),
             workdir=config.pop('workdir', None),
             num_nodes=config.pop('num_nodes', None),

konduktor/templates/pod.yaml.j2

@@ -1,12 +1,12 @@
 kubernetes:
   pod_config:
     metadata:
-      {% if accelerator_type %}
       labels:
         parent: trainy
-        trainy.ai/accelerator: {{ accelerator_type }}
         trainy.ai/username: {{ user }}
-      {% endif %}
+        {% if accelerator_type %}
+        trainy.ai/accelerator: {{ accelerator_type }}
+        {% endif %}
     spec:
       restartPolicy: "Never"
       # trigger this on GPU request
@@ -15,9 +15,39 @@ kubernetes:
       - key: "nvidia.com/gpu"
         operator: "Exists"
       {% endif %}
+      initContainers:
+        - name: setup-synchronizer
+          image: "alpine:3.19"
+          restartPolicy: Always
+          command: ["/bin/sh", "-c"]
+          args:
+          - |  
+            apk add --no-cache socat 
+            wget https://raw.githubusercontent.com/asaiacai/dumb_barrier/refs/heads/main/dumb_barrier.sh 
+            sh -x dumb_barrier.sh
+          volumeMounts:
+            - name: sync
+              mountPath: /tmp/konduktor
+          env:
+            - name: MASTER_ADDR
+              value: "{{ master_addr }}"
+            - name: RANK
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.annotations['batch.kubernetes.io/job-completion-index']
+            - name: WORLD_SIZE
+              value: "{{ num_nodes }}"
+            - name: MASTER_PORT
+              value: "11111"
+            - name: GO_PORT
+              value: "11112"
       containers:
         # TODO(asaiacai): should decide here whether we add the fabric interfaces/containers init etc.        
         - name: konduktor-container
+          {% if enable_ssh %}
+          ports:
+            - containerPort: 2222
+          {% endif %}
           image: {{ image_id }}
           # this is set during jobset definition since we need to know the jobset
           # name and number of nodes to set all the environment variables correctly here
@@ -56,6 +86,18 @@ kubernetes:
               fieldRef:
                 fieldPath: metadata.uid
           {% endif %}
+          {% if enable_ssh %}
+          - name: KONDUKTOR_SSHPUB
+            valueFrom:
+              secretKeyRef:
+                name: {{ secret_name }}
+                key: PUBKEY
+          - name: KONDUKTOR_SSHPRIV
+            valueFrom:
+              secretKeyRef:
+                name: {{ secret_name }}
+                key: PRIVKEY
+          {% endif %}
           # these are for compatibility with skypilot
           - name: SKYPILOT_NODE_IPS
             value: "{{ node_hostnames }}"
@@ -70,6 +112,8 @@ kubernetes:
           volumeMounts:
           - name: shared-memory
             mountPath: /dev/shm
+          - name: sync
+            mountPath: /tmp/konduktor
           {% for secret_type, secret_name in mount_secrets.items() %}
           - name: {{ secret_type }}-secret
             mountPath: /run/konduktor/{{ secret_type }}-secret
@@ -89,22 +133,22 @@ kubernetes:
 
 
               PACKAGES="";
-              {% if 'rsync' in run_cmd %}
+              {% if 'rsync' in run_cmd or 'rsync' in setup_cmd %}
               PACKAGES="$PACKAGES rsync";
               {% endif %}
-              {% if 'curl' in run_cmd or tailscale_secret %}
+              {% if 'curl' in run_cmd or 'curl' in setup_cmd or tailscale_secret %}
               PACKAGES="$PACKAGES curl";
               {% endif %}
               {% if 'gs' in mount_secrets or 's3' in mount_secrets %}
               PACKAGES="$PACKAGES unzip wget";
               {% endif %}
-              {% if 'git' in run_cmd %}
+              {% if 'git' in run_cmd or 'git' in setup_cmd %}
               PACKAGES="$PACKAGES git";
               {% endif %}
 
               if [ ! -z "${PACKAGES}" ]; then
                 # Run apt update, install missing packages
-                DEBIAN_FRONTEND=noninteractive $(prefix_cmd) apt-get update > ~/.konduktor/tmp/apt-update.log 2>&1 || \
+                DEBIAN_FRONTEND=noninteractive $(prefix_cmd) apt-get update 2>&1 | tee -a ~/.konduktor/tmp/apt-update.log 2>&1 || \
                 $(prefix_cmd) echo "Warning: apt-get update failed. Continuing anyway..." >> ~/.konduktor/tmp/apt-update.log
               fi
               
@@ -125,22 +169,112 @@ kubernetes:
               done;
               if [ ! -z "$INSTALL_FIRST" ]; then
                 $(prefix_cmd) echo "Installing core packages: $INSTALL_FIRST";
-                DEBIAN_FRONTEND=noninteractive $(prefix_cmd) apt-get install -y $INSTALL_FIRST >> ~/.konduktor/tmp/apt-install.log;
+                DEBIAN_FRONTEND=noninteractive $(prefix_cmd) apt-get install -y $INSTALL_FIRST 2>&1 | tee -a ~/.konduktor/tmp/apt-install.log;
               fi;
 
               if [ ! -z "$MISSING_PACKAGES" ]; then
                 $(prefix_cmd) echo "Installing missing packages: $MISSING_PACKAGES";
-                DEBIAN_FRONTEND=noninteractive $(prefix_cmd) apt-get install -y $MISSING_PACKAGES >> ~/.konduktor/tmp/apt-install.log;
+                DEBIAN_FRONTEND=noninteractive $(prefix_cmd) apt-get install -y $MISSING_PACKAGES 2>&1 | tee -a ~/.konduktor/tmp/apt-install.log;
               fi;
               end_epoch=$(date +%s);
 
+              {% if enable_ssh %}
+
+              function InstallSSH {
+                export DEBIAN_FRONTEND=noninteractive
+                export TZ=Etc/UTC
+                if service sshd status > /dev/null 2>&1; then
+                  $(prefix_cmd) echo "OpenSSH server is already started."
+                  return
+                fi
+                # Check if OpenSSH server is already installed
+                if ! command -v sshd &> /dev/null; then
+                    $(prefix_cmd) echo "OpenSSH server is not installed. Installing..."
+
+                    DEBIAN_FRONTEND=noninteractive $(prefix_cmd) apt update
+                    DEBIAN_FRONTEND=noninteractive $(prefix_cmd) apt install -y openssh-server
+
+                    $(prefix_cmd) echo "OpenSSH server installation complete."
+                else
+                    $(prefix_cmd) echo "OpenSSH server is already installed."
+                fi
+
+                # Set root password if SSHKEY is provided
+                # Enable root login in SSH configuration
+                $(prefix_cmd) sed -i '/^#PermitRootLogin/c\PermitRootLogin yes' /etc/ssh/sshd_config
+                $(prefix_cmd) sed -i '/^PermitRootLogin/c\PermitRootLogin yes' /etc/ssh/sshd_config
+                $(prefix_cmd) echo "Root login is enabled."
+
+                # Create the .ssh directory and authorized_keys file if they don't exist
+                if [ ! -d "$HOME/.ssh" ]; then
+                    $(prefix_cmd) mkdir -p "$HOME/.ssh"
+                    $(prefix_cmd) chmod 0700 "$HOME/.ssh"
+                    $(prefix_cmd) echo "Directory $HOME/.ssh created."
+                fi
+                if [ ! -f "$HOME/.ssh/authorized_keys" ]; then
+                    $(prefix_cmd) touch "$HOME/.ssh/authorized_keys"
+                    $(prefix_cmd) chmod 0600 "$HOME/.ssh/authorized_keys"
+                    $(prefix_cmd) echo "File $HOME/.ssh/authorized_keys created."
+                fi
+                # Check if the public key is not already present in authorized_keys
+                if ! grep -q "${KONDUKTOR_SSHPUB}" "$HOME/.ssh/authorized_keys"; then
+                    # Append the public key to authorized_keys
+                    $(prefix_cmd) echo "${KONDUKTOR_SSHPUB}" >> "$HOME/.ssh/authorized_keys"
+                    $(prefix_cmd) echo "Public key added."
+                fi
+                if [ ! -f "$HOME/.ssh/konduktor-key" ]; then
+                    # create the private key to authorized_keys
+                    $(prefix_cmd) touch "$HOME/.ssh/konduktor-key"
+                    $(prefix_cmd) chmod 0600 "$HOME/.ssh/konduktor-key"
+                    $(prefix_cmd) echo "${KONDUKTOR_SSHPRIV}" >> "$HOME/.ssh/konduktor-key"
+                    $(prefix_cmd) echo "private key added."
+                fi
+                if [ ! -f "$HOME/.ssh/config" ]; then
+                    # create the private key to authorized_keys
+                    $(prefix_cmd) touch "$HOME/.ssh/config"
+                    $(prefix_cmd) chmod 0600 "$HOME/.ssh/config"
+                    $(prefix_cmd) printf '\nHost *\n    StrictHostKeyChecking no\n' >> "$HOME/.ssh/config"
+                    $(prefix_cmd) echo "ssh config set"
+                fi
+
+                # turn off PAM to fix sshd login issue
+                $(prefix_cmd) sed -i 's/UsePAM yes/UsePAM no/' /etc/ssh/sshd_config
+
+                # set default port to 2222
+                $(prefix_cmd) sed -i 's/#Port 22/Port 2222/' /etc/ssh/sshd_config
+
+                echo "Exposing ENV variables"
+                env -0 | awk -v RS='\0' '
+                  {
+                    gsub(/\\/,"\\\\");      # escape existing backslashes first
+                    gsub(/"/,"\\\"");       # escape any double quotes
+                    gsub(/\n/,"\\n");       # turn real newlines into the two characters \n
+                    sub(/=/,"=\"");         # open the value-quoting
+                    print $0 "\"";          # close the quote and add a newline record separator
+                  }
+                ' > /etc/environment
+                echo "set -a; source /etc/environment; set +a;" >> /root/.bashrc
+
+                $(prefix_cmd) mkdir /run/sshd
+                $(prefix_cmd) chmod 0755 /run/sshd
+
+                $(prefix_cmd) service ssh start
+                $(prefix_cmd) echo "sshd service started"
+              }
+
+              InstallSSH
+              {% endif %}
+
               {% if tailscale_secret %}
-              if ! command -v tailscale >/dev/null 2>&1; then
-                export TS_HOSTNAME=$(echo "$POD_NAME" | sed 's/-[^-]*$//')
-                $(prefix_cmd) curl -fsSL https://tailscale.com/install.sh | DEBIAN_FRONTEND=noninteractive $(prefix_cmd) sh >> ~/.konduktor/tmp/tailscale-install.log
-                $(prefix_cmd) tailscaled --tun=userspace-networking >/dev/null 2>&1 &
-                $(prefix_cmd) tailscale up --auth-key=${TS_AUTHKEY} --ssh --hostname=${TS_HOSTNAME} >/dev/null 2>&1
-              fi
+              function InstallTailscale {
+                if ! command -v tailscale >/dev/null 2>&1; then
+                  export TS_HOSTNAME=$(echo "$POD_NAME" | sed 's/-[^-]*$//')
+                  $(prefix_cmd) curl -fsSL https://tailscale.com/install.sh | DEBIAN_FRONTEND=noninteractive $(prefix_cmd) sh 2>&1 | tee -a ~/.konduktor/tmp/tailscale-install.log
+                  $(prefix_cmd) tailscaled --tun=userspace-networking >/dev/null 2>&1 &
+                  $(prefix_cmd) tailscale up --auth-key=${TS_AUTHKEY} --ssh --hostname=${TS_HOSTNAME} >/dev/null 2>&1
+                fi
+              }
+              InstallTailscale &
               {% endif %}
               end_epoch=$(date +%s);
 
@@ -182,14 +316,42 @@ kubernetes:
               end_setup_time=$((end_epoch - start_setup));
               ulimit -Sc 0 && ulimit -Hc 0
               $(prefix_cmd) echo "===== KONDUKTOR: Initialization took $end_setup_time seconds ====="
-              # run task  
+              set +eo pipefail
               $(prefix_cmd) cd {{ remote_workdir }}
+              {% if setup_cmd %}
+              # setup task
+              $(prefix_cmd) echo "===== KONDUKTOR: Running setup ======="
+              {{ setup_cmd | indent( width=14 ) }}
+              {% endif %}
+
+              # synchronize workers before executing `run`
+              set -e
+              touch "/tmp/konduktor/SETUP"
+              # TODO(asaiacai): should we make this value tuneable for users?
+              TIMEOUT=3600
+              start_sync=$(date +%s);
+              DEADLINE=$(( $(date +%s) + TIMEOUT ))
+
+              echo "[KONDUKTOR: main] Waiting for workers to synchronize"
+              while [ ! -f "/tmp/konduktor/READY" ]; do
+                  if [ "$(date +%s)" -ge "$DEADLINE" ]; then
+                      echo "[KONDUKTOR: main] ERROR: Timed out after 2 minutes of waiting for worker synchronization"
+                      exit 1
+                  fi
+                  sleep 0.5
+              done
+              echo "[KONDUKTOR: main] All workers have joined"
+              end_sync=$(date +%s);
+              echo "[KONDUKTOR: main] Synchronization took $((end_sync - start_sync)) seconds"
               set +eo pipefail
+              # run task  
+              $(prefix_cmd) cd {{ remote_workdir }}
               $(prefix_cmd) echo "===== KONDUKTOR: Running task ====="
               start_epoch=$(date +%s);
               {{ run_cmd | indent( width=14 ) }}
               end_epoch=$(date +%s);
               exit_code=$?
+              set +ex
               $(prefix_cmd) echo "===== KONDUKTOR: Running task took $((end_epoch - start_epoch)) seconds and finished with exit code: $exit_code ====="
               exit $exit_code
           resources:
@@ -216,6 +378,8 @@ kubernetes:
         emptyDir:
           medium: "Memory"
           sizeLimit: 4Gi
+      - name: sync
+        emptyDir: {}
       {% for secret_type, secret_name in mount_secrets.items() %}
       - name: {{ secret_type }}-secret
         secret:

konduktor/utils/exceptions.py

@@ -130,6 +130,10 @@ class CommandError(Exception):
     pass
 
 
+class CreateSecretError(Exception):
+    pass
+
+
 class MissingSecretError(Exception):
     pass
 

konduktor/utils/kubernetes_utils.py

@@ -54,7 +54,7 @@ NO_ACCELERATOR_HELP_MESSAGE = (
     '(e.g. `nvidia.com/gpu` are setup correctly. '
 )
 
-_K8S_CLIENT_LOCK_PATH = '~/.konduktor/k8s_client.lock'
+_K8S_CLIENT_LOCK_PATH = os.path.expanduser('~/.konduktor/k8s_client.lock')
 _K8s_CLIENT_LOCK = filelock.FileLock(_K8S_CLIENT_LOCK_PATH)
 
 logger = logging.get_logger(__name__)
@@ -578,11 +578,14 @@ def set_secret(
     secret_name: str,
     namespace: str,
     context: Optional[str],
-    secret_key: str,
-    secret_value: str,
+    data: Dict[str, str],
 ) -> Tuple[bool, Optional[str]]:
     """
     Create/update a secret in a namespace. Values are encoded to base64.
+    `secret` must be base64 encoded ie
+    ```
+    base64.b64encode(secret).decode()
+    ```
     """
     with _K8s_CLIENT_LOCK:
         secret_exists, response = check_secret_exists(
@@ -598,7 +601,7 @@ def set_secret(
         secret = kubernetes.client.V1Secret(
             metadata=kubernetes.client.V1ObjectMeta(**secret_metadata),
             type='Opaque',
-            data={secret_key: secret_value},
+            data=data,
         )
 
         try:

konduktor/utils/loki_utils.py

@@ -67,7 +67,8 @@ def tail_loki_logs_ws(
         logger.debug(f'Loki URL: {loki_url}')
         params = {
             'query': urllib.parse.quote(
-                f'{{k8s_job_name="{job_name}-workers-0"}} '
+                r'{' + f'k8s_job_name="{job_name}-workers-0",'
+                r' k8s_container_name="konduktor-container"} '
                 f' | batch_kubernetes_io_job_completion_index = `{worker_id}`'
             ),
             'limit': num_logs,

konduktor/utils/schemas.py

@@ -87,6 +87,7 @@ def _get_single_resources_schema():
             '_cluster_config_overrides': {
                 'type': 'object',
             },
+            'job_config': {'type': 'object'},
         },
     }
 
@@ -153,8 +154,6 @@ def get_resources_schema():
                 'items': multi_resources_schema,
             },
         },
-        # Avoid job_recovery and spot_recovery being present at the same time.
-        **_check_not_both_fields_present('job_recovery', 'spot_recovery'),
     }
 
 
@@ -337,84 +336,6 @@ def get_cluster_schema():
     }
 
 
-_NETWORK_CONFIG_SCHEMA = {
-    'vpc_name': {
-        'oneOf': [
-            {
-                'type': 'string',
-            },
-            {
-                'type': 'null',
-            },
-        ],
-    },
-    'use_internal_ips': {
-        'type': 'boolean',
-    },
-    'ssh_proxy_command': {
-        'oneOf': [
-            {
-                'type': 'string',
-            },
-            {
-                'type': 'null',
-            },
-            {
-                'type': 'object',
-                'required': [],
-                'additionalProperties': {
-                    'anyOf': [
-                        {'type': 'string'},
-                        {'type': 'null'},
-                    ]
-                },
-            },
-        ]
-    },
-}
-
-_LABELS_SCHEMA = {
-    # Deprecated: 'instance_tags' is replaced by 'labels'. Keeping for backward
-    # compatibility. Will be removed after 0.8.0.
-    'instance_tags': {
-        'type': 'object',
-        'required': [],
-        'additionalProperties': {
-            'type': 'string',
-        },
-    },
-    'labels': {
-        'type': 'object',
-        'required': [],
-        'additionalProperties': {
-            'type': 'string',
-        },
-    },
-}
-
-_PRORPERTY_NAME_OR_CLUSTER_NAME_TO_PROPERTY = {
-    'oneOf': [
-        {'type': 'string'},
-        {
-            # A list of single-element dict to pretain the
-            # order.
-            # Example:
-            #  property_name:
-            #    - my-cluster1-*: my-property-1
-            #    - my-cluster2-*: my-property-2
-            #    - "*"": my-property-3
-            'type': 'array',
-            'items': {
-                'type': 'object',
-                'additionalProperties': {'type': 'string'},
-                'maxProperties': 1,
-                'minProperties': 1,
-            },
-        },
-    ]
-}
-
-
 class RemoteIdentityOptions(enum.Enum):
     """Enum for remote identity types.
 
@@ -454,9 +375,8 @@ _REMOTE_IDENTITY_SCHEMA_KUBERNETES = {
 
 def get_storage_schema():
     # pylint: disable=import-outside-toplevel
-    from knoduktor.registry import registry
-
     from konduktor.data import storage
+    from konduktor.registry import registry
 
     return {
         '$schema': 'https://json-schema.org/draft/2020-12/schema',
@@ -496,6 +416,21 @@ def get_storage_schema():
     }
 
 
+def get_job_schema():
+    """Schema for a job spec, which is defined under resources."""
+    return {
+        '$schema': 'https://json-schema.org/draft/2020-12/schema',
+        'type': 'object',
+        'required': [],
+        'additionalProperties': False,
+        'properties': {
+            'completions': {
+                'type': 'number',
+            },
+        },
+    }
+
+
 def get_config_schema():
     # pylint: disable=import-outside-toplevel
     from konduktor.data import registry
@@ -574,6 +509,17 @@ def get_config_schema():
         },
     }
 
+    ssh_configs = {
+        'type': 'object',
+        'required': [],
+        'additionalProperties': False,
+        'properties': {
+            'enable': {
+                'type': 'boolean',
+            },
+        },
+    }
+
     for cloud, config in cloud_configs.items():
         if cloud == 'kubernetes':
             config['properties'].update(_REMOTE_IDENTITY_SCHEMA_KUBERNETES)
@@ -589,6 +535,7 @@ def get_config_schema():
             'nvidia_gpus': gpu_configs,
             'allowed_clouds': allowed_clouds,
             'tailscale': tailscale_configs,
+            'ssh': ssh_configs,
             **cloud_configs,
         },
     }

{konduktor_nightly-0.1.0.dev20250512104920.dist-info → konduktor_nightly-0.1.0.dev20250514104854.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: konduktor-nightly
-Version: 0.1.0.dev20250512104920
+Version: 0.1.0.dev20250514104854
 Summary: GPU Cluster Health Management
 Author: Andrew Aikawa
 Author-email: asai@berkeley.edu

{konduktor_nightly-0.1.0.dev20250512104920.dist-info → konduktor_nightly-0.1.0.dev20250514104854.dist-info}/RECORD

@@ -1,12 +1,13 @@
-konduktor/__init__.py,sha256=FMLSHfj8jo627JKX7MyYFZH99jqw1EKMC7kpIpPgAPU,1540
+konduktor/__init__.py,sha256=ODIjRocI7dlxyMFYh5S2VYJai-4MWm98MSyiSzaGDbA,1540
 konduktor/adaptors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 konduktor/adaptors/aws.py,sha256=s47Ra-GaqCQibzVfmD0pmwEWHif1EGO5opMbwkLxTCU,8244
 konduktor/adaptors/common.py,sha256=uTdpKvgBSwYMmynx9wR5kiZQyTrdaw9ZI4KH6Z2E5Hw,4296
 konduktor/adaptors/gcp.py,sha256=ierTF4z7vwpJ9BsC7LSiwv4uLcjGXscwZOwQrddr2vM,4102
+konduktor/authentication.py,sha256=jhw_virbyvrY_9WZqOXX3LyOP_HkpfmTssWUMxJVyBg,4564
 konduktor/backends/__init__.py,sha256=1Q6sqqdeMYarpTX_U-QVywJYf7idiUTRsyP-E4BQSOw,129
 konduktor/backends/backend.py,sha256=qh0bp94lzoTYZkzyQv2-CVrB5l91FkG2vclXg24UFC0,2910
-konduktor/backends/jobset.py,sha256=veptYGXtk-ugWxBsBV5SnqI4rGKOlGfm_N3wApvNhSQ,8326
-konduktor/backends/jobset_utils.py,sha256=B0N0sx-pWF9_CDeuSXAU4nm3ZIwroyVcq6aUAlNZZRs,18376
+konduktor/backends/jobset.py,sha256=UdhwAuZODLMbLY51Y2zOBsh6wg4Pb84oHVvUKzx3Z2w,8434
+konduktor/backends/jobset_utils.py,sha256=4vMYOhTENfBL9khzFuj69-Vy4g0sBkUpXX-1bfPnVys,20054
 konduktor/check.py,sha256=JennyWoaqSKhdyfUldd266KwVXTPJpcYQa4EED4a_BA,7569
 konduktor/cli.py,sha256=Ii9-2mrc-1f2ksLasA-xRb-JnEi_9ZeCXZ3lJ1GG8H8,23515
 konduktor/config.py,sha256=J50JxC6MsXMnlrJPXdDUMr38C89xvOO7mR8KJ6fyils,15520
@@ -50,12 +51,12 @@ konduktor/dashboard/frontend/server.js,sha256=jcp6_Ww9YJD3uKY07jR3KMlAM6n1QZdxZn
 konduktor/dashboard/frontend/tailwind.config.js,sha256=fCnc48wvioIDOe5ldQ_6RE7F76cP7aU7pDrxBPJx-Fk,366
 konduktor/data/__init__.py,sha256=KMR2i3E9YcIpiIuCxtRdS7BQ1w2vUAbbve7agziJrLo,213
 konduktor/data/aws/__init__.py,sha256=_6zWfNNAK1QGgyKqg_yPYWcXlnffchyvIMErYa6tw_U,331
-konduktor/data/aws/s3.py,sha256=2hvbgZ9NuwXY88blxfdjSbONSXcyWF0CtheDZkMYorQ,48296
+konduktor/data/aws/s3.py,sha256=T4FnCxilNp35bsgmE7j5O3j15FVbgWRdUH8YFXCiwSw,48335
 konduktor/data/constants.py,sha256=yXVEoTI2we1xOjVSU-bjRCQCLpVvpEvJ0GedXvSwEfw,127
 konduktor/data/data_utils.py,sha256=yrnu8_cY63TXqfWfFG3yqY2w_tE9UQK9jIQAFQCDVg0,9668
 konduktor/data/gcp/__init__.py,sha256=rlQxACBC_Vu36mdgPyJgUy4mGc_6Nt_a96JAuaPz2pQ,489
 konduktor/data/gcp/constants.py,sha256=dMfOiFccM8O6rUi9kClJcbvw1K1VnS1JzzQk3apq8ho,1483
-konduktor/data/gcp/gcs.py,sha256=kDbUzf8ALYzsw_G3sBRn_enQ8fjI-UKV0jeWuFZiULA,42018
+konduktor/data/gcp/gcs.py,sha256=nqhCvQuGpHFPoxT5SKgxL25KtZuSg377Nh1bICiQwlc,42057
 konduktor/data/gcp/utils.py,sha256=FJQcMXZqtMIzjZ98b3lTTc0UbdPUKTDLsOsfJaaH5-s,214
 konduktor/data/registry.py,sha256=CUbMsN_Q17Pf4wRHkqZrycErEjTP7cLEdgcfwVGcEpc,696
 konduktor/data/storage.py,sha256=SDKRWDd7PCT9ytuz4cH0CejZj5QmWG_EZhUMVoTzWsc,35308
@@ -67,10 +68,10 @@ konduktor/manifests/controller_deployment.yaml,sha256=6p3oSLkEVONZsvKZGqVop0Dhn4
 konduktor/manifests/dashboard_deployment.yaml,sha256=xJLd4FbPMAosI0fIv5_8y7dV9bw0Vsf81l-w4MB_aU8,2837
 konduktor/manifests/dmesg_daemonset.yaml,sha256=pSWt7YOeTYjS0l0iki1fvHOs7MhY-sH-RQfVW6JJyno,1391
 konduktor/manifests/pod_cleanup_controller.yaml,sha256=hziL1Ka1kCAEL9R7Tjvpb80iw1vcq9_3gwHCu75Bi0A,3939
-konduktor/resource.py,sha256=68z8gC8Ivqktwv0R6ylMn9ZNocgkcRT0yIRGGKOdwcM,18491
-konduktor/task.py,sha256=Vu1TzYtLvSBz-HyHY2gsM2cMcUhMNQu44L3CWmYRXKE,35232
+konduktor/resource.py,sha256=w2PdIrmQaJWA-GLSmVBcg4lxwuxvPulz35_YSKa5o24,19254
+konduktor/task.py,sha256=2JOHRS4JE2FdN-M3qZKhII1hkUvWHbreNtkf30Mo2lo,35196
 konduktor/templates/jobset.yaml.j2,sha256=onYiHtXAgk-XBtji994hPu_g0hxnLzvmfxwjbdKdeZc,960
-konduktor/templates/pod.yaml.j2,sha256=s3eECjLevUWR-zvyeI8WjQWxQYJh_AMk1tdQVGNXpEM,9835
+konduktor/templates/pod.yaml.j2,sha256=AobmCpvXRnZuQjfT000vN72Nuk380CCmWPHC_BVrUhM,17161
 konduktor/usage/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 konduktor/usage/constants.py,sha256=gCL8afIHZhO0dcxbJGpESE9sCC1cBSbeRnQ8GwNOY4M,612
 konduktor/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -80,18 +81,18 @@ konduktor/utils/base64_utils.py,sha256=mF-Tw98mFRG70YE4w6s9feuQSCYZHOb8YatBZwMug
 konduktor/utils/common_utils.py,sha256=F5x7k4AdBB44u8PYRkaugORnZKnK3JLqGn1jHOKgUYo,14960
 konduktor/utils/constants.py,sha256=1DneiTR21lvKUcWdBGwC4I4fD4uPjbjLUilEnJS7rzA,216
 konduktor/utils/env_options.py,sha256=T41Slzf4Mzl-n45CGXXqdy2fCrYhPNZQ7RP5vmnN4xc,2258
-konduktor/utils/exceptions.py,sha256=IHyaP5ERZpPvWZeKWV3MVTyKsxo2Fq-13nhI0PRNQzk,6629
+konduktor/utils/exceptions.py,sha256=bOYHk3SHR3XO__p9bPwVPz8g9k6weIRxGRFNkyzgZOA,6676
 konduktor/utils/kubernetes_enums.py,sha256=SabUueF6Bpzbpa57gyH5VB65xla2N9l8CZmAeYTfGmM,176
-konduktor/utils/kubernetes_utils.py,sha256=ivFVh90Gez19_JD5U4bgCO5zNtQUflF0hJsM5nZLj8A,23864
+konduktor/utils/kubernetes_utils.py,sha256=1MZHwU4vy-exA4TA5_oTiV-zm1A2ayfeA0T_75DMFM8,23937
 konduktor/utils/log_utils.py,sha256=lgHCq4OdtJNfbpso-uYGONUCVNsUrUkUWjROarsHt6s,9897
-konduktor/utils/loki_utils.py,sha256=ND1pbbbFhLhLKw3870j44LpR_9MB0EkDJSs5K7nWdY4,3473
+konduktor/utils/loki_utils.py,sha256=h2ZvZQr1nE_wXXsKsGMjhG2s2MXknNd4icydTR_ruKU,3539
 konduktor/utils/rich_utils.py,sha256=kdjNe6S2LlpOxyzhFHqMzCz7g4ROC4e7TPWgcbRsrQE,3577
-konduktor/utils/schemas.py,sha256=_VCWnsSgyP3u5cpACEmJeuqcy5mzu_fr0McHyZdiXd8,17757
+konduktor/utils/schemas.py,sha256=2fHsTi3t9q3LXqOPrcpkmPsMbaoJBnuJstd6ULmDiUo,16455
 konduktor/utils/subprocess_utils.py,sha256=WoFkoFhGecPR8-rF8WJxbIe-YtV94LXz9UG64SDhCY4,9448
 konduktor/utils/ux_utils.py,sha256=czCwiS1bDqgeKtzAJctczpLwFZzAse7WuozdvzEFYJ4,7437
 konduktor/utils/validator.py,sha256=tgBghVyedyzGx84-U2Qfoh_cJBE3oUk9gclMW90ORks,691
-konduktor_nightly-0.1.0.dev20250512104920.dist-info/LICENSE,sha256=MuuqTZbHvmqXR_aNKAXzggdV45ANd3wQ5YI7tnpZhm0,6586
-konduktor_nightly-0.1.0.dev20250512104920.dist-info/METADATA,sha256=Ega2hwBGc5bYNmkhPKQdMzLWCO1ru_UcvdczmGosZBE,4366
-konduktor_nightly-0.1.0.dev20250512104920.dist-info/WHEEL,sha256=XbeZDeTWKc1w7CSIyre5aMDU_-PohRwTQceYnisIYYY,88
-konduktor_nightly-0.1.0.dev20250512104920.dist-info/entry_points.txt,sha256=k3nG5wDFIJhNqsZWrHk4d0irIB2Ns9s47cjRWYsTCT8,48
-konduktor_nightly-0.1.0.dev20250512104920.dist-info/RECORD,,
+konduktor_nightly-0.1.0.dev20250514104854.dist-info/LICENSE,sha256=MuuqTZbHvmqXR_aNKAXzggdV45ANd3wQ5YI7tnpZhm0,6586
+konduktor_nightly-0.1.0.dev20250514104854.dist-info/METADATA,sha256=ErMUfOWxJPkbM0by718uNtBgUv-2w7m5sqFzJ_cHc64,4366
+konduktor_nightly-0.1.0.dev20250514104854.dist-info/WHEEL,sha256=XbeZDeTWKc1w7CSIyre5aMDU_-PohRwTQceYnisIYYY,88
+konduktor_nightly-0.1.0.dev20250514104854.dist-info/entry_points.txt,sha256=k3nG5wDFIJhNqsZWrHk4d0irIB2Ns9s47cjRWYsTCT8,48
+konduktor_nightly-0.1.0.dev20250514104854.dist-info/RECORD,,