konduktor-nightly 0.1.0.dev20251030104830__tar.gz → 0.1.0.dev20251211105235__tar.gz

{konduktor_nightly-0.1.0.dev20251030104830 → konduktor_nightly-0.1.0.dev20251211105235}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: konduktor-nightly
-Version: 0.1.0.dev20251030104830
+Version: 0.1.0.dev20251211105235
 Summary: GPU Cluster Health Management
 Author: Andrew Aikawa
 Author-email: asai@berkeley.edu
@@ -29,6 +29,7 @@ Requires-Dist: prettytable (>=3.12.0,<4.0.0)
 Requires-Dist: psutil (>=7.0.0,<8.0.0)
 Requires-Dist: python-dotenv (>=1.0.1,<2.0.0)
 Requires-Dist: rich (>=13.9.4,<14.0.0)
+Requires-Dist: sniffio (>=1.3,<2.0)
 Requires-Dist: websockets (>=15.0.1,<16.0.0)
 Description-Content-Type: text/markdown
 

{konduktor_nightly-0.1.0.dev20251030104830 → konduktor_nightly-0.1.0.dev20251211105235}/konduktor/__init__.py

@@ -11,7 +11,7 @@ from konduktor.task import Task
 __all__ = ['launch', 'Resources', 'Task', 'Serving']
 
 # Replaced with the current commit when building the wheels.
-_KONDUKTOR_COMMIT_SHA = '2d0baca8b432c156afdb9cc49add78305fe0ed21'
+_KONDUKTOR_COMMIT_SHA = '421390595e3a1b9f263e790323deae61d94da231'
 os.makedirs(os.path.expanduser('~/.konduktor'), exist_ok=True)
 
 
@@ -45,5 +45,5 @@ def _get_git_commit():
 
 
 __commit__ = _get_git_commit()
-__version__ = '1.0.0.dev0.1.0.dev20251030104830'
+__version__ = '1.0.0.dev0.1.0.dev20251211105235'
 __root_dir__ = os.path.dirname(os.path.abspath(__file__))

{konduktor_nightly-0.1.0.dev20251030104830 → konduktor_nightly-0.1.0.dev20251211105235}/konduktor/backends/constants.py

@@ -18,3 +18,4 @@ STOP_USERNAME_LABEL = 'trainy.ai/stop-username'
 SECRET_BASENAME_LABEL = 'trainy.ai/secret-basename'
 SECRET_KIND_LABEL = 'trainy.ai/secret-kind'
 SECRET_OWNER_LABEL = 'trainy.ai/secret-owner'
+ROOT_NAME = 'trainy.ai/root-name'

{konduktor_nightly-0.1.0.dev20251030104830 → konduktor_nightly-0.1.0.dev20251211105235}/konduktor/backends/deployment.py

@@ -54,8 +54,19 @@ def _wait_for_all_ready(namespace: str, name: str):
         except ApiException:
             services_map = {}
 
-        autoscaler = deployment_utils.get_autoscaler(namespace, name)
-        autoscalers_map = {name: autoscaler} if autoscaler else {}
+        autoscalers_map = {}
+        try:
+            autoscaler_obj = deployment_utils.get_autoscaler(namespace, name)
+            if autoscaler_obj:
+                # detect aibrix vs general from deployment labels
+                labels = (deployment.metadata.labels or {}) if deployment else {}
+                is_aibrix = deployment_utils.AIBRIX_NAME_LABEL in labels
+                if is_aibrix:
+                    autoscalers_map[name] = {'kpa': autoscaler_obj}
+                else:
+                    autoscalers_map[name] = {'hpa': autoscaler_obj}
+        except ApiException:
+            pass
 
         status = deployment_utils.get_model_status(
             name, deployments_map, services_map, autoscalers_map

{konduktor_nightly-0.1.0.dev20251030104830 → konduktor_nightly-0.1.0.dev20251211105235}/konduktor/backends/deployment_utils.py

@@ -998,13 +998,13 @@ def get_envoy_external_ip() -> Optional[str]:
 
 
 def get_ingress_nginx_external_ip() -> Optional[str]:
-    """Get the external IP of the ingress-nginx-controller LoadBalancer."""
+    """Get the external IP of the keda-ingress-nginx-controller LoadBalancer."""
     context = kubernetes_utils.get_current_kube_config_context_name()
     core_api = kube_client.core_api(context=context)
     try:
-        # Look for ingress-nginx-controller service in keda namespace
+        # Look for keda-ingress-nginx-controller service in keda namespace
         service = core_api.read_namespaced_service(
-            name='ingress-nginx-controller', namespace='keda'
+            name='keda-ingress-nginx-controller', namespace='keda'
         )
         if service.spec.type == 'LoadBalancer':
             ingress = service.status.load_balancer.ingress

{konduktor_nightly-0.1.0.dev20251030104830 → konduktor_nightly-0.1.0.dev20251211105235}/konduktor/backends/jobset_utils.py

@@ -449,7 +449,8 @@ def _format_timestamp(timestamp: str) -> str:
 
 
 def _get_job_start_time(job: Dict[str, Any]) -> str:
-    for condition in job['status'].get('conditions', []):
+    status = job.get('status', {})
+    for condition in status.get('conditions', []):
         if condition['reason'] == 'ResumeJobs':
             return condition.get('lastTransitionTime', '')
     return '-'

{konduktor_nightly-0.1.0.dev20251030104830 → konduktor_nightly-0.1.0.dev20251211105235}/konduktor/backends/pod_utils.py

@@ -28,6 +28,8 @@ if typing.TYPE_CHECKING:
 logger = logging.get_logger(__name__)
 
 _RUN_DURATION_ANNOTATION_KEY = 'kueue.x-k8s.io/maxRunDurationSeconds'
+# Use a large default (7 days) to mimic "infinite" runtime.
+_DEFAULT_MAX_RUN_DURATION_SECONDS = 604800
 
 
 def create_pod_spec(task: 'konduktor.Task') -> Dict[str, Any]:
@@ -153,7 +155,9 @@ def create_pod_spec(task: 'konduktor.Task') -> Dict[str, Any]:
     git_ssh_secret_name = None
     env_secret_envs = []
     default_secrets = []
+    basename_by_k8s: Dict[str, str] = {}
 
+    # only get own secrets
     user_hash = common_utils.get_user_hash()
     label_selector = f'{backend_constants.SECRET_OWNER_LABEL}={user_hash}'
     user_secrets = kubernetes_utils.list_secrets(
@@ -162,19 +166,36 @@ def create_pod_spec(task: 'konduktor.Task') -> Dict[str, Any]:
 
     for secret in user_secrets:
         kind = kubernetes_utils.get_secret_kind(secret)
+
+        # incase the user modified their secret to have no key:value data
+        if secret.data is None:
+            secret.data = {}
+
+        # fill the map for *all* secrets we see
+        k8s_name = secret.metadata.name
+        lbls = secret.metadata.labels or {}
+        base = lbls.get(
+            backend_constants.SECRET_BASENAME_LABEL,
+            # fallback: strip trailing "-<something>" once if present
+            k8s_name.rsplit('-', 1)[0] if '-' in k8s_name else k8s_name,
+        )
+        basename_by_k8s[k8s_name] = base
+
         if kind == 'git-ssh' and git_ssh_secret_name is None:
             git_ssh_secret_name = secret.metadata.name
         elif kind == 'env':
             env_secret_name = secret.metadata.name
-            key = next(iter(secret.data))
-            env_secret_envs.append(
-                {
-                    'name': key,
-                    'valueFrom': {
-                        'secretKeyRef': {'name': env_secret_name, 'key': key}
-                    },
-                }
-            )
+            # iterate ALL keys, not just one (ex. if user made a multi-key env secret)
+            for key, _ in secret.data.items():
+                # wire the env var to read its value from a k8s secret
+                env_secret_envs.append(
+                    {
+                        'name': key,
+                        'valueFrom': {
+                            'secretKeyRef': {'name': env_secret_name, 'key': key}
+                        },
+                    }
+                )
         elif kind == 'default':
             default_secret_name = secret.metadata.name
             basename = secret.metadata.labels.get(
@@ -184,6 +205,22 @@ def create_pod_spec(task: 'konduktor.Task') -> Dict[str, Any]:
                 {'k8s_name': default_secret_name, 'mount_name': basename}
             )
 
+    # Check if the task references KONDUKTOR_DEFAULT_SECRETS and that it exists
+    uses_default_secret_var = (
+        'KONDUKTOR_DEFAULT_SECRETS' in (task.run or '')
+        or 'KONDUKTOR_DEFAULT_SECRETS' in (task.setup or '')
+        or '/konduktor/default-secrets/' in (task.run or '')
+        or '/konduktor/default-secrets/' in (task.setup or '')
+    )
+    if uses_default_secret_var and not default_secrets:
+        raise exceptions.MissingSecretError(
+            f'Task references KONDUKTOR_DEFAULT_SECRETS or '
+            f'/konduktor/default-secrets but '
+            f'user {common_utils.get_cleaned_username()} '
+            f'has no default secrets. Paths like '
+            f'$KONDUKTOR_DEFAULT_SECRETS/<secret_name>/... will not exist.'
+        )
+
     # Inject --served-model-name, --host, and --port into serving run command
     if task.serving and task.run and 'vllm.entrypoints.openai.api_server' in task.run:
         if '--served-model-name' and '--host' and '--port' not in task.run:
@@ -262,31 +299,111 @@ def create_pod_spec(task: 'konduktor.Task') -> Dict[str, Any]:
             },
             temp.name,
         )
+
+        # Capture the template env names BEFORE user config is merged
+        pod_config_template = common_utils.read_yaml(temp.name)
+        tmpl_envs = pod_config_template['kubernetes']['pod_config']['spec'][
+            'containers'
+        ][0].get('env', [])
+        tmpl_env_names = {e['name'] for e in tmpl_envs}
+
         pod_config = common_utils.read_yaml(temp.name)
-        # merge with `~/.konduktor/config.yaml``
+        # merge with `~/.konduktor/config.yaml`` (config.yaml overrides template)
         kubernetes_utils.combine_pod_config_fields(temp.name, pod_config)
         pod_config = common_utils.read_yaml(temp.name)
 
-    # Priority order: task.envs > secret envs > existing pod_config envs
-    existing_envs = pod_config['kubernetes']['pod_config']['spec']['containers'][0].get(
+    # Find what came from user config (appeared after combine, not in template)
+    premerge_envs = pod_config['kubernetes']['pod_config']['spec']['containers'][0].get(
         'env', []
     )
-    env_map = {env['name']: env for env in existing_envs}
+    premerge_names = {e['name'] for e in premerge_envs}
+    config_env_names0 = premerge_names - tmpl_env_names
+
+    # Build final env list
+    env_map = {env['name']: env for env in premerge_envs}
 
-    # Inject secret envs
+    # Inject secret envs (env secrets override config.yaml)
     for env in env_secret_envs:
         env_map[env['name']] = env
 
-    # Inject task.envs
+    # Inject task envs
+    # CLI+task.yaml overrides everything else
+    # CLI already overrode task.yaml in other code
     for k, v in task.envs.items():
         env_map[k] = {'name': k, 'value': v}
 
-    # Replace the container's env section with the merged and prioritized map
-    pod_config['kubernetes']['pod_config']['spec']['containers'][0]['env'] = list(
-        env_map.values()
+    final_envs_list = list(env_map.values())
+    pod_config['kubernetes']['pod_config']['spec']['containers'][0]['env'] = (
+        final_envs_list
     )
+    container = pod_config['kubernetes']['pod_config']['spec']['containers'][0]
+    final_envs = container['env']
+    final_names = {e['name'] for e in final_envs}
+
     logger.debug(f'rendered pod spec: \n\t{json.dumps(pod_config, indent=2)}')
 
+    # 1) Get secret envs actually used in the final env list
+    secret_details = sorted(
+        (e['name'], e['valueFrom']['secretKeyRef']['name'])
+        for e in final_envs
+        if isinstance(e, dict)
+        and e.get('valueFrom', {})
+        and e['valueFrom'].get('secretKeyRef')
+    )
+    secret_names = [n for n, _ in secret_details]
+
+    # 2) Get task-sourced (CLI+task.yaml) envs actually used in the final env list
+    task_all_names = sorted(
+        n
+        for n in (task.envs or {}).keys()
+        if n in final_names and n not in secret_names
+    )
+
+    # 3) Get Config.yaml envs actually used in the final env list
+    config_names = sorted(
+        n
+        for n in config_env_names0
+        if n in final_names and n not in secret_names and n not in task_all_names
+    )
+
+    # 4) Get other envs (template/system) actually used in the final env list
+    other_names = sorted(
+        final_names - set(secret_names) - set(task_all_names) - set(config_names)
+    )
+
+    # Export helper envs for the startup script (names only)
+    def _append_helper(name: str, values):
+        container['env'].append({'name': name, 'value': ','.join(values)})
+
+    # to show user basenames of k8s secrets instead of actual
+    # k8s secret names (which have added suffixes)
+    secret_map_pairs = [
+        f'{var}={basename_by_k8s.get(secret_k8s, secret_k8s)}'
+        for (var, secret_k8s) in secret_details
+    ]
+
+    # Priority order: CLI > task.yaml > env secret > config > template/system
+    _append_helper(
+        'KONDUKTOR_ENV_SECRETS_HOPEFULLY_NO_NAME_COLLISION',
+        secret_names,
+    )
+    _append_helper(
+        'KONDUKTOR_ENV_SECRETS_MAP_HOPEFULLY_NO_NAME_COLLISION',
+        secret_map_pairs,
+    )
+    _append_helper(
+        'KONDUKTOR_ENV_TASK_ALL_HOPEFULLY_NO_NAME_COLLISION',
+        task_all_names,
+    )
+    _append_helper(
+        'KONDUKTOR_ENV_CONFIG_HOPEFULLY_NO_NAME_COLLISION',
+        config_names,
+    )
+    _append_helper(
+        'KONDUKTOR_ENV_OTHER_HOPEFULLY_NO_NAME_COLLISION',
+        other_names,
+    )
+
     # validate pod spec using json schema
     try:
         validator.validate_pod_spec(pod_config['kubernetes']['pod_config']['spec'])
@@ -356,18 +473,21 @@ def inject_jobset_metadata(jobset_spec: Dict[str, Any], task: 'konduktor.Task')
         jobset_spec: The JobSet spec dictionary to modify
         task: The task object containing resource information
     """
-    # Add max run duration annotation
-    assert task.resources is not None and task.resources.labels is not None
-    maxRunDurationSeconds = task.resources.labels.get('maxRunDurationSeconds', None)
-    if not maxRunDurationSeconds:
-        raise ValueError('maxRunDurationSeconds is required')
-    jobset_spec['jobset']['metadata']['annotations'][_RUN_DURATION_ANNOTATION_KEY] = (
-        str(maxRunDurationSeconds)
+    assert task.resources is not None, 'Task resources are required'
+    labels = task.resources.labels or {}
+
+    # Add max run duration annotation, defaulting to a practically infinite value.
+    maxRunDurationSeconds = labels.get('maxRunDurationSeconds')
+    metadata = jobset_spec['jobset']['metadata']
+    metadata.setdefault('annotations', {})[_RUN_DURATION_ANNOTATION_KEY] = str(
+        maxRunDurationSeconds
+        if maxRunDurationSeconds is not None
+        else _DEFAULT_MAX_RUN_DURATION_SECONDS
     )
 
     # Inject resource labels into JobSet metadata.
-    if task.resources and task.resources.labels:
-        jobset_spec['jobset']['metadata']['labels'].update(task.resources.labels)
+    if labels:
+        jobset_spec['jobset']['metadata']['labels'].update(labels)
 
 
 def merge_pod_into_jobset_template(