konduktor-nightly 0.1.0.dev20251029104819__py3-none-any.whl → 0.1.0.dev20251031104711__py3-none-any.whl

konduktor/__init__.py

@@ -11,7 +11,7 @@ from konduktor.task import Task
 __all__ = ['launch', 'Resources', 'Task', 'Serving']
 
 # Replaced with the current commit when building the wheels.
-_KONDUKTOR_COMMIT_SHA = '2d0baca8b432c156afdb9cc49add78305fe0ed21'
+_KONDUKTOR_COMMIT_SHA = 'd5fddf4e144c4887227e1c6943c70bcd72d364d5'
 os.makedirs(os.path.expanduser('~/.konduktor'), exist_ok=True)
 
 
@@ -45,5 +45,5 @@ def _get_git_commit():
 
 
 __commit__ = _get_git_commit()
-__version__ = '1.0.0.dev0.1.0.dev20251029104819'
+__version__ = '1.0.0.dev0.1.0.dev20251031104711'
 __root_dir__ = os.path.dirname(os.path.abspath(__file__))

konduktor/backends/constants.py

@@ -18,3 +18,4 @@ STOP_USERNAME_LABEL = 'trainy.ai/stop-username'
 SECRET_BASENAME_LABEL = 'trainy.ai/secret-basename'
 SECRET_KIND_LABEL = 'trainy.ai/secret-kind'
 SECRET_OWNER_LABEL = 'trainy.ai/secret-owner'
+ROOT_NAME = 'trainy.ai/root-name'

konduktor/backends/deployment.py

@@ -54,8 +54,19 @@ def _wait_for_all_ready(namespace: str, name: str):
         except ApiException:
             services_map = {}
 
-        autoscaler = deployment_utils.get_autoscaler(namespace, name)
-        autoscalers_map = {name: autoscaler} if autoscaler else {}
+        autoscalers_map = {}
+        try:
+            autoscaler_obj = deployment_utils.get_autoscaler(namespace, name)
+            if autoscaler_obj:
+                # detect aibrix vs general from deployment labels
+                labels = (deployment.metadata.labels or {}) if deployment else {}
+                is_aibrix = deployment_utils.AIBRIX_NAME_LABEL in labels
+                if is_aibrix:
+                    autoscalers_map[name] = {'kpa': autoscaler_obj}
+                else:
+                    autoscalers_map[name] = {'hpa': autoscaler_obj}
+        except ApiException:
+            pass
 
         status = deployment_utils.get_model_status(
             name, deployments_map, services_map, autoscalers_map

konduktor/backends/deployment_utils.py

@@ -998,13 +998,13 @@ def get_envoy_external_ip() -> Optional[str]:
 
 
 def get_ingress_nginx_external_ip() -> Optional[str]:
-    """Get the external IP of the ingress-nginx-controller LoadBalancer."""
+    """Get the external IP of the keda-ingress-nginx-controller LoadBalancer."""
     context = kubernetes_utils.get_current_kube_config_context_name()
     core_api = kube_client.core_api(context=context)
     try:
-        # Look for ingress-nginx-controller service in keda namespace
+        # Look for keda-ingress-nginx-controller service in keda namespace
         service = core_api.read_namespaced_service(
-            name='ingress-nginx-controller', namespace='keda'
+            name='keda-ingress-nginx-controller', namespace='keda'
         )
         if service.spec.type == 'LoadBalancer':
             ingress = service.status.load_balancer.ingress

konduktor/backends/jobset_utils.py

@@ -449,7 +449,8 @@ def _format_timestamp(timestamp: str) -> str:
 
 
 def _get_job_start_time(job: Dict[str, Any]) -> str:
-    for condition in job['status'].get('conditions', []):
+    status = job.get('status', {})
+    for condition in status.get('conditions', []):
         if condition['reason'] == 'ResumeJobs':
             return condition.get('lastTransitionTime', '')
     return '-'

konduktor/backends/pod_utils.py

@@ -153,7 +153,9 @@ def create_pod_spec(task: 'konduktor.Task') -> Dict[str, Any]:
     git_ssh_secret_name = None
     env_secret_envs = []
     default_secrets = []
+    basename_by_k8s: Dict[str, str] = {}
 
+    # only get own secrets
     user_hash = common_utils.get_user_hash()
     label_selector = f'{backend_constants.SECRET_OWNER_LABEL}={user_hash}'
     user_secrets = kubernetes_utils.list_secrets(
@@ -162,19 +164,36 @@ def create_pod_spec(task: 'konduktor.Task') -> Dict[str, Any]:
 
     for secret in user_secrets:
         kind = kubernetes_utils.get_secret_kind(secret)
+
+        # incase the user modified their secret to have no key:value data
+        if secret.data is None:
+            secret.data = {}
+
+        # fill the map for *all* secrets we see
+        k8s_name = secret.metadata.name
+        lbls = secret.metadata.labels or {}
+        base = lbls.get(
+            backend_constants.SECRET_BASENAME_LABEL,
+            # fallback: strip trailing "-<something>" once if present
+            k8s_name.rsplit('-', 1)[0] if '-' in k8s_name else k8s_name,
+        )
+        basename_by_k8s[k8s_name] = base
+
         if kind == 'git-ssh' and git_ssh_secret_name is None:
             git_ssh_secret_name = secret.metadata.name
         elif kind == 'env':
             env_secret_name = secret.metadata.name
-            key = next(iter(secret.data))
-            env_secret_envs.append(
-                {
-                    'name': key,
-                    'valueFrom': {
-                        'secretKeyRef': {'name': env_secret_name, 'key': key}
-                    },
-                }
-            )
+            # iterate ALL keys, not just one (ex. if user made a multi-key env secret)
+            for key, _ in secret.data.items():
+                # wire the env var to read its value from a k8s secret
+                env_secret_envs.append(
+                    {
+                        'name': key,
+                        'valueFrom': {
+                            'secretKeyRef': {'name': env_secret_name, 'key': key}
+                        },
+                    }
+                )
         elif kind == 'default':
             default_secret_name = secret.metadata.name
             basename = secret.metadata.labels.get(
@@ -184,6 +203,22 @@ def create_pod_spec(task: 'konduktor.Task') -> Dict[str, Any]:
                 {'k8s_name': default_secret_name, 'mount_name': basename}
             )
 
+    # Check if the task references KONDUKTOR_DEFAULT_SECRETS and that it exists
+    uses_default_secret_var = (
+        'KONDUKTOR_DEFAULT_SECRETS' in (task.run or '')
+        or 'KONDUKTOR_DEFAULT_SECRETS' in (task.setup or '')
+        or '/konduktor/default-secrets/' in (task.run or '')
+        or '/konduktor/default-secrets/' in (task.setup or '')
+    )
+    if uses_default_secret_var and not default_secrets:
+        raise exceptions.MissingSecretError(
+            f'Task references KONDUKTOR_DEFAULT_SECRETS or '
+            f'/konduktor/default-secrets but '
+            f'user {common_utils.get_cleaned_username()} '
+            f'has no default secrets. Paths like '
+            f'$KONDUKTOR_DEFAULT_SECRETS/<secret_name>/... will not exist.'
+        )
+
     # Inject --served-model-name, --host, and --port into serving run command
     if task.serving and task.run and 'vllm.entrypoints.openai.api_server' in task.run:
         if '--served-model-name' and '--host' and '--port' not in task.run:
@@ -262,31 +297,111 @@ def create_pod_spec(task: 'konduktor.Task') -> Dict[str, Any]:
             },
             temp.name,
         )
+
+        # Capture the template env names BEFORE user config is merged
+        pod_config_template = common_utils.read_yaml(temp.name)
+        tmpl_envs = pod_config_template['kubernetes']['pod_config']['spec'][
+            'containers'
+        ][0].get('env', [])
+        tmpl_env_names = {e['name'] for e in tmpl_envs}
+
         pod_config = common_utils.read_yaml(temp.name)
-        # merge with `~/.konduktor/config.yaml``
+        # merge with `~/.konduktor/config.yaml`` (config.yaml overrides template)
         kubernetes_utils.combine_pod_config_fields(temp.name, pod_config)
         pod_config = common_utils.read_yaml(temp.name)
 
-    # Priority order: task.envs > secret envs > existing pod_config envs
-    existing_envs = pod_config['kubernetes']['pod_config']['spec']['containers'][0].get(
+    # Find what came from user config (appeared after combine, not in template)
+    premerge_envs = pod_config['kubernetes']['pod_config']['spec']['containers'][0].get(
         'env', []
     )
-    env_map = {env['name']: env for env in existing_envs}
+    premerge_names = {e['name'] for e in premerge_envs}
+    config_env_names0 = premerge_names - tmpl_env_names
 
-    # Inject secret envs
+    # Build final env list
+    env_map = {env['name']: env for env in premerge_envs}
+
+    # Inject secret envs (env secrets override config.yaml)
     for env in env_secret_envs:
         env_map[env['name']] = env
 
-    # Inject task.envs
+    # Inject task envs
+    # CLI+task.yaml overrides everything else
+    # CLI already overrode task.yaml in other code
     for k, v in task.envs.items():
         env_map[k] = {'name': k, 'value': v}
 
-    # Replace the container's env section with the merged and prioritized map
-    pod_config['kubernetes']['pod_config']['spec']['containers'][0]['env'] = list(
-        env_map.values()
+    final_envs_list = list(env_map.values())
+    pod_config['kubernetes']['pod_config']['spec']['containers'][0]['env'] = (
+        final_envs_list
     )
+    container = pod_config['kubernetes']['pod_config']['spec']['containers'][0]
+    final_envs = container['env']
+    final_names = {e['name'] for e in final_envs}
+
     logger.debug(f'rendered pod spec: \n\t{json.dumps(pod_config, indent=2)}')
 
+    # 1) Get secret envs actually used in the final env list
+    secret_details = sorted(
+        (e['name'], e['valueFrom']['secretKeyRef']['name'])
+        for e in final_envs
+        if isinstance(e, dict)
+        and e.get('valueFrom', {})
+        and e['valueFrom'].get('secretKeyRef')
+    )
+    secret_names = [n for n, _ in secret_details]
+
+    # 2) Get task-sourced (CLI+task.yaml) envs actually used in the final env list
+    task_all_names = sorted(
+        n
+        for n in (task.envs or {}).keys()
+        if n in final_names and n not in secret_names
+    )
+
+    # 3) Get Config.yaml envs actually used in the final env list
+    config_names = sorted(
+        n
+        for n in config_env_names0
+        if n in final_names and n not in secret_names and n not in task_all_names
+    )
+
+    # 4) Get other envs (template/system) actually used in the final env list
+    other_names = sorted(
+        final_names - set(secret_names) - set(task_all_names) - set(config_names)
+    )
+
+    # Export helper envs for the startup script (names only)
+    def _append_helper(name: str, values):
+        container['env'].append({'name': name, 'value': ','.join(values)})
+
+    # to show user basenames of k8s secrets instead of actual
+    # k8s secret names (which have added suffixes)
+    secret_map_pairs = [
+        f'{var}={basename_by_k8s.get(secret_k8s, secret_k8s)}'
+        for (var, secret_k8s) in secret_details
+    ]
+
+    # Priority order: CLI > task.yaml > env secret > config > template/system
+    _append_helper(
+        'KONDUKTOR_ENV_SECRETS_HOPEFULLY_NO_NAME_COLLISION',
+        secret_names,
+    )
+    _append_helper(
+        'KONDUKTOR_ENV_SECRETS_MAP_HOPEFULLY_NO_NAME_COLLISION',
+        secret_map_pairs,
+    )
+    _append_helper(
+        'KONDUKTOR_ENV_TASK_ALL_HOPEFULLY_NO_NAME_COLLISION',
+        task_all_names,
+    )
+    _append_helper(
+        'KONDUKTOR_ENV_CONFIG_HOPEFULLY_NO_NAME_COLLISION',
+        config_names,
+    )
+    _append_helper(
+        'KONDUKTOR_ENV_OTHER_HOPEFULLY_NO_NAME_COLLISION',
+        other_names,
+    )
+
     # validate pod spec using json schema
     try:
         validator.validate_pod_spec(pod_config['kubernetes']['pod_config']['spec'])

konduktor/cli.py

@@ -54,6 +54,7 @@ from konduktor import logging
 from konduktor.backends import constants as backend_constants
 from konduktor.backends import deployment_utils, jobset_utils
 from konduktor.utils import (
+    base64_utils,
     common_utils,
     kubernetes_utils,
     log_utils,
@@ -1491,12 +1492,21 @@ def create(kind, from_file, from_directory, inline, name):
     data = {}
     if from_directory:
         click.echo(f'Creating secret from directory: {from_directory}')
-        base_path = pathlib.Path(from_directory)
-        for path in base_path.rglob('*'):
-            if path.is_file():
-                rel_path = path.relative_to(base_path)
-                with open(path, 'rb') as f:
-                    data[str(rel_path)] = b64encode(f.read()).decode()
+        # Use ABSOLUTE directory path so the top-level folder name is preserved
+        base_dir_abs = os.path.abspath(os.path.expanduser(from_directory))
+        if not os.path.isdir(base_dir_abs):
+            raise click.BadParameter(
+                f"--from-directory {from_directory} doesn't exist or is not a directory"
+            )
+        # Ensure there is at least one file inside
+        if not any(p.is_file() for p in pathlib.Path(base_dir_abs).rglob('*')):
+            raise click.BadParameter(f'--from-directory {from_directory} is empty.')
+
+        # Zip + base64 the WHOLE directory (this preserves the inner structure)
+        archive_b64 = base64_utils.zip_base64encode([base_dir_abs])
+
+        # Store as a single key; pod will unzip to the expanded path
+        data = {'payload.zip': archive_b64}
     elif from_file:
         click.echo(f'Creating secret from file: {from_file}')
         key = os.path.basename(from_file)

konduktor/manifests/aibrix-setup.yaml

@@ -34,6 +34,34 @@ metadata:
   name: aibrix-activator
 ---
 apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: aibrix-activator
+  namespace: aibrix-activator
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: aibrix-activator
+rules:
+- apiGroups: ["apps"]
+  resources: ["deployments"]
+  verbs: ["get", "list", "watch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: aibrix-activator
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: aibrix-activator
+subjects:
+- kind: ServiceAccount
+  name: aibrix-activator
+  namespace: aibrix-activator
+---
+apiVersion: v1
 kind: ConfigMap
 metadata:
   name: activator-code
@@ -44,16 +72,26 @@ data:
     from collections import defaultdict, deque
     from fastapi import FastAPI, Request
     from fastapi.responses import PlainTextResponse, JSONResponse
+    import asyncio
+    from kubernetes import client, config
 
     NAMESPACE = os.getenv("NAMESPACE", "default")
     WINDOW_SEC = int(os.getenv("WINDOW_SEC", "30"))        # demand lookback
     CAPACITY_RPS = float(os.getenv("CAPACITY_RPS", "1.0")) # per-replica capacity
     MIN_WAKE = int(os.getenv("MIN_REPLICA_ON_WAKE", "1"))
     MAX_REPLICAS = int(os.getenv("MAX_REPLICAS", "8"))
+    CLEANUP_INTERVAL = int(os.getenv("CLEANUP_INTERVAL", "300"))  # 5 minutes
 
     app = FastAPI()
     events = defaultdict(deque)  # key=(ns,model) -> deque[timestamps]
 
+    # Initialize Kubernetes client
+    try:
+        config.load_incluster_config()
+        k8s_apps_v1 = client.AppsV1Api()
+    except:
+        k8s_apps_v1 = None
+
     def _prune(q, now):
         while q and now - q[0] > WINDOW_SEC: q.popleft()
 
@@ -89,6 +127,48 @@ data:
             pass
         return None
 
+    def _get_existing_deployments():
+        """Get list of existing Aibrix deployments from Kubernetes"""
+        if not k8s_apps_v1:
+            return set()
+        try:
+            deployments = k8s_apps_v1.list_namespaced_deployment(
+                namespace=NAMESPACE,
+                label_selector="model.aibrix.ai/name"
+            )
+            return {d.metadata.name for d in deployments.items}
+        except Exception:
+            return set()
+
+    def _cleanup_stale_entries():
+        """Remove entries for deployments that no longer exist"""
+        if not k8s_apps_v1:
+            return
+        try:
+            existing_deployments = _get_existing_deployments()
+            # Remove entries for deployments that no longer exist
+            keys_to_remove = []
+            for (ns, model) in list(events.keys()):
+                if ns == NAMESPACE and model not in existing_deployments:
+                    keys_to_remove.append((ns, model))
+            
+            for key in keys_to_remove:
+                del events[key]
+                print(f"Cleaned up stale entry for deployment: {key[1]}")
+        except Exception as e:
+            print(f"Error during cleanup: {e}")
+
+    async def _cleanup_task():
+        """Background task to periodically clean up stale entries"""
+        while True:
+            await asyncio.sleep(CLEANUP_INTERVAL)
+            _cleanup_stale_entries()
+
+    @app.on_event("startup")
+    async def startup_event():
+        """Start background cleanup task"""
+        asyncio.create_task(_cleanup_task())
+
     # Mirror endpoints (same as your API paths); quick 204 response
     @app.post("/v1/completions")
     @app.post("/v1/chat/completions")
@@ -108,6 +188,37 @@ data:
             _bump(NAMESPACE, model)
         return JSONResponse({"ok": True}, status_code=204)
 
+    # Prometheus-friendly aggregate endpoint: export ALL (ns, model)
+    @app.get("/metrics", response_class=PlainTextResponse)
+    async def metrics_all():
+        lines = []
+        # Idiomatic names
+        lines.append("# HELP vllm_deployment_replicas Number of suggested replicas.")
+        lines.append("# TYPE vllm_deployment_replicas gauge")
+        lines.append("# HELP vllm_observed_rps Incoming requests per second.")
+        lines.append("# TYPE vllm_observed_rps gauge")
+        now = time.time()
+        for (ns, model), q in list(events.items()):
+            _prune(q, now)
+            rps = len(q) / max(WINDOW_SEC, 1)
+            d = _desired(ns, model)
+            lines.append(f'vllm_deployment_replicas{{namespace="{ns}",model_name="{model}"}} {d}')
+            lines.append(f'vllm_observed_rps{{namespace="{ns}",model_name="{model}"}} {rps:.6f}')
+        # (Optional) keep legacy names with colons for back-compat
+        lines.append("# HELP vllm:deployment_replicas Number of suggested replicas.")
+        lines.append("# TYPE vllm:deployment_replicas gauge")
+        lines.append("# HELP vllm:observed_rps Incoming requests per second.")
+        lines.append("# TYPE vllm:observed_rps gauge")
+        now = time.time()
+        for (ns, model), q in list(events.items()):
+            _prune(q, now)
+            rps = len(q) / max(WINDOW_SEC, 1)
+            d = _desired(ns, model)
+            lines.append(f'vllm:deployment_replicas{{namespace="{ns}",model_name="{model}"}} {d}')
+            lines.append(f'vllm:observed_rps{{namespace="{ns}",model_name="{model}"}} {rps:.6f}')
+        return "\n".join(lines) + "\n"
+
+
     # Metrics for KPA and Debugging
     @app.get("/metrics/{ns}/{model}", response_class=PlainTextResponse)
     async def metrics(ns: str, model: str):
@@ -142,7 +253,7 @@ spec:
         command: ["bash","-lc"]
         args:
           - |
-            pip install fastapi uvicorn >/dev/null && \
+            pip install fastapi uvicorn kubernetes >/dev/null && \
             uvicorn activator:app --host 0.0.0.0 --port 8080
         env:
         - { name: NAMESPACE, value: "default" }
@@ -150,10 +261,12 @@ spec:
         - { name: CAPACITY_RPS, value: "1.0" }
         - { name: MIN_REPLICA_ON_WAKE, value: "1" }
         - { name: MAX_REPLICAS, value: "8" }
+        - { name: CLEANUP_INTERVAL, value: "300" }
         ports: [{containerPort: 8080}]
         volumeMounts:
         - { name: code, mountPath: /app/activator.py, subPath: activator.py }
         workingDir: /app
+      serviceAccountName: aibrix-activator
       volumes:
       - name: code
         configMap: { name: activator-code }
@@ -163,6 +276,13 @@ kind: Service
 metadata:
   name: aibrix-activator
   namespace: aibrix-activator
+  annotations:
+    prometheus.io/scrape: "true"
+    prometheus.io/port: "8080"
+    prometheus.io/path: "/metrics"
+  labels:
+    app: aibrix-activator
+    prometheus-discovery: "true"
 spec:
   selector: { app: aibrix-activator }
   ports:
@@ -172,6 +292,42 @@ spec:
     protocol: TCP
   type: ClusterIP
 ---
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: aibrix-activator
+  namespace: prometheus
+  labels:
+    app: aibrix-activator
+spec:
+  selector:
+    matchLabels:
+      app: aibrix-activator
+  namespaceSelector:
+    matchNames:
+    - aibrix-activator
+  endpoints:
+  - port: http
+    path: /metrics
+---
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: vllm-deployments
+  namespace: prometheus
+  labels:
+    app: vllm-deployments
+spec:
+  selector:
+    matchLabels:
+      prometheus-discovery: "true"
+  namespaceSelector:
+    matchNames:
+    - default
+  endpoints:
+  - port: serve
+    path: /metrics
+---
 apiVersion: gateway.networking.k8s.io/v1beta1
 kind: ReferenceGrant
 metadata:

konduktor/manifests/apoxy-setup2.yaml

@@ -59,7 +59,7 @@ metadata:
   name: UNIQUE-TEMPNAME-backend2
 spec:
   endpoints:
-    - fqdn: ingress-nginx-controller.keda.UNIQUE-TEMPNAME.tun.apoxy.net
+    - fqdn: keda-ingress-nginx-controller.keda.UNIQUE-TEMPNAME.tun.apoxy.net
 ---
 # HTTPRoute for general deployments
 apiVersion: gateway.apoxy.dev/v1

konduktor/templates/deployment.yaml.j2

@@ -41,11 +41,9 @@ metadata:
     {{ deployment_name_label }}: "{{ name }}"
     {{ deployment_user_label }}: "{{ user }}"
     trainy.ai/has-autoscaler: "{{ autoscaler }}"
-  {% if not general %}
   annotations:
     prometheus.io/scrape: "true"
-    prometheus.io/port: "8080"
-  {% endif %}
+    prometheus.io/port: "9000"
   name: {{ name }}
   namespace: default
 spec:
@@ -142,6 +140,10 @@ apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
   name: {{ name }}-ingress
+  labels:
+    {{ deployment_name_label }}: "{{ name }}"
+    {{ deployment_user_label }}: "{{ user }}"
+    trainy.ai/konduktor-managed: "true"
   annotations:
     nginx.ingress.kubernetes.io/use-regex: "true"
     nginx.ingress.kubernetes.io/rewrite-target: /$1

konduktor/templates/pod.yaml.j2

@@ -28,16 +28,21 @@ kubernetes:
       containers:
         # TODO(asaiacai): should decide here whether we add the fabric interfaces/containers init etc.        
         - name: konduktor-container
-          {% if enable_ssh %}
+          {% if enable_ssh or serving %}
           ports:
+            {% if enable_ssh %}
             - name: ssh
               containerPort: {{ konduktor_ssh_port }}
+            {% endif %}
+
+            {% if serving %}
+            - name: serving
+              containerPort: {{ ports }}
+            {% endif %}
           {% endif %}
-          {% if serving %}
-          ports:
-            - containerPort: {{ ports }}
+
+          {% if serving and probe %}
           # TODO (ryan): allow modification of thresholds and timings
-          {% if probe %}
           livenessProbe:
             httpGet:
               path: {{ probe }}
@@ -68,7 +73,6 @@ kubernetes:
             successThreshold: 1
             timeoutSeconds: 1
           {% endif %}
-          {% endif %}
           image: {{ image_id }}
           # this is set during jobset definition since we need to know the jobset
           # name and number of nodes to set all the environment variables correctly here
@@ -134,6 +138,8 @@ kubernetes:
           {% if default_secrets %}
           - name: KONDUKTOR_DEFAULT_SECRETS
             value: "/konduktor/default-secrets"
+          - name: KONDUKTOR_DEFAULT_SECRETS_EXPANDED
+            value: "/run/konduktor/expanded-default-secrets"
           {% endif %}
           # these are for compatibility with skypilot
           - name: SKYPILOT_NODE_IPS
@@ -146,6 +152,10 @@ kubernetes:
             value: "{{ num_nodes }}"
           - name: SKYPILOT_NUM_GPUS_PER_NODE
             value: "{{ num_gpus }}"
+          - name: RESTART_ATTEMPT
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.labels['jobset.sigs.k8s.io/restart-attempt']
           volumeMounts:
           - name: shared-memory
             mountPath: /dev/shm
@@ -159,6 +169,10 @@ kubernetes:
           - name: default-secret-{{ secret.mount_name }}
             mountPath: /konduktor/default-secrets/{{ secret.mount_name }}
           {% endfor %}
+          {% if default_secrets %}
+          - name: default-secrets-expanded
+            mountPath: /run/konduktor/expanded-default-secrets
+          {% endif %}
           {% if git_ssh %}
           - name: git-ssh-secret
             mountPath: /run/konduktor/git-ssh-secret
@@ -192,7 +206,7 @@ kubernetes:
               {% if 'curl' in run_cmd or 'curl' in setup_cmd or tailscale_secret %}
               PACKAGES="$PACKAGES curl";
               {% endif %}
-              {% if 'gs' in mount_secrets or 's3' in mount_secrets %}
+              {% if 'gs' in mount_secrets or 's3' in mount_secrets or default_secrets %}
               PACKAGES="$PACKAGES unzip wget";
               {% endif %}
               {% if 'git' in run_cmd or 'git' in setup_cmd %}
@@ -231,7 +245,7 @@ kubernetes:
               fi;
               end_epoch=$(date +%s);
               
-              echo "Exposing ENV variables"
+              echo "===== KONDUKTOR: Exposing ENV variables ====="
               $(prefix_cmd) env -0 | awk -v RS='\0' '
                 {
                   gsub(/\\/,"\\\\");      # escape existing backslashes first
@@ -346,8 +360,41 @@ kubernetes:
 
               $(prefix_cmd) echo "===== KONDUKTOR: Installing packages took $((end_epoch - start_epoch)) seconds ====="
 
+              $(prefix_cmd) echo "===== KONDUKTOR: Environment variable summary ====="
+              start_epoch=$(date +%s);
+
+              print_bucket () {
+                title="$1"; list="${2:-}"
+                echo "--- $title ---"
+                if [ -n "$list" ]; then
+                  echo "$list" | tr ',' '\n' | sed "s/^/[$title] /"
+                else
+                  echo "[none]"
+                fi
+              }
+
+              # Secrets: prefer detailed mapping if available
+              echo "--- env secret ---"
+              if [ -n "${KONDUKTOR_ENV_SECRETS_MAP_HOPEFULLY_NO_NAME_COLLISION:-}" ]; then
+                echo "${KONDUKTOR_ENV_SECRETS_MAP_HOPEFULLY_NO_NAME_COLLISION}" \
+                  | tr ',' '\n' \
+                  | awk -F'=' '{ printf("[secret: %s] %s\n", $2, $1) }'
+              elif [ -n "${KONDUKTOR_ENV_SECRETS_HOPEFULLY_NO_NAME_COLLISION:-}" ]; then
+                echo "${KONDUKTOR_ENV_SECRETS_HOPEFULLY_NO_NAME_COLLISION}" \
+                  | tr ',' '\n' | sed 's/^/[secret] /'
+              else
+                echo "[none]"
+              fi
+
+              print_bucket "CLI + task.yaml" "${KONDUKTOR_ENV_TASK_ALL_HOPEFULLY_NO_NAME_COLLISION}"
+              print_bucket "config.yaml"     "${KONDUKTOR_ENV_CONFIG_HOPEFULLY_NO_NAME_COLLISION}"
+              print_bucket "other"           "${KONDUKTOR_ENV_OTHER_HOPEFULLY_NO_NAME_COLLISION}"
+
+              end_epoch=$(date +%s);
+              $(prefix_cmd) echo "===== KONDUKTOR: Environment variable summary took $((end_epoch - start_epoch)) seconds ====="
+
               # unpack secrets credentials
-              $(prefix_cmd) echo "===== KONDUKTOR: Unpacking secrets credentials ====="
+              $(prefix_cmd) echo "===== KONDUKTOR: Unpacking cloud storage secret credentials ====="
               start_epoch=$(date +%s);
               mkdir -p ~/.konduktor
               mkdir -p {{ remote_workdir }}
@@ -362,12 +409,71 @@ kubernetes:
               $(prefix_cmd) unzip /run/konduktor/s3-secret/awscredentials -d ~/.aws
               {% endif %}
               {% endfor %}
+              
+              {% if default_secrets %}
+              $(prefix_cmd) echo "===== KONDUKTOR: Unpacking default secrets ====="
+              $(prefix_cmd) mkdir -p "${KONDUKTOR_DEFAULT_SECRETS_EXPANDED}"
+
+              # For each mounted default secret folder:
+              #  - if payload.zip exists, unzip it into the expanded dir
+              #  - otherwise, copy the files as-is
+              for src in "${KONDUKTOR_DEFAULT_SECRETS}"/*; do
+                [ -d "$src" ] || continue
+                name="$(basename "$src")"
+                dst="${KONDUKTOR_DEFAULT_SECRETS_EXPANDED}/${name}"
+                $(prefix_cmd) mkdir -p "$dst"
+
+                if [ -f "${src}/payload.zip" ]; then
+                  $(prefix_cmd) unzip -oq "${src}/payload.zip" -d "$dst"
+                else
+                  $(prefix_cmd) cp -a "${src}/." "$dst/"
+                fi
+              done
+
+              # Point callers to the expanded (writable) path going forward
+              export KONDUKTOR_DEFAULT_SECRETS="${KONDUKTOR_DEFAULT_SECRETS_EXPANDED}"
+              $(prefix_cmd) echo "KONDUKTOR_DEFAULT_SECRETS=${KONDUKTOR_DEFAULT_SECRETS_EXPANDED}" >> /etc/environment
+              {% endif %}
+              
               {% if git_ssh %}
               $(prefix_cmd) echo "Unpacking GIT-SSH secret"
               {% endif %}
               end_epoch=$(date +%s);
               $(prefix_cmd) echo "===== KONDUKTOR: Unpacking secrets credentials took $((end_epoch - start_epoch)) seconds ====="
 
+              $(prefix_cmd) echo "===== KONDUKTOR: Default secret summary ====="
+              start_epoch=$(date +%s)
+
+              root="${KONDUKTOR_DEFAULT_SECRETS:-}"
+              if [[ -z "$root" || ! -d "$root" ]]; then
+                $(prefix_cmd) echo "NO DEFAULT SECRETS FOUND."
+              else
+                for dir in "$root"/*; do
+                  [ -d "$dir" ] || continue
+                  name="$(basename "$dir")"
+
+                  # Pretty header that mirrors the logical mount base:
+                  $(prefix_cmd) echo "/konduktor/default-secrets/${name}:"
+
+                  # Print relative paths only; skip macOS junk and k8s secret internals
+                  (
+                    cd "$dir"
+                    out="$(find . \
+                      \( -name '.DS_Store' -o -name '__MACOSX' -o -name '..data' -o -name '..*' \) -prune -o \
+                      \( -type f -o -type l \) -print \
+                      | sed 's|^\./||' \
+                      | sort)"
+                    if [ -n "$out" ]; then
+                      printf "%s\n" "$out"
+                    fi
+                  )
+                done
+              fi
+
+              end_epoch=$(date +%s)
+              $(prefix_cmd) echo "===== KONDUKTOR: Default secret summary took $((end_epoch - start_epoch)) seconds ====="
+
+
               # sync file mounts
               {% for mkdir_command in mkdir_commands %}
               $(prefix_cmd) {{ mkdir_command }}
@@ -436,6 +542,10 @@ kubernetes:
         secret:
           secretName: {{ secret.k8s_name }}
       {% endfor %}
+      {% if default_secrets %}
+      - name: default-secrets-expanded
+        emptyDir: {}
+      {% endif %}
       {% if git_ssh %}
       - name: git-ssh-secret
         secret:

konduktor/utils/base64_utils.py

@@ -44,6 +44,8 @@ def zip_base64encode(files: List[str]) -> str:
                 else:
                     for root, _, files in os.walk(item_path):
                         for file in files:
+                            if file == '.DS_Store':
+                                continue
                             file_path = os.path.join(root, file)
                             arcname = os.path.relpath(file_path, temp_dir)
                             zipf.write(file_path, arcname)

{konduktor_nightly-0.1.0.dev20251029104819.dist-info → konduktor_nightly-0.1.0.dev20251031104711.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: konduktor-nightly
-Version: 0.1.0.dev20251029104819
+Version: 0.1.0.dev20251031104711
 Summary: GPU Cluster Health Management
 Author: Andrew Aikawa
 Author-email: asai@berkeley.edu

{konduktor_nightly-0.1.0.dev20251029104819.dist-info → konduktor_nightly-0.1.0.dev20251031104711.dist-info}/RECORD

@@ -1,4 +1,4 @@
-konduktor/__init__.py,sha256=k8KGCtWLCd6iV6SbRfrsIwz85hqDDELB3WpYd0KVMPs,1574
+konduktor/__init__.py,sha256=i1gtM3DmNjTNZ6fT3_GMPMukq_7nyxNvsOUPJSpYwXY,1574
 konduktor/adaptors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 konduktor/adaptors/aws.py,sha256=s47Ra-GaqCQibzVfmD0pmwEWHif1EGO5opMbwkLxTCU,8244
 konduktor/adaptors/common.py,sha256=ZIqzjx77PIHUwpjfAQ1uX8B2aX78YMuGj4Bppd-MdyM,4183
@@ -6,14 +6,14 @@ konduktor/adaptors/gcp.py,sha256=ierTF4z7vwpJ9BsC7LSiwv4uLcjGXscwZOwQrddr2vM,410
 konduktor/authentication.py,sha256=_mVy3eqoKohicHostFiGwG1-2ybxP-l7ouofQ0LRlCY,4570
 konduktor/backends/__init__.py,sha256=usWJ8HdZJEyg7MIsN8Zcz9rk9e2Lq5dWJ8dv6hCN3ys,199
 konduktor/backends/backend.py,sha256=qh0bp94lzoTYZkzyQv2-CVrB5l91FkG2vclXg24UFC0,2910
-konduktor/backends/constants.py,sha256=nt9G9AmFCOMwO4GuKgRQSzJJuKapOmaROp4_Y0tMF5A,732
-konduktor/backends/deployment.py,sha256=VXz0GMvFzqq8kd1-pXaNaTsEtSyMmVO2OSd-_-T9nIE,6171
-konduktor/backends/deployment_utils.py,sha256=rfVpiM0KlXma8SxA029TczrpbY-gRG3T0pj2eeARkE8,49257
+konduktor/backends/constants.py,sha256=uAU-edQ_9DNYnu6x7fwNYXIEM7KMfJMOrnG74rlZ8mY,766
+konduktor/backends/deployment.py,sha256=d0a3F7dxDbnRKIt4ZO_kQ0_vet0pZvg4bWYzVZ8DZIQ,6640
+konduktor/backends/deployment_utils.py,sha256=9CmB9CYC_3wxIfIOmTSCN2hbURZ5MpEMTvPwYMUXBRM,49272
 konduktor/backends/jobset.py,sha256=drt8Gc0iYQx18JWXBU6XfhUvC2xCKd8szSJ2JC4O20Q,8640
-konduktor/backends/jobset_utils.py,sha256=veY0O1Yn5YK9e3uZhBdeoATvKFjyARU9FYUzX5Ypyrc,26496
-konduktor/backends/pod_utils.py,sha256=KP_PAgsdNHFgt4Od-5gAtpifAKIL7DMBg7NJ44uqikg,14885
+konduktor/backends/jobset_utils.py,sha256=g49NY8RFhL_NNd4c1adRLG_Bq3UTFtRURxcAzxnMEYw,26524
+konduktor/backends/pod_utils.py,sha256=kOi3cLbTI3abZFCNQswWrkrOiBBm3gW_9N4INjxeS-w,19276
 konduktor/check.py,sha256=JennyWoaqSKhdyfUldd266KwVXTPJpcYQa4EED4a_BA,7569
-konduktor/cli.py,sha256=vb9SD8eJYjddiInVVQMq8K2BwqVDFoaMSTuaPhGp50I,57907
+konduktor/cli.py,sha256=B3Pp3RCwkGj8r9YgH-TgC85XU4zcc3eema1kpcDTQ3I,58452
 konduktor/config.py,sha256=9upqgCCYvcu6fKw7tovEYC1MWTkAAir0_WHPdayylbI,15536
 konduktor/constants.py,sha256=T3AeXXxuQHINW_bAWyztvDeS8r4g8kXBGIwIq13cys0,1814
 konduktor/controller/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -68,9 +68,9 @@ konduktor/data/storage_utils.py,sha256=n4GivkN0KMqmyOTDznF0Z-hzsJvm7KCEh5i5HgFAT
 konduktor/execution.py,sha256=d0EP79iSrW2uFsoqn0YV_4kgIupPIqpMOParXx0y3kg,18519
 konduktor/kube_client.py,sha256=HtM3d-_GigHnfGINRANchApR9_OigqczBgeYJ6Dj4j0,8504
 konduktor/logging.py,sha256=xtcCdnecmC3rqMTyunK-klQRINojI7NI4Apag78i9jM,3221
-konduktor/manifests/aibrix-setup.yaml,sha256=cUi7ToRRasRt2bXq53htV4QoCDEcxp2pvnCi-esJ4yk,9109
+konduktor/manifests/aibrix-setup.yaml,sha256=Foe3M1C0zVt-CVSJGr5SlQkMaNGs9kU2CvIZoANm3f8,14133
 konduktor/manifests/apoxy-setup.yaml,sha256=EipknCq33aBdxu9BIo6y5novjO0B_d_DCWqY44zYNuU,4262
-konduktor/manifests/apoxy-setup2.yaml,sha256=Benmg-io8audt_vi5T_T7vK8MYnaTnpX-bNDgRXM5t0,2512
+konduktor/manifests/apoxy-setup2.yaml,sha256=fc1tDwVopPVFzvUygkqxDGVqHHeo1cF9ERTnzUdgaGs,2517
 konduktor/manifests/controller_deployment.yaml,sha256=6p3oSLkEVONZsvKZGqVop0Dhn4bo3lrigRmhf8NXBHE,1730
 konduktor/manifests/dashboard_deployment.yaml,sha256=xJLd4FbPMAosI0fIv5_8y7dV9bw0Vsf81l-w4MB_aU8,2837
 konduktor/manifests/dmesg_daemonset.yaml,sha256=pSWt7YOeTYjS0l0iki1fvHOs7MhY-sH-RQfVW6JJyno,1391
@@ -78,15 +78,15 @@ konduktor/manifests/pod_cleanup_controller.yaml,sha256=hziL1Ka1kCAEL9R7Tjvpb80iw
 konduktor/resource.py,sha256=JqEE3LZiBBd5vqAiHDk-nlLve_VUQHhgdo6BIgx2Xfk,21215
 konduktor/serving.py,sha256=4s8cQhsVjf-HByZF65pbMxuqaV319hUSQE9pC8gP4Sg,5405
 konduktor/task.py,sha256=FIWm_rC_63GPBoe-Hi8a_eJ0H8Szw747SwXYPrNtOWE,37820
-konduktor/templates/deployment.yaml.j2,sha256=ByA33oaBgWnSx_QZ6yKeaMcCjktphcwF8R5KiOlM4pE,4859
+konduktor/templates/deployment.yaml.j2,sha256=0Cer53I8YHtYgUeEBQ_NVgC36FdOcjMNejgfP8teJC4,4964
 konduktor/templates/jobset.yaml.j2,sha256=NQcVeRNsTLLmTnJRnkL1vr45mSeth-b11YShXn_RoSg,1323
-konduktor/templates/pod.yaml.j2,sha256=p9yE-AQkCF2Tgjd1QQiOLtzgI6Gbpps-MZYJZ9fQWIs,19159
+konduktor/templates/pod.yaml.j2,sha256=3PXtpF4rycRFtU1AeIR6io7dTg3B63bJalNLQN1kEuQ,23982
 konduktor/usage/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 konduktor/usage/constants.py,sha256=gCL8afIHZhO0dcxbJGpESE9sCC1cBSbeRnQ8GwNOY4M,612
 konduktor/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 konduktor/utils/accelerator_registry.py,sha256=ythz3ynulP1DSSU7Jj5VUsQeBzSYRkxCVDZ5oOg0xtc,560
 konduktor/utils/annotations.py,sha256=oy2-BLydkFt3KWkXDuaGY84d6b7iISuy4eAT9uXk0Fc,2225
-konduktor/utils/base64_utils.py,sha256=mF-Tw98mFRG70YE4w6s9feuQSCYZHOb8YatBZwMugyI,3130
+konduktor/utils/base64_utils.py,sha256=TzKxe_SPHrurJ1lTOwOi4OmGUNkGxomstFcTWcWLQhw,3223
 konduktor/utils/common_utils.py,sha256=8gBpzYiC1bQ8sbgHIFLkKCGT5nLs1afpejod60kVSos,15076
 konduktor/utils/constants.py,sha256=1DneiTR21lvKUcWdBGwC4I4fD4uPjbjLUilEnJS7rzA,216
 konduktor/utils/env_options.py,sha256=T41Slzf4Mzl-n45CGXXqdy2fCrYhPNZQ7RP5vmnN4xc,2258
@@ -100,8 +100,8 @@ konduktor/utils/schemas.py,sha256=cr39nEAgjluhXoUYnvIwCwLBH8rLds37MBsF1uQv1rw,19
 konduktor/utils/subprocess_utils.py,sha256=WoFkoFhGecPR8-rF8WJxbIe-YtV94LXz9UG64SDhCY4,9448
 konduktor/utils/ux_utils.py,sha256=LSH4b5lckD157qDF4keThxtkGdxNrAfGKmH1ewhZkm4,8646
 konduktor/utils/validator.py,sha256=UcLvZCk9Cpbbhw8r_ZJtTpMSTfY1NKqcyciKsPzRPZM,17222
-konduktor_nightly-0.1.0.dev20251029104819.dist-info/LICENSE,sha256=MuuqTZbHvmqXR_aNKAXzggdV45ANd3wQ5YI7tnpZhm0,6586
-konduktor_nightly-0.1.0.dev20251029104819.dist-info/METADATA,sha256=Q_EXNlfTP6O88O_PGkF0cjGuD2zVJbH8PDfrWUw7CRs,4247
-konduktor_nightly-0.1.0.dev20251029104819.dist-info/WHEEL,sha256=XbeZDeTWKc1w7CSIyre5aMDU_-PohRwTQceYnisIYYY,88
-konduktor_nightly-0.1.0.dev20251029104819.dist-info/entry_points.txt,sha256=k3nG5wDFIJhNqsZWrHk4d0irIB2Ns9s47cjRWYsTCT8,48
-konduktor_nightly-0.1.0.dev20251029104819.dist-info/RECORD,,
+konduktor_nightly-0.1.0.dev20251031104711.dist-info/LICENSE,sha256=MuuqTZbHvmqXR_aNKAXzggdV45ANd3wQ5YI7tnpZhm0,6586
+konduktor_nightly-0.1.0.dev20251031104711.dist-info/METADATA,sha256=s4QzldEHu4WvsWWHhlRwF8ogMyxPBxD8oRjonvt9ENk,4247
+konduktor_nightly-0.1.0.dev20251031104711.dist-info/WHEEL,sha256=XbeZDeTWKc1w7CSIyre5aMDU_-PohRwTQceYnisIYYY,88
+konduktor_nightly-0.1.0.dev20251031104711.dist-info/entry_points.txt,sha256=k3nG5wDFIJhNqsZWrHk4d0irIB2Ns9s47cjRWYsTCT8,48
+konduktor_nightly-0.1.0.dev20251031104711.dist-info/RECORD,,