konduktor-nightly 0.1.0.dev20251022104926__py3-none-any.whl → 0.1.0.dev20251107104752__py3-none-any.whl

konduktor/__init__.py

@@ -11,7 +11,7 @@ from konduktor.task import Task
 __all__ = ['launch', 'Resources', 'Task', 'Serving']
 
 # Replaced with the current commit when building the wheels.
-_KONDUKTOR_COMMIT_SHA = '77ea1a3b26250e8ce18a507a4ab0d2538e351cb5'
+_KONDUKTOR_COMMIT_SHA = '5ceef9b8f579ac23f7a2bd863820aaa2341055e3'
 os.makedirs(os.path.expanduser('~/.konduktor'), exist_ok=True)
 
 
@@ -45,5 +45,5 @@ def _get_git_commit():
 
 
 __commit__ = _get_git_commit()
-__version__ = '1.0.0.dev0.1.0.dev20251022104926'
+__version__ = '1.0.0.dev0.1.0.dev20251107104752'
 __root_dir__ = os.path.dirname(os.path.abspath(__file__))

konduktor/backends/constants.py

@@ -18,3 +18,4 @@ STOP_USERNAME_LABEL = 'trainy.ai/stop-username'
 SECRET_BASENAME_LABEL = 'trainy.ai/secret-basename'
 SECRET_KIND_LABEL = 'trainy.ai/secret-kind'
 SECRET_OWNER_LABEL = 'trainy.ai/secret-owner'
+ROOT_NAME = 'trainy.ai/root-name'

konduktor/backends/deployment.py

@@ -54,8 +54,19 @@ def _wait_for_all_ready(namespace: str, name: str):
         except ApiException:
             services_map = {}
 
-        autoscaler = deployment_utils.get_autoscaler(namespace, name)
-        autoscalers_map = {name: autoscaler} if autoscaler else {}
+        autoscalers_map = {}
+        try:
+            autoscaler_obj = deployment_utils.get_autoscaler(namespace, name)
+            if autoscaler_obj:
+                # detect aibrix vs general from deployment labels
+                labels = (deployment.metadata.labels or {}) if deployment else {}
+                is_aibrix = deployment_utils.AIBRIX_NAME_LABEL in labels
+                if is_aibrix:
+                    autoscalers_map[name] = {'kpa': autoscaler_obj}
+                else:
+                    autoscalers_map[name] = {'hpa': autoscaler_obj}
+        except ApiException:
+            pass
 
         status = deployment_utils.get_model_status(
             name, deployments_map, services_map, autoscalers_map

konduktor/backends/deployment_utils.py

@@ -998,13 +998,13 @@ def get_envoy_external_ip() -> Optional[str]:
 
 
 def get_ingress_nginx_external_ip() -> Optional[str]:
-    """Get the external IP of the ingress-nginx-controller LoadBalancer."""
+    """Get the external IP of the keda-ingress-nginx-controller LoadBalancer."""
     context = kubernetes_utils.get_current_kube_config_context_name()
     core_api = kube_client.core_api(context=context)
     try:
-        # Look for ingress-nginx-controller service in keda namespace
+        # Look for keda-ingress-nginx-controller service in keda namespace
         service = core_api.read_namespaced_service(
-            name='ingress-nginx-controller', namespace='keda'
+            name='keda-ingress-nginx-controller', namespace='keda'
         )
         if service.spec.type == 'LoadBalancer':
             ingress = service.status.load_balancer.ingress

konduktor/backends/jobset_utils.py

@@ -5,15 +5,12 @@ import json
 import tempfile
 import time
 import typing
-from datetime import datetime, timezone
+from datetime import datetime, timedelta, timezone
 from typing import Any, Dict, Optional, Tuple
 
 import click
 import colorama
 
-if typing.TYPE_CHECKING:
-    from datetime import timedelta
-
 import konduktor
 from konduktor import kube_client, logging
 from konduktor.backends import constants as backend_constants
@@ -428,7 +425,9 @@ def _parse_timestamp_filter(timestamp_str: str) -> datetime:
                         seconds=abs(local_offset)
                     )
                 else:
-                    dt = dt.replace(tzinfo=timezone.utc)
+                    # Handle date-only format (local midnight --> UTC)
+                    local_tz = datetime.now().astimezone().tzinfo
+                    return dt.replace(tzinfo=local_tz).astimezone(timezone.utc)
             return dt
         except ValueError:
             continue
@@ -450,7 +449,8 @@ def _format_timestamp(timestamp: str) -> str:
 
 
 def _get_job_start_time(job: Dict[str, Any]) -> str:
-    for condition in job['status'].get('conditions', []):
+    status = job.get('status', {})
+    for condition in status.get('conditions', []):
         if condition['reason'] == 'ResumeJobs':
             return condition.get('lastTransitionTime', '')
     return '-'

konduktor/backends/pod_utils.py

@@ -153,7 +153,9 @@ def create_pod_spec(task: 'konduktor.Task') -> Dict[str, Any]:
     git_ssh_secret_name = None
     env_secret_envs = []
     default_secrets = []
+    basename_by_k8s: Dict[str, str] = {}
 
+    # only get own secrets
     user_hash = common_utils.get_user_hash()
     label_selector = f'{backend_constants.SECRET_OWNER_LABEL}={user_hash}'
     user_secrets = kubernetes_utils.list_secrets(
@@ -162,19 +164,36 @@ def create_pod_spec(task: 'konduktor.Task') -> Dict[str, Any]:
 
     for secret in user_secrets:
         kind = kubernetes_utils.get_secret_kind(secret)
+
+        # incase the user modified their secret to have no key:value data
+        if secret.data is None:
+            secret.data = {}
+
+        # fill the map for *all* secrets we see
+        k8s_name = secret.metadata.name
+        lbls = secret.metadata.labels or {}
+        base = lbls.get(
+            backend_constants.SECRET_BASENAME_LABEL,
+            # fallback: strip trailing "-<something>" once if present
+            k8s_name.rsplit('-', 1)[0] if '-' in k8s_name else k8s_name,
+        )
+        basename_by_k8s[k8s_name] = base
+
         if kind == 'git-ssh' and git_ssh_secret_name is None:
             git_ssh_secret_name = secret.metadata.name
         elif kind == 'env':
             env_secret_name = secret.metadata.name
-            key = next(iter(secret.data))
-            env_secret_envs.append(
-                {
-                    'name': key,
-                    'valueFrom': {
-                        'secretKeyRef': {'name': env_secret_name, 'key': key}
-                    },
-                }
-            )
+            # iterate ALL keys, not just one (ex. if user made a multi-key env secret)
+            for key, _ in secret.data.items():
+                # wire the env var to read its value from a k8s secret
+                env_secret_envs.append(
+                    {
+                        'name': key,
+                        'valueFrom': {
+                            'secretKeyRef': {'name': env_secret_name, 'key': key}
+                        },
+                    }
+                )
         elif kind == 'default':
             default_secret_name = secret.metadata.name
             basename = secret.metadata.labels.get(
@@ -184,6 +203,22 @@ def create_pod_spec(task: 'konduktor.Task') -> Dict[str, Any]:
                 {'k8s_name': default_secret_name, 'mount_name': basename}
             )
 
+    # Check if the task references KONDUKTOR_DEFAULT_SECRETS and that it exists
+    uses_default_secret_var = (
+        'KONDUKTOR_DEFAULT_SECRETS' in (task.run or '')
+        or 'KONDUKTOR_DEFAULT_SECRETS' in (task.setup or '')
+        or '/konduktor/default-secrets/' in (task.run or '')
+        or '/konduktor/default-secrets/' in (task.setup or '')
+    )
+    if uses_default_secret_var and not default_secrets:
+        raise exceptions.MissingSecretError(
+            f'Task references KONDUKTOR_DEFAULT_SECRETS or '
+            f'/konduktor/default-secrets but '
+            f'user {common_utils.get_cleaned_username()} '
+            f'has no default secrets. Paths like '
+            f'$KONDUKTOR_DEFAULT_SECRETS/<secret_name>/... will not exist.'
+        )
+
     # Inject --served-model-name, --host, and --port into serving run command
     if task.serving and task.run and 'vllm.entrypoints.openai.api_server' in task.run:
         if '--served-model-name' and '--host' and '--port' not in task.run:
@@ -262,31 +297,111 @@ def create_pod_spec(task: 'konduktor.Task') -> Dict[str, Any]:
             },
             temp.name,
         )
+
+        # Capture the template env names BEFORE user config is merged
+        pod_config_template = common_utils.read_yaml(temp.name)
+        tmpl_envs = pod_config_template['kubernetes']['pod_config']['spec'][
+            'containers'
+        ][0].get('env', [])
+        tmpl_env_names = {e['name'] for e in tmpl_envs}
+
         pod_config = common_utils.read_yaml(temp.name)
-        # merge with `~/.konduktor/config.yaml``
+        # merge with `~/.konduktor/config.yaml`` (config.yaml overrides template)
         kubernetes_utils.combine_pod_config_fields(temp.name, pod_config)
         pod_config = common_utils.read_yaml(temp.name)
 
-    # Priority order: task.envs > secret envs > existing pod_config envs
-    existing_envs = pod_config['kubernetes']['pod_config']['spec']['containers'][0].get(
+    # Find what came from user config (appeared after combine, not in template)
+    premerge_envs = pod_config['kubernetes']['pod_config']['spec']['containers'][0].get(
         'env', []
     )
-    env_map = {env['name']: env for env in existing_envs}
+    premerge_names = {e['name'] for e in premerge_envs}
+    config_env_names0 = premerge_names - tmpl_env_names
 
-    # Inject secret envs
+    # Build final env list
+    env_map = {env['name']: env for env in premerge_envs}
+
+    # Inject secret envs (env secrets override config.yaml)
     for env in env_secret_envs:
         env_map[env['name']] = env
 
-    # Inject task.envs
+    # Inject task envs
+    # CLI+task.yaml overrides everything else
+    # CLI already overrode task.yaml in other code
     for k, v in task.envs.items():
         env_map[k] = {'name': k, 'value': v}
 
-    # Replace the container's env section with the merged and prioritized map
-    pod_config['kubernetes']['pod_config']['spec']['containers'][0]['env'] = list(
-        env_map.values()
+    final_envs_list = list(env_map.values())
+    pod_config['kubernetes']['pod_config']['spec']['containers'][0]['env'] = (
+        final_envs_list
     )
+    container = pod_config['kubernetes']['pod_config']['spec']['containers'][0]
+    final_envs = container['env']
+    final_names = {e['name'] for e in final_envs}
+
     logger.debug(f'rendered pod spec: \n\t{json.dumps(pod_config, indent=2)}')
 
+    # 1) Get secret envs actually used in the final env list
+    secret_details = sorted(
+        (e['name'], e['valueFrom']['secretKeyRef']['name'])
+        for e in final_envs
+        if isinstance(e, dict)
+        and e.get('valueFrom', {})
+        and e['valueFrom'].get('secretKeyRef')
+    )
+    secret_names = [n for n, _ in secret_details]
+
+    # 2) Get task-sourced (CLI+task.yaml) envs actually used in the final env list
+    task_all_names = sorted(
+        n
+        for n in (task.envs or {}).keys()
+        if n in final_names and n not in secret_names
+    )
+
+    # 3) Get Config.yaml envs actually used in the final env list
+    config_names = sorted(
+        n
+        for n in config_env_names0
+        if n in final_names and n not in secret_names and n not in task_all_names
+    )
+
+    # 4) Get other envs (template/system) actually used in the final env list
+    other_names = sorted(
+        final_names - set(secret_names) - set(task_all_names) - set(config_names)
+    )
+
+    # Export helper envs for the startup script (names only)
+    def _append_helper(name: str, values):
+        container['env'].append({'name': name, 'value': ','.join(values)})
+
+    # to show user basenames of k8s secrets instead of actual
+    # k8s secret names (which have added suffixes)
+    secret_map_pairs = [
+        f'{var}={basename_by_k8s.get(secret_k8s, secret_k8s)}'
+        for (var, secret_k8s) in secret_details
+    ]
+
+    # Priority order: CLI > task.yaml > env secret > config > template/system
+    _append_helper(
+        'KONDUKTOR_ENV_SECRETS_HOPEFULLY_NO_NAME_COLLISION',
+        secret_names,
+    )
+    _append_helper(
+        'KONDUKTOR_ENV_SECRETS_MAP_HOPEFULLY_NO_NAME_COLLISION',
+        secret_map_pairs,
+    )
+    _append_helper(
+        'KONDUKTOR_ENV_TASK_ALL_HOPEFULLY_NO_NAME_COLLISION',
+        task_all_names,
+    )
+    _append_helper(
+        'KONDUKTOR_ENV_CONFIG_HOPEFULLY_NO_NAME_COLLISION',
+        config_names,
+    )
+    _append_helper(
+        'KONDUKTOR_ENV_OTHER_HOPEFULLY_NO_NAME_COLLISION',
+        other_names,
+    )
+
     # validate pod spec using json schema
     try:
         validator.validate_pod_spec(pod_config['kubernetes']['pod_config']['spec'])

konduktor/cli.py

@@ -54,6 +54,7 @@ from konduktor import logging
 from konduktor.backends import constants as backend_constants
 from konduktor.backends import deployment_utils, jobset_utils
 from konduktor.utils import (
+    base64_utils,
     common_utils,
     kubernetes_utils,
     log_utils,
@@ -161,7 +162,9 @@ def _make_task_with_overrides(
     if workdir is not None:
         task.workdir = workdir
 
-    task.set_resources_override(override_params)
+    # perform overrides from CLI
+    if override_params:
+        task.set_resources_override(override_params)
     if task.serving:
         task.set_serving_override(serving_override_params)
 
@@ -653,28 +656,23 @@ def status(
     all_users: bool, limit: Optional[int], after: Optional[str], before: Optional[str]
 ):
     # NOTE(dev): Keep the docstring consistent between the Python API and CLI.
-    """Shows list of all the jobs with optional filtering and pagination
-
-    Args:
-        all_users (bool): whether to show all jobs for all users
-        limit (Optional[int]): maximum number of jobs to display
-        after (Optional[str]): show jobs created after this timestamp
-        before (Optional[str]): show jobs created before this timestamp
+    """Shows list of all the jobs with optional filtering and pagination.
 
+    \b
     Examples:
-        konduktor status --limit 10
-        konduktor status --before "08/06/25 03:53PM"
-        konduktor status --all-users --limit 10 --after "08/06/25 03:53PM"
-
-    Note:
-        When using --before or --after timestamps, passing in "08/06/25" is
-        equivalent to passing in "08/06/25 00:00".
-        When using --before or --after timestamps, passing in "03:53PM" is
-        equivalent to passing in "03:53:00PM".
-        Timestamps shown in "konduktor startus" are truncated and are in the
-        local timezone. ex. "03:53:55PM" --> "03:53PM"
-        and would show up in --after "03:53PM" but not in --before "03:53PM"
-        despite status showing as "03:53PM".
+      konduktor status --limit 10
+      konduktor status --before "08/06/25 03:53PM"
+      konduktor status --all-users --limit 10 --after "08/06/25 03:53PM"
+
+    \b
+    Notes:
+      • When using --before or --after timestamps, "08/06/25"
+        is equivalent to "08/06/25 00:00".
+      • "03:53PM" is equivalent to "03:53:00PM".
+      • Timestamps shown in "konduktor status" are truncated
+        and are in the local timezone.
+        Example: "03:53:55PM" → "03:53PM" — would show up in
+        --after "03:53PM" but not in --before "03:53PM".
     """
     context = kubernetes_utils.get_current_kube_config_context_name()
     namespace = kubernetes_utils.get_kube_config_context_namespace(context)
@@ -802,6 +800,13 @@ def logs(
     # pylint: disable=bad-docstring-quotes
     help='Skip confirmation prompt.',
 )
+@click.option(
+    '--skip-image-check',
+    '-s',
+    is_flag=True,
+    default=False,
+    help='Skip Docker image validation checks for faster startup.',
+)
 def launch(
     entrypoint: Tuple[str, ...],
     dryrun: bool,
@@ -820,6 +825,7 @@ def launch(
     env: List[Tuple[str, str]],
     disk_size: Optional[int],
     yes: bool,
+    skip_image_check: bool,
 ):
     """Launch a task.
 
@@ -829,6 +835,9 @@ def launch(
     # NOTE(dev): Keep the docstring consistent between the Python API and CLI.
     env = _merge_env_vars(env_file, env)
 
+    if skip_image_check:
+        os.environ['KONDUKTOR_SKIP_IMAGE_CHECK'] = '1'
+
     task = _make_task_with_overrides(
         entrypoint=entrypoint,
         name=name,
@@ -1483,12 +1492,21 @@ def create(kind, from_file, from_directory, inline, name):
     data = {}
     if from_directory:
         click.echo(f'Creating secret from directory: {from_directory}')
-        base_path = pathlib.Path(from_directory)
-        for path in base_path.rglob('*'):
-            if path.is_file():
-                rel_path = path.relative_to(base_path)
-                with open(path, 'rb') as f:
-                    data[str(rel_path)] = b64encode(f.read()).decode()
+        # Use ABSOLUTE directory path so the top-level folder name is preserved
+        base_dir_abs = os.path.abspath(os.path.expanduser(from_directory))
+        if not os.path.isdir(base_dir_abs):
+            raise click.BadParameter(
+                f"--from-directory {from_directory} doesn't exist or is not a directory"
+            )
+        # Ensure there is at least one file inside
+        if not any(p.is_file() for p in pathlib.Path(base_dir_abs).rglob('*')):
+            raise click.BadParameter(f'--from-directory {from_directory} is empty.')
+
+        # Zip + base64 the WHOLE directory (this preserves the inner structure)
+        archive_b64 = base64_utils.zip_base64encode([base_dir_abs])
+
+        # Store as a single key; pod will unzip to the expanded path
+        data = {'payload.zip': archive_b64}
     elif from_file:
         click.echo(f'Creating secret from file: {from_file}')
         key = os.path.basename(from_file)
@@ -1694,6 +1712,13 @@ def serve():
     # pylint: disable=bad-docstring-quotes
     help='Skip confirmation prompt.',
 )
+@click.option(
+    '--skip-image-check',
+    '-s',
+    is_flag=True,
+    default=False,
+    help='Skip Docker image validation checks for faster startup.',
+)
 def serve_launch(
     entrypoint: Tuple[str, ...],
     dryrun: bool,
@@ -1716,6 +1741,7 @@ def serve_launch(
     ports: Optional[int],
     probe: Optional[str],
     yes: bool,
+    skip_image_check: bool = False,
 ):
     """Launch a deployment to serve.
 
@@ -1725,6 +1751,9 @@ def serve_launch(
     # NOTE(dev): Keep the docstring consistent between the Python API and CLI.
     env = _merge_env_vars(env_file, env)
 
+    if skip_image_check:
+        os.environ['KONDUKTOR_SKIP_IMAGE_CHECK'] = '1'
+
     task = _make_task_with_overrides(
         entrypoint=entrypoint,
         name=name,
@@ -1739,6 +1768,7 @@ def serve_launch(
         image_id=image_id,
         env=env,
         disk_size=disk_size,
+        # serving stuff
         min_replicas=min_replicas,
         max_replicas=max_replicas,
         ports=ports,

konduktor/manifests/aibrix-setup.yaml

@@ -34,6 +34,34 @@ metadata:
   name: aibrix-activator
 ---
 apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: aibrix-activator
+  namespace: aibrix-activator
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: aibrix-activator
+rules:
+- apiGroups: ["apps"]
+  resources: ["deployments"]
+  verbs: ["get", "list", "watch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: aibrix-activator
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: aibrix-activator
+subjects:
+- kind: ServiceAccount
+  name: aibrix-activator
+  namespace: aibrix-activator
+---
+apiVersion: v1
 kind: ConfigMap
 metadata:
   name: activator-code
@@ -44,16 +72,26 @@ data:
     from collections import defaultdict, deque
     from fastapi import FastAPI, Request
     from fastapi.responses import PlainTextResponse, JSONResponse
+    import asyncio
+    from kubernetes import client, config
 
     NAMESPACE = os.getenv("NAMESPACE", "default")
     WINDOW_SEC = int(os.getenv("WINDOW_SEC", "30"))        # demand lookback
     CAPACITY_RPS = float(os.getenv("CAPACITY_RPS", "1.0")) # per-replica capacity
     MIN_WAKE = int(os.getenv("MIN_REPLICA_ON_WAKE", "1"))
     MAX_REPLICAS = int(os.getenv("MAX_REPLICAS", "8"))
+    CLEANUP_INTERVAL = int(os.getenv("CLEANUP_INTERVAL", "300"))  # 5 minutes
 
     app = FastAPI()
     events = defaultdict(deque)  # key=(ns,model) -> deque[timestamps]
 
+    # Initialize Kubernetes client
+    try:
+        config.load_incluster_config()
+        k8s_apps_v1 = client.AppsV1Api()
+    except:
+        k8s_apps_v1 = None
+
     def _prune(q, now):
         while q and now - q[0] > WINDOW_SEC: q.popleft()
 
@@ -89,6 +127,48 @@ data:
             pass
         return None
 
+    def _get_existing_deployments():
+        """Get list of existing Aibrix deployments from Kubernetes"""
+        if not k8s_apps_v1:
+            return set()
+        try:
+            deployments = k8s_apps_v1.list_namespaced_deployment(
+                namespace=NAMESPACE,
+                label_selector="model.aibrix.ai/name"
+            )
+            return {d.metadata.name for d in deployments.items}
+        except Exception:
+            return set()
+
+    def _cleanup_stale_entries():
+        """Remove entries for deployments that no longer exist"""
+        if not k8s_apps_v1:
+            return
+        try:
+            existing_deployments = _get_existing_deployments()
+            # Remove entries for deployments that no longer exist
+            keys_to_remove = []
+            for (ns, model) in list(events.keys()):
+                if ns == NAMESPACE and model not in existing_deployments:
+                    keys_to_remove.append((ns, model))
+            
+            for key in keys_to_remove:
+                del events[key]
+                print(f"Cleaned up stale entry for deployment: {key[1]}")
+        except Exception as e:
+            print(f"Error during cleanup: {e}")
+
+    async def _cleanup_task():
+        """Background task to periodically clean up stale entries"""
+        while True:
+            await asyncio.sleep(CLEANUP_INTERVAL)
+            _cleanup_stale_entries()
+
+    @app.on_event("startup")
+    async def startup_event():
+        """Start background cleanup task"""
+        asyncio.create_task(_cleanup_task())
+
     # Mirror endpoints (same as your API paths); quick 204 response
     @app.post("/v1/completions")
     @app.post("/v1/chat/completions")
@@ -108,6 +188,37 @@ data:
             _bump(NAMESPACE, model)
         return JSONResponse({"ok": True}, status_code=204)
 
+    # Prometheus-friendly aggregate endpoint: export ALL (ns, model)
+    @app.get("/metrics", response_class=PlainTextResponse)
+    async def metrics_all():
+        lines = []
+        # Idiomatic names
+        lines.append("# HELP vllm_deployment_replicas Number of suggested replicas.")
+        lines.append("# TYPE vllm_deployment_replicas gauge")
+        lines.append("# HELP vllm_observed_rps Incoming requests per second.")
+        lines.append("# TYPE vllm_observed_rps gauge")
+        now = time.time()
+        for (ns, model), q in list(events.items()):
+            _prune(q, now)
+            rps = len(q) / max(WINDOW_SEC, 1)
+            d = _desired(ns, model)
+            lines.append(f'vllm_deployment_replicas{{namespace="{ns}",model_name="{model}"}} {d}')
+            lines.append(f'vllm_observed_rps{{namespace="{ns}",model_name="{model}"}} {rps:.6f}')
+        # (Optional) keep legacy names with colons for back-compat
+        lines.append("# HELP vllm:deployment_replicas Number of suggested replicas.")
+        lines.append("# TYPE vllm:deployment_replicas gauge")
+        lines.append("# HELP vllm:observed_rps Incoming requests per second.")
+        lines.append("# TYPE vllm:observed_rps gauge")
+        now = time.time()
+        for (ns, model), q in list(events.items()):
+            _prune(q, now)
+            rps = len(q) / max(WINDOW_SEC, 1)
+            d = _desired(ns, model)
+            lines.append(f'vllm:deployment_replicas{{namespace="{ns}",model_name="{model}"}} {d}')
+            lines.append(f'vllm:observed_rps{{namespace="{ns}",model_name="{model}"}} {rps:.6f}')
+        return "\n".join(lines) + "\n"
+
+
     # Metrics for KPA and Debugging
     @app.get("/metrics/{ns}/{model}", response_class=PlainTextResponse)
     async def metrics(ns: str, model: str):
@@ -142,7 +253,7 @@ spec:
         command: ["bash","-lc"]
         args:
           - |
-            pip install fastapi uvicorn >/dev/null && \
+            pip install fastapi uvicorn kubernetes >/dev/null && \
             uvicorn activator:app --host 0.0.0.0 --port 8080
         env:
         - { name: NAMESPACE, value: "default" }
@@ -150,10 +261,12 @@ spec:
         - { name: CAPACITY_RPS, value: "1.0" }
         - { name: MIN_REPLICA_ON_WAKE, value: "1" }
         - { name: MAX_REPLICAS, value: "8" }
+        - { name: CLEANUP_INTERVAL, value: "300" }
         ports: [{containerPort: 8080}]
         volumeMounts:
         - { name: code, mountPath: /app/activator.py, subPath: activator.py }
         workingDir: /app
+      serviceAccountName: aibrix-activator
       volumes:
       - name: code
         configMap: { name: activator-code }
@@ -163,6 +276,13 @@ kind: Service
 metadata:
   name: aibrix-activator
   namespace: aibrix-activator
+  annotations:
+    prometheus.io/scrape: "true"
+    prometheus.io/port: "8080"
+    prometheus.io/path: "/metrics"
+  labels:
+    app: aibrix-activator
+    prometheus-discovery: "true"
 spec:
   selector: { app: aibrix-activator }
   ports:
@@ -172,6 +292,42 @@ spec:
     protocol: TCP
   type: ClusterIP
 ---
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: aibrix-activator
+  namespace: prometheus
+  labels:
+    app: aibrix-activator
+spec:
+  selector:
+    matchLabels:
+      app: aibrix-activator
+  namespaceSelector:
+    matchNames:
+    - aibrix-activator
+  endpoints:
+  - port: http
+    path: /metrics
+---
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: vllm-deployments
+  namespace: prometheus
+  labels:
+    app: vllm-deployments
+spec:
+  selector:
+    matchLabels:
+      prometheus-discovery: "true"
+  namespaceSelector:
+    matchNames:
+    - default
+  endpoints:
+  - port: serve
+    path: /metrics
+---
 apiVersion: gateway.networking.k8s.io/v1beta1
 kind: ReferenceGrant
 metadata:

konduktor/manifests/apoxy-setup2.yaml

@@ -59,7 +59,7 @@ metadata:
   name: UNIQUE-TEMPNAME-backend2
 spec:
   endpoints:
-    - fqdn: ingress-nginx-controller.keda.UNIQUE-TEMPNAME.tun.apoxy.net
+    - fqdn: keda-ingress-nginx-controller.keda.UNIQUE-TEMPNAME.tun.apoxy.net
 ---
 # HTTPRoute for general deployments
 apiVersion: gateway.apoxy.dev/v1

konduktor/resource.py

@@ -59,6 +59,8 @@ class Resources:
         # Internal use only.
         # pylint: disable=invalid-name
         _cluster_config_overrides: Optional[Dict[str, Any]] = None,
+        # used to prevent double validation of image (would happen from overrides)
+        _validate_image: bool = True,
     ):
         """Initialize a Resources object.
 
@@ -124,7 +126,8 @@ class Resources:
         if isinstance(image_id, str):
             self._image_id = image_id.strip()
             # Validate Docker image format and existence
-            validator.validate_and_warn_image(self._image_id, 'task')
+            if _validate_image:
+                validator.validate_and_warn_image(self._image_id, 'task')
 
         self._labels = labels
         self._cluster_config_overrides = _cluster_config_overrides
@@ -435,15 +438,19 @@ class Resources:
 
     def copy(self, **override) -> 'Resources':
         """Returns a copy of the given Resources."""
+        # used to prevent double validation of image (would happen from overrides)
+        new_image_id = override.pop('image_id', self.image_id)
         resources = Resources(
             cloud=override.pop('cloud', self.cloud),
             cpus=override.pop('cpus', self._cpus),
             memory=override.pop('memory', self.memory),
             accelerators=override.pop('accelerators', self.accelerators),
             disk_size=override.pop('disk_size', self.disk_size),
-            image_id=override.pop('image_id', self.image_id),
+            image_id=new_image_id,
             labels=override.pop('labels', self.labels),
             job_config=override.pop('job_config', self.job_config),
+            # used to prevent double validation of image (would happen from overrides)
+            _validate_image=(new_image_id != self.image_id),
         )
         assert len(override) == 0
         return resources

konduktor/task.py

@@ -29,7 +29,7 @@ import konduktor
 from konduktor import constants, logging
 from konduktor.data import data_utils
 from konduktor.data import storage as storage_lib
-from konduktor.utils import common_utils, exceptions, schemas, ux_utils, validator
+from konduktor.utils import common_utils, exceptions, schemas, ux_utils
 
 logger = logging.get_logger(__name__)
 
@@ -388,10 +388,6 @@ class Task:
             )
             resources_config['_cluster_config_overrides'] = cluster_config_override
 
-        # Validate Docker image if specified in resources
-        if 'image_id' in resources_config and resources_config['image_id']:
-            validator.validate_and_warn_image(resources_config['image_id'], 'task')
-
         task.set_resources(konduktor.Resources.from_yaml_config(resources_config))
 
         # Parse serving field.

konduktor/templates/deployment.yaml.j2

@@ -41,11 +41,9 @@ metadata:
     {{ deployment_name_label }}: "{{ name }}"
     {{ deployment_user_label }}: "{{ user }}"
     trainy.ai/has-autoscaler: "{{ autoscaler }}"
-  {% if not general %}
   annotations:
     prometheus.io/scrape: "true"
-    prometheus.io/port: "8080"
-  {% endif %}
+    prometheus.io/port: "9000"
   name: {{ name }}
   namespace: default
 spec:
@@ -142,6 +140,10 @@ apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
   name: {{ name }}-ingress
+  labels:
+    {{ deployment_name_label }}: "{{ name }}"
+    {{ deployment_user_label }}: "{{ user }}"
+    trainy.ai/konduktor-managed: "true"
   annotations:
     nginx.ingress.kubernetes.io/use-regex: "true"
     nginx.ingress.kubernetes.io/rewrite-target: /$1

konduktor/templates/pod.yaml.j2

@@ -28,16 +28,21 @@ kubernetes:
       containers:
         # TODO(asaiacai): should decide here whether we add the fabric interfaces/containers init etc.        
         - name: konduktor-container
-          {% if enable_ssh %}
+          {% if enable_ssh or serving %}
           ports:
+            {% if enable_ssh %}
             - name: ssh
               containerPort: {{ konduktor_ssh_port }}
+            {% endif %}
+
+            {% if serving %}
+            - name: serving
+              containerPort: {{ ports }}
+            {% endif %}
           {% endif %}
-          {% if serving %}
-          ports:
-            - containerPort: {{ ports }}
+
+          {% if serving and probe %}
           # TODO (ryan): allow modification of thresholds and timings
-          {% if probe %}
           livenessProbe:
             httpGet:
               path: {{ probe }}
@@ -68,7 +73,6 @@ kubernetes:
             successThreshold: 1
             timeoutSeconds: 1
           {% endif %}
-          {% endif %}
           image: {{ image_id }}
           # this is set during jobset definition since we need to know the jobset
           # name and number of nodes to set all the environment variables correctly here
@@ -77,6 +81,10 @@ kubernetes:
           # flush logs immediately to stdout for more reactive log streaming
           - name: PYTHONUNBUFFERED
             value: "0"
+          - name: KONDUKTOR_NODENAME
+            valueFrom:
+              fieldRef:
+                fieldPath: spec.nodeName
           - name: KONDUKTOR_JOB_NAME
             value: "{{ job_name }}"
           - name: NODE_HOST_IPS
@@ -134,6 +142,8 @@ kubernetes:
           {% if default_secrets %}
           - name: KONDUKTOR_DEFAULT_SECRETS
             value: "/konduktor/default-secrets"
+          - name: KONDUKTOR_DEFAULT_SECRETS_EXPANDED
+            value: "/run/konduktor/expanded-default-secrets"
           {% endif %}
           # these are for compatibility with skypilot
           - name: SKYPILOT_NODE_IPS
@@ -146,6 +156,10 @@ kubernetes:
             value: "{{ num_nodes }}"
           - name: SKYPILOT_NUM_GPUS_PER_NODE
             value: "{{ num_gpus }}"
+          - name: RESTART_ATTEMPT
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.labels['jobset.sigs.k8s.io/restart-attempt']
           volumeMounts:
           - name: shared-memory
             mountPath: /dev/shm
@@ -159,6 +173,10 @@ kubernetes:
           - name: default-secret-{{ secret.mount_name }}
             mountPath: /konduktor/default-secrets/{{ secret.mount_name }}
           {% endfor %}
+          {% if default_secrets %}
+          - name: default-secrets-expanded
+            mountPath: /run/konduktor/expanded-default-secrets
+          {% endif %}
           {% if git_ssh %}
           - name: git-ssh-secret
             mountPath: /run/konduktor/git-ssh-secret
@@ -192,7 +210,7 @@ kubernetes:
               {% if 'curl' in run_cmd or 'curl' in setup_cmd or tailscale_secret %}
               PACKAGES="$PACKAGES curl";
               {% endif %}
-              {% if 'gs' in mount_secrets or 's3' in mount_secrets %}
+              {% if 'gs' in mount_secrets or 's3' in mount_secrets or default_secrets %}
               PACKAGES="$PACKAGES unzip wget";
               {% endif %}
               {% if 'git' in run_cmd or 'git' in setup_cmd %}
@@ -231,7 +249,7 @@ kubernetes:
               fi;
               end_epoch=$(date +%s);
               
-              echo "Exposing ENV variables"
+              echo "===== KONDUKTOR: Exposing ENV variables ====="
               $(prefix_cmd) env -0 | awk -v RS='\0' '
                 {
                   gsub(/\\/,"\\\\");      # escape existing backslashes first
@@ -346,8 +364,41 @@ kubernetes:
 
               $(prefix_cmd) echo "===== KONDUKTOR: Installing packages took $((end_epoch - start_epoch)) seconds ====="
 
+              $(prefix_cmd) echo "===== KONDUKTOR: Environment variable summary ====="
+              start_epoch=$(date +%s);
+
+              print_bucket () {
+                title="$1"; list="${2:-}"
+                echo "--- $title ---"
+                if [ -n "$list" ]; then
+                  echo "$list" | tr ',' '\n' | sed "s/^/[$title] /"
+                else
+                  echo "[none]"
+                fi
+              }
+
+              # Secrets: prefer detailed mapping if available
+              echo "--- env secret ---"
+              if [ -n "${KONDUKTOR_ENV_SECRETS_MAP_HOPEFULLY_NO_NAME_COLLISION:-}" ]; then
+                echo "${KONDUKTOR_ENV_SECRETS_MAP_HOPEFULLY_NO_NAME_COLLISION}" \
+                  | tr ',' '\n' \
+                  | awk -F'=' '{ printf("[secret: %s] %s\n", $2, $1) }'
+              elif [ -n "${KONDUKTOR_ENV_SECRETS_HOPEFULLY_NO_NAME_COLLISION:-}" ]; then
+                echo "${KONDUKTOR_ENV_SECRETS_HOPEFULLY_NO_NAME_COLLISION}" \
+                  | tr ',' '\n' | sed 's/^/[secret] /'
+              else
+                echo "[none]"
+              fi
+
+              print_bucket "CLI + task.yaml" "${KONDUKTOR_ENV_TASK_ALL_HOPEFULLY_NO_NAME_COLLISION}"
+              print_bucket "config.yaml"     "${KONDUKTOR_ENV_CONFIG_HOPEFULLY_NO_NAME_COLLISION}"
+              print_bucket "other"           "${KONDUKTOR_ENV_OTHER_HOPEFULLY_NO_NAME_COLLISION}"
+
+              end_epoch=$(date +%s);
+              $(prefix_cmd) echo "===== KONDUKTOR: Environment variable summary took $((end_epoch - start_epoch)) seconds ====="
+
               # unpack secrets credentials
-              $(prefix_cmd) echo "===== KONDUKTOR: Unpacking secrets credentials ====="
+              $(prefix_cmd) echo "===== KONDUKTOR: Unpacking cloud storage secret credentials ====="
               start_epoch=$(date +%s);
               mkdir -p ~/.konduktor
               mkdir -p {{ remote_workdir }}
@@ -362,12 +413,71 @@ kubernetes:
               $(prefix_cmd) unzip /run/konduktor/s3-secret/awscredentials -d ~/.aws
               {% endif %}
               {% endfor %}
+              
+              {% if default_secrets %}
+              $(prefix_cmd) echo "===== KONDUKTOR: Unpacking default secrets ====="
+              $(prefix_cmd) mkdir -p "${KONDUKTOR_DEFAULT_SECRETS_EXPANDED}"
+
+              # For each mounted default secret folder:
+              #  - if payload.zip exists, unzip it into the expanded dir
+              #  - otherwise, copy the files as-is
+              for src in "${KONDUKTOR_DEFAULT_SECRETS}"/*; do
+                [ -d "$src" ] || continue
+                name="$(basename "$src")"
+                dst="${KONDUKTOR_DEFAULT_SECRETS_EXPANDED}/${name}"
+                $(prefix_cmd) mkdir -p "$dst"
+
+                if [ -f "${src}/payload.zip" ]; then
+                  $(prefix_cmd) unzip -oq "${src}/payload.zip" -d "$dst"
+                else
+                  $(prefix_cmd) cp -a "${src}/." "$dst/"
+                fi
+              done
+
+              # Point callers to the expanded (writable) path going forward
+              export KONDUKTOR_DEFAULT_SECRETS="${KONDUKTOR_DEFAULT_SECRETS_EXPANDED}"
+              $(prefix_cmd) echo "KONDUKTOR_DEFAULT_SECRETS=${KONDUKTOR_DEFAULT_SECRETS_EXPANDED}" >> /etc/environment
+              {% endif %}
+              
               {% if git_ssh %}
               $(prefix_cmd) echo "Unpacking GIT-SSH secret"
               {% endif %}
               end_epoch=$(date +%s);
               $(prefix_cmd) echo "===== KONDUKTOR: Unpacking secrets credentials took $((end_epoch - start_epoch)) seconds ====="
 
+              $(prefix_cmd) echo "===== KONDUKTOR: Default secret summary ====="
+              start_epoch=$(date +%s)
+
+              root="${KONDUKTOR_DEFAULT_SECRETS:-}"
+              if [[ -z "$root" || ! -d "$root" ]]; then
+                $(prefix_cmd) echo "NO DEFAULT SECRETS FOUND."
+              else
+                for dir in "$root"/*; do
+                  [ -d "$dir" ] || continue
+                  name="$(basename "$dir")"
+
+                  # Pretty header that mirrors the logical mount base:
+                  $(prefix_cmd) echo "/konduktor/default-secrets/${name}:"
+
+                  # Print relative paths only; skip macOS junk and k8s secret internals
+                  (
+                    cd "$dir"
+                    out="$(find . \
+                      \( -name '.DS_Store' -o -name '__MACOSX' -o -name '..data' -o -name '..*' \) -prune -o \
+                      \( -type f -o -type l \) -print \
+                      | sed 's|^\./||' \
+                      | sort)"
+                    if [ -n "$out" ]; then
+                      printf "%s\n" "$out"
+                    fi
+                  )
+                done
+              fi
+
+              end_epoch=$(date +%s)
+              $(prefix_cmd) echo "===== KONDUKTOR: Default secret summary took $((end_epoch - start_epoch)) seconds ====="
+
+
               # sync file mounts
               {% for mkdir_command in mkdir_commands %}
               $(prefix_cmd) {{ mkdir_command }}
@@ -436,6 +546,10 @@ kubernetes:
         secret:
           secretName: {{ secret.k8s_name }}
       {% endfor %}
+      {% if default_secrets %}
+      - name: default-secrets-expanded
+        emptyDir: {}
+      {% endif %}
       {% if git_ssh %}
       - name: git-ssh-secret
         secret:

konduktor/utils/base64_utils.py

@@ -44,6 +44,8 @@ def zip_base64encode(files: List[str]) -> str:
                 else:
                     for root, _, files in os.walk(item_path):
                         for file in files:
+                            if file == '.DS_Store':
+                                continue
                             file_path = os.path.join(root, file)
                             arcname = os.path.relpath(file_path, temp_dir)
                             zipf.write(file_path, arcname)

konduktor/utils/validator.py

@@ -37,6 +37,11 @@ SCHEMA_URLS = {
 logger = logging.get_logger(__name__)
 
 
+def _skip_image_checks() -> bool:
+    val = os.getenv('KONDUKTOR_SKIP_IMAGE_CHECK', '')
+    return val.lower() in ('1', 'true', 'yes', 'y')
+
+
 def case_insensitive_enum(validator, enums, instance, schema):
     del validator, schema  # Unused.
     if instance.lower() not in [enum.lower() for enum in enums]:
@@ -419,6 +424,13 @@ def validate_and_warn_image(image_id: str, context: str = 'task') -> None:
     if not image_id:
         return
 
+    if _skip_image_checks():
+        logger.info(
+            'Skipping Docker image validation for %s',
+            image_id,
+        )
+        return
+
     status, message = validate_docker_image(image_id)
 
     if status == 'invalid':

{konduktor_nightly-0.1.0.dev20251022104926.dist-info → konduktor_nightly-0.1.0.dev20251107104752.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: konduktor-nightly
-Version: 0.1.0.dev20251022104926
+Version: 0.1.0.dev20251107104752
 Summary: GPU Cluster Health Management
 Author: Andrew Aikawa
 Author-email: asai@berkeley.edu

{konduktor_nightly-0.1.0.dev20251022104926.dist-info → konduktor_nightly-0.1.0.dev20251107104752.dist-info}/RECORD

@@ -1,4 +1,4 @@
-konduktor/__init__.py,sha256=CXLDGaXChd34z8ikUEZS5gmJ267X6oT4ikunLd99C6g,1574
+konduktor/__init__.py,sha256=mHmTi0owXeaxTt6NwGboUKlwfKWw6xwzbdcUjq9-1DM,1574
 konduktor/adaptors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 konduktor/adaptors/aws.py,sha256=s47Ra-GaqCQibzVfmD0pmwEWHif1EGO5opMbwkLxTCU,8244
 konduktor/adaptors/common.py,sha256=ZIqzjx77PIHUwpjfAQ1uX8B2aX78YMuGj4Bppd-MdyM,4183
@@ -6,14 +6,14 @@ konduktor/adaptors/gcp.py,sha256=ierTF4z7vwpJ9BsC7LSiwv4uLcjGXscwZOwQrddr2vM,410
 konduktor/authentication.py,sha256=_mVy3eqoKohicHostFiGwG1-2ybxP-l7ouofQ0LRlCY,4570
 konduktor/backends/__init__.py,sha256=usWJ8HdZJEyg7MIsN8Zcz9rk9e2Lq5dWJ8dv6hCN3ys,199
 konduktor/backends/backend.py,sha256=qh0bp94lzoTYZkzyQv2-CVrB5l91FkG2vclXg24UFC0,2910
-konduktor/backends/constants.py,sha256=nt9G9AmFCOMwO4GuKgRQSzJJuKapOmaROp4_Y0tMF5A,732
-konduktor/backends/deployment.py,sha256=VXz0GMvFzqq8kd1-pXaNaTsEtSyMmVO2OSd-_-T9nIE,6171
-konduktor/backends/deployment_utils.py,sha256=rfVpiM0KlXma8SxA029TczrpbY-gRG3T0pj2eeARkE8,49257
+konduktor/backends/constants.py,sha256=uAU-edQ_9DNYnu6x7fwNYXIEM7KMfJMOrnG74rlZ8mY,766
+konduktor/backends/deployment.py,sha256=d0a3F7dxDbnRKIt4ZO_kQ0_vet0pZvg4bWYzVZ8DZIQ,6640
+konduktor/backends/deployment_utils.py,sha256=9CmB9CYC_3wxIfIOmTSCN2hbURZ5MpEMTvPwYMUXBRM,49272
 konduktor/backends/jobset.py,sha256=drt8Gc0iYQx18JWXBU6XfhUvC2xCKd8szSJ2JC4O20Q,8640
-konduktor/backends/jobset_utils.py,sha256=If4pv5peB_yXrJJwjkySgVzbjcxDEDWfkOQxUkwSlOk,26386
-konduktor/backends/pod_utils.py,sha256=KP_PAgsdNHFgt4Od-5gAtpifAKIL7DMBg7NJ44uqikg,14885
+konduktor/backends/jobset_utils.py,sha256=g49NY8RFhL_NNd4c1adRLG_Bq3UTFtRURxcAzxnMEYw,26524
+konduktor/backends/pod_utils.py,sha256=kOi3cLbTI3abZFCNQswWrkrOiBBm3gW_9N4INjxeS-w,19276
 konduktor/check.py,sha256=JennyWoaqSKhdyfUldd266KwVXTPJpcYQa4EED4a_BA,7569
-konduktor/cli.py,sha256=HOL_O7BpYCl7uF5mKkMvXLjFNMTeAh2hYkraaJ6zi3w,57674
+konduktor/cli.py,sha256=B3Pp3RCwkGj8r9YgH-TgC85XU4zcc3eema1kpcDTQ3I,58452
 konduktor/config.py,sha256=9upqgCCYvcu6fKw7tovEYC1MWTkAAir0_WHPdayylbI,15536
 konduktor/constants.py,sha256=T3AeXXxuQHINW_bAWyztvDeS8r4g8kXBGIwIq13cys0,1814
 konduktor/controller/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -68,25 +68,25 @@ konduktor/data/storage_utils.py,sha256=n4GivkN0KMqmyOTDznF0Z-hzsJvm7KCEh5i5HgFAT
 konduktor/execution.py,sha256=d0EP79iSrW2uFsoqn0YV_4kgIupPIqpMOParXx0y3kg,18519
 konduktor/kube_client.py,sha256=HtM3d-_GigHnfGINRANchApR9_OigqczBgeYJ6Dj4j0,8504
 konduktor/logging.py,sha256=xtcCdnecmC3rqMTyunK-klQRINojI7NI4Apag78i9jM,3221
-konduktor/manifests/aibrix-setup.yaml,sha256=cUi7ToRRasRt2bXq53htV4QoCDEcxp2pvnCi-esJ4yk,9109
+konduktor/manifests/aibrix-setup.yaml,sha256=Foe3M1C0zVt-CVSJGr5SlQkMaNGs9kU2CvIZoANm3f8,14133
 konduktor/manifests/apoxy-setup.yaml,sha256=EipknCq33aBdxu9BIo6y5novjO0B_d_DCWqY44zYNuU,4262
-konduktor/manifests/apoxy-setup2.yaml,sha256=Benmg-io8audt_vi5T_T7vK8MYnaTnpX-bNDgRXM5t0,2512
+konduktor/manifests/apoxy-setup2.yaml,sha256=fc1tDwVopPVFzvUygkqxDGVqHHeo1cF9ERTnzUdgaGs,2517
 konduktor/manifests/controller_deployment.yaml,sha256=6p3oSLkEVONZsvKZGqVop0Dhn4bo3lrigRmhf8NXBHE,1730
 konduktor/manifests/dashboard_deployment.yaml,sha256=xJLd4FbPMAosI0fIv5_8y7dV9bw0Vsf81l-w4MB_aU8,2837
 konduktor/manifests/dmesg_daemonset.yaml,sha256=pSWt7YOeTYjS0l0iki1fvHOs7MhY-sH-RQfVW6JJyno,1391
 konduktor/manifests/pod_cleanup_controller.yaml,sha256=hziL1Ka1kCAEL9R7Tjvpb80iw1vcq9_3gwHCu75Bi0A,3939
-konduktor/resource.py,sha256=kfdhnUR_9kDLSm2sUAkv1sLQXyAkI08p3wIzCz1p7-M,20791
+konduktor/resource.py,sha256=JqEE3LZiBBd5vqAiHDk-nlLve_VUQHhgdo6BIgx2Xfk,21215
 konduktor/serving.py,sha256=4s8cQhsVjf-HByZF65pbMxuqaV319hUSQE9pC8gP4Sg,5405
-konduktor/task.py,sha256=LcTblvTNCSFNBHNUIOGN-i43whR8B6ccBm3HZTDoCMs,38050
-konduktor/templates/deployment.yaml.j2,sha256=ByA33oaBgWnSx_QZ6yKeaMcCjktphcwF8R5KiOlM4pE,4859
+konduktor/task.py,sha256=FIWm_rC_63GPBoe-Hi8a_eJ0H8Szw747SwXYPrNtOWE,37820
+konduktor/templates/deployment.yaml.j2,sha256=0Cer53I8YHtYgUeEBQ_NVgC36FdOcjMNejgfP8teJC4,4964
 konduktor/templates/jobset.yaml.j2,sha256=NQcVeRNsTLLmTnJRnkL1vr45mSeth-b11YShXn_RoSg,1323
-konduktor/templates/pod.yaml.j2,sha256=p9yE-AQkCF2Tgjd1QQiOLtzgI6Gbpps-MZYJZ9fQWIs,19159
+konduktor/templates/pod.yaml.j2,sha256=gGYwdXsPxStiua9Mm-OF7byVfuKjcH-TYNjoQmdpX_Q,24107
 konduktor/usage/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 konduktor/usage/constants.py,sha256=gCL8afIHZhO0dcxbJGpESE9sCC1cBSbeRnQ8GwNOY4M,612
 konduktor/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 konduktor/utils/accelerator_registry.py,sha256=ythz3ynulP1DSSU7Jj5VUsQeBzSYRkxCVDZ5oOg0xtc,560
 konduktor/utils/annotations.py,sha256=oy2-BLydkFt3KWkXDuaGY84d6b7iISuy4eAT9uXk0Fc,2225
-konduktor/utils/base64_utils.py,sha256=mF-Tw98mFRG70YE4w6s9feuQSCYZHOb8YatBZwMugyI,3130
+konduktor/utils/base64_utils.py,sha256=TzKxe_SPHrurJ1lTOwOi4OmGUNkGxomstFcTWcWLQhw,3223
 konduktor/utils/common_utils.py,sha256=8gBpzYiC1bQ8sbgHIFLkKCGT5nLs1afpejod60kVSos,15076
 konduktor/utils/constants.py,sha256=1DneiTR21lvKUcWdBGwC4I4fD4uPjbjLUilEnJS7rzA,216
 konduktor/utils/env_options.py,sha256=T41Slzf4Mzl-n45CGXXqdy2fCrYhPNZQ7RP5vmnN4xc,2258
@@ -99,9 +99,9 @@ konduktor/utils/rich_utils.py,sha256=ycADW6Ij3wX3uT8ou7T8qxX519RxlkJivsLvUahQaJo
 konduktor/utils/schemas.py,sha256=cr39nEAgjluhXoUYnvIwCwLBH8rLds37MBsF1uQv1rw,19067
 konduktor/utils/subprocess_utils.py,sha256=WoFkoFhGecPR8-rF8WJxbIe-YtV94LXz9UG64SDhCY4,9448
 konduktor/utils/ux_utils.py,sha256=LSH4b5lckD157qDF4keThxtkGdxNrAfGKmH1ewhZkm4,8646
-konduktor/utils/validator.py,sha256=gCB5v9Up9bCWD_92fS5ChfRRXj_m56Ky9uzd_77wXGI,16927
-konduktor_nightly-0.1.0.dev20251022104926.dist-info/LICENSE,sha256=MuuqTZbHvmqXR_aNKAXzggdV45ANd3wQ5YI7tnpZhm0,6586
-konduktor_nightly-0.1.0.dev20251022104926.dist-info/METADATA,sha256=udvmiRtWYgRGnpgjsOFcqQuluizG3yuDUCK41z4nhT0,4247
-konduktor_nightly-0.1.0.dev20251022104926.dist-info/WHEEL,sha256=XbeZDeTWKc1w7CSIyre5aMDU_-PohRwTQceYnisIYYY,88
-konduktor_nightly-0.1.0.dev20251022104926.dist-info/entry_points.txt,sha256=k3nG5wDFIJhNqsZWrHk4d0irIB2Ns9s47cjRWYsTCT8,48
-konduktor_nightly-0.1.0.dev20251022104926.dist-info/RECORD,,
+konduktor/utils/validator.py,sha256=UcLvZCk9Cpbbhw8r_ZJtTpMSTfY1NKqcyciKsPzRPZM,17222
+konduktor_nightly-0.1.0.dev20251107104752.dist-info/LICENSE,sha256=MuuqTZbHvmqXR_aNKAXzggdV45ANd3wQ5YI7tnpZhm0,6586
+konduktor_nightly-0.1.0.dev20251107104752.dist-info/METADATA,sha256=EEA9KjVBKhzBk4hO1-mWEacCmBul0d5GqMbB_VUKWbQ,4247
+konduktor_nightly-0.1.0.dev20251107104752.dist-info/WHEEL,sha256=XbeZDeTWKc1w7CSIyre5aMDU_-PohRwTQceYnisIYYY,88
+konduktor_nightly-0.1.0.dev20251107104752.dist-info/entry_points.txt,sha256=k3nG5wDFIJhNqsZWrHk4d0irIB2Ns9s47cjRWYsTCT8,48
+konduktor_nightly-0.1.0.dev20251107104752.dist-info/RECORD,,