konduktor-nightly 0.1.0.dev20250915104603__py3-none-any.whl → 0.1.0.dev20251107104752__py3-none-any.whl

konduktor/backends/jobset_utils.py

@@ -5,15 +5,12 @@ import json
 import tempfile
 import time
 import typing
-from datetime import datetime, timezone
+from datetime import datetime, timedelta, timezone
 from typing import Any, Dict, Optional, Tuple
 
 import click
 import colorama
 
-if typing.TYPE_CHECKING:
-    from datetime import timedelta
-
 import konduktor
 from konduktor import kube_client, logging
 from konduktor.backends import constants as backend_constants
@@ -428,7 +425,9 @@ def _parse_timestamp_filter(timestamp_str: str) -> datetime:
                         seconds=abs(local_offset)
                     )
                 else:
-                    dt = dt.replace(tzinfo=timezone.utc)
+                    # Handle date-only format (local midnight --> UTC)
+                    local_tz = datetime.now().astimezone().tzinfo
+                    return dt.replace(tzinfo=local_tz).astimezone(timezone.utc)
             return dt
         except ValueError:
             continue
@@ -450,7 +449,8 @@ def _format_timestamp(timestamp: str) -> str:
 
 
 def _get_job_start_time(job: Dict[str, Any]) -> str:
-    for condition in job['status'].get('conditions', []):
+    status = job.get('status', {})
+    for condition in status.get('conditions', []):
         if condition['reason'] == 'ResumeJobs':
             return condition.get('lastTransitionTime', '')
     return '-'

konduktor/backends/pod_utils.py

@@ -153,7 +153,9 @@ def create_pod_spec(task: 'konduktor.Task') -> Dict[str, Any]:
     git_ssh_secret_name = None
     env_secret_envs = []
     default_secrets = []
+    basename_by_k8s: Dict[str, str] = {}
 
+    # only get own secrets
     user_hash = common_utils.get_user_hash()
     label_selector = f'{backend_constants.SECRET_OWNER_LABEL}={user_hash}'
     user_secrets = kubernetes_utils.list_secrets(
@@ -162,19 +164,36 @@ def create_pod_spec(task: 'konduktor.Task') -> Dict[str, Any]:
 
     for secret in user_secrets:
         kind = kubernetes_utils.get_secret_kind(secret)
+
+        # incase the user modified their secret to have no key:value data
+        if secret.data is None:
+            secret.data = {}
+
+        # fill the map for *all* secrets we see
+        k8s_name = secret.metadata.name
+        lbls = secret.metadata.labels or {}
+        base = lbls.get(
+            backend_constants.SECRET_BASENAME_LABEL,
+            # fallback: strip trailing "-<something>" once if present
+            k8s_name.rsplit('-', 1)[0] if '-' in k8s_name else k8s_name,
+        )
+        basename_by_k8s[k8s_name] = base
+
         if kind == 'git-ssh' and git_ssh_secret_name is None:
             git_ssh_secret_name = secret.metadata.name
         elif kind == 'env':
             env_secret_name = secret.metadata.name
-            key = next(iter(secret.data))
-            env_secret_envs.append(
-                {
-                    'name': key,
-                    'valueFrom': {
-                        'secretKeyRef': {'name': env_secret_name, 'key': key}
-                    },
-                }
-            )
+            # iterate ALL keys, not just one (ex. if user made a multi-key env secret)
+            for key, _ in secret.data.items():
+                # wire the env var to read its value from a k8s secret
+                env_secret_envs.append(
+                    {
+                        'name': key,
+                        'valueFrom': {
+                            'secretKeyRef': {'name': env_secret_name, 'key': key}
+                        },
+                    }
+                )
         elif kind == 'default':
             default_secret_name = secret.metadata.name
             basename = secret.metadata.labels.get(
@@ -184,6 +203,22 @@ def create_pod_spec(task: 'konduktor.Task') -> Dict[str, Any]:
                 {'k8s_name': default_secret_name, 'mount_name': basename}
             )
 
+    # Check if the task references KONDUKTOR_DEFAULT_SECRETS and that it exists
+    uses_default_secret_var = (
+        'KONDUKTOR_DEFAULT_SECRETS' in (task.run or '')
+        or 'KONDUKTOR_DEFAULT_SECRETS' in (task.setup or '')
+        or '/konduktor/default-secrets/' in (task.run or '')
+        or '/konduktor/default-secrets/' in (task.setup or '')
+    )
+    if uses_default_secret_var and not default_secrets:
+        raise exceptions.MissingSecretError(
+            f'Task references KONDUKTOR_DEFAULT_SECRETS or '
+            f'/konduktor/default-secrets but '
+            f'user {common_utils.get_cleaned_username()} '
+            f'has no default secrets. Paths like '
+            f'$KONDUKTOR_DEFAULT_SECRETS/<secret_name>/... will not exist.'
+        )
+
     # Inject --served-model-name, --host, and --port into serving run command
     if task.serving and task.run and 'vllm.entrypoints.openai.api_server' in task.run:
         if '--served-model-name' and '--host' and '--port' not in task.run:
@@ -262,31 +297,111 @@ def create_pod_spec(task: 'konduktor.Task') -> Dict[str, Any]:
             },
             temp.name,
         )
+
+        # Capture the template env names BEFORE user config is merged
+        pod_config_template = common_utils.read_yaml(temp.name)
+        tmpl_envs = pod_config_template['kubernetes']['pod_config']['spec'][
+            'containers'
+        ][0].get('env', [])
+        tmpl_env_names = {e['name'] for e in tmpl_envs}
+
         pod_config = common_utils.read_yaml(temp.name)
-        # merge with `~/.konduktor/config.yaml``
+        # merge with `~/.konduktor/config.yaml`` (config.yaml overrides template)
         kubernetes_utils.combine_pod_config_fields(temp.name, pod_config)
         pod_config = common_utils.read_yaml(temp.name)
 
-    # Priority order: task.envs > secret envs > existing pod_config envs
-    existing_envs = pod_config['kubernetes']['pod_config']['spec']['containers'][0].get(
+    # Find what came from user config (appeared after combine, not in template)
+    premerge_envs = pod_config['kubernetes']['pod_config']['spec']['containers'][0].get(
         'env', []
     )
-    env_map = {env['name']: env for env in existing_envs}
+    premerge_names = {e['name'] for e in premerge_envs}
+    config_env_names0 = premerge_names - tmpl_env_names
 
-    # Inject secret envs
+    # Build final env list
+    env_map = {env['name']: env for env in premerge_envs}
+
+    # Inject secret envs (env secrets override config.yaml)
     for env in env_secret_envs:
         env_map[env['name']] = env
 
-    # Inject task.envs
+    # Inject task envs
+    # CLI+task.yaml overrides everything else
+    # CLI already overrode task.yaml in other code
     for k, v in task.envs.items():
         env_map[k] = {'name': k, 'value': v}
 
-    # Replace the container's env section with the merged and prioritized map
-    pod_config['kubernetes']['pod_config']['spec']['containers'][0]['env'] = list(
-        env_map.values()
+    final_envs_list = list(env_map.values())
+    pod_config['kubernetes']['pod_config']['spec']['containers'][0]['env'] = (
+        final_envs_list
     )
+    container = pod_config['kubernetes']['pod_config']['spec']['containers'][0]
+    final_envs = container['env']
+    final_names = {e['name'] for e in final_envs}
+
     logger.debug(f'rendered pod spec: \n\t{json.dumps(pod_config, indent=2)}')
 
+    # 1) Get secret envs actually used in the final env list
+    secret_details = sorted(
+        (e['name'], e['valueFrom']['secretKeyRef']['name'])
+        for e in final_envs
+        if isinstance(e, dict)
+        and e.get('valueFrom', {})
+        and e['valueFrom'].get('secretKeyRef')
+    )
+    secret_names = [n for n, _ in secret_details]
+
+    # 2) Get task-sourced (CLI+task.yaml) envs actually used in the final env list
+    task_all_names = sorted(
+        n
+        for n in (task.envs or {}).keys()
+        if n in final_names and n not in secret_names
+    )
+
+    # 3) Get Config.yaml envs actually used in the final env list
+    config_names = sorted(
+        n
+        for n in config_env_names0
+        if n in final_names and n not in secret_names and n not in task_all_names
+    )
+
+    # 4) Get other envs (template/system) actually used in the final env list
+    other_names = sorted(
+        final_names - set(secret_names) - set(task_all_names) - set(config_names)
+    )
+
+    # Export helper envs for the startup script (names only)
+    def _append_helper(name: str, values):
+        container['env'].append({'name': name, 'value': ','.join(values)})
+
+    # to show user basenames of k8s secrets instead of actual
+    # k8s secret names (which have added suffixes)
+    secret_map_pairs = [
+        f'{var}={basename_by_k8s.get(secret_k8s, secret_k8s)}'
+        for (var, secret_k8s) in secret_details
+    ]
+
+    # Priority order: CLI > task.yaml > env secret > config > template/system
+    _append_helper(
+        'KONDUKTOR_ENV_SECRETS_HOPEFULLY_NO_NAME_COLLISION',
+        secret_names,
+    )
+    _append_helper(
+        'KONDUKTOR_ENV_SECRETS_MAP_HOPEFULLY_NO_NAME_COLLISION',
+        secret_map_pairs,
+    )
+    _append_helper(
+        'KONDUKTOR_ENV_TASK_ALL_HOPEFULLY_NO_NAME_COLLISION',
+        task_all_names,
+    )
+    _append_helper(
+        'KONDUKTOR_ENV_CONFIG_HOPEFULLY_NO_NAME_COLLISION',
+        config_names,
+    )
+    _append_helper(
+        'KONDUKTOR_ENV_OTHER_HOPEFULLY_NO_NAME_COLLISION',
+        other_names,
+    )
+
     # validate pod spec using json schema
     try:
         validator.validate_pod_spec(pod_config['kubernetes']['pod_config']['spec'])

konduktor/cli.py

@@ -54,6 +54,7 @@ from konduktor import logging
 from konduktor.backends import constants as backend_constants
 from konduktor.backends import deployment_utils, jobset_utils
 from konduktor.utils import (
+    base64_utils,
     common_utils,
     kubernetes_utils,
     log_utils,
@@ -161,7 +162,9 @@ def _make_task_with_overrides(
     if workdir is not None:
         task.workdir = workdir
 
-    task.set_resources_override(override_params)
+    # perform overrides from CLI
+    if override_params:
+        task.set_resources_override(override_params)
     if task.serving:
         task.set_serving_override(serving_override_params)
 
@@ -653,28 +656,23 @@ def status(
     all_users: bool, limit: Optional[int], after: Optional[str], before: Optional[str]
 ):
     # NOTE(dev): Keep the docstring consistent between the Python API and CLI.
-    """Shows list of all the jobs with optional filtering and pagination
-
-    Args:
-        all_users (bool): whether to show all jobs for all users
-        limit (Optional[int]): maximum number of jobs to display
-        after (Optional[str]): show jobs created after this timestamp
-        before (Optional[str]): show jobs created before this timestamp
+    """Shows list of all the jobs with optional filtering and pagination.
 
+    \b
     Examples:
-        konduktor status --limit 10
-        konduktor status --before "08/06/25 03:53PM"
-        konduktor status --all-users --limit 10 --after "08/06/25 03:53PM"
-
-    Note:
-        When using --before or --after timestamps, passing in "08/06/25" is
-        equivalent to passing in "08/06/25 00:00".
-        When using --before or --after timestamps, passing in "03:53PM" is
-        equivalent to passing in "03:53:00PM".
-        Timestamps shown in "konduktor startus" are truncated and are in the
-        local timezone. ex. "03:53:55PM" --> "03:53PM"
-        and would show up in --after "03:53PM" but not in --before "03:53PM"
-        despite status showing as "03:53PM".
+      konduktor status --limit 10
+      konduktor status --before "08/06/25 03:53PM"
+      konduktor status --all-users --limit 10 --after "08/06/25 03:53PM"
+
+    \b
+    Notes:
+      • When using --before or --after timestamps, "08/06/25"
+        is equivalent to "08/06/25 00:00".
+      • "03:53PM" is equivalent to "03:53:00PM".
+      • Timestamps shown in "konduktor status" are truncated
+        and are in the local timezone.
+        Example: "03:53:55PM" → "03:53PM" — would show up in
+        --after "03:53PM" but not in --before "03:53PM".
     """
     context = kubernetes_utils.get_current_kube_config_context_name()
     namespace = kubernetes_utils.get_kube_config_context_namespace(context)
@@ -802,6 +800,13 @@ def logs(
     # pylint: disable=bad-docstring-quotes
     help='Skip confirmation prompt.',
 )
+@click.option(
+    '--skip-image-check',
+    '-s',
+    is_flag=True,
+    default=False,
+    help='Skip Docker image validation checks for faster startup.',
+)
 def launch(
     entrypoint: Tuple[str, ...],
     dryrun: bool,
@@ -820,6 +825,7 @@ def launch(
     env: List[Tuple[str, str]],
     disk_size: Optional[int],
     yes: bool,
+    skip_image_check: bool,
 ):
     """Launch a task.
 
@@ -829,6 +835,9 @@ def launch(
     # NOTE(dev): Keep the docstring consistent between the Python API and CLI.
     env = _merge_env_vars(env_file, env)
 
+    if skip_image_check:
+        os.environ['KONDUKTOR_SKIP_IMAGE_CHECK'] = '1'
+
     task = _make_task_with_overrides(
         entrypoint=entrypoint,
         name=name,
@@ -973,7 +982,9 @@ def down(
 
     if all:
         assert jobs_specs is not None, f'No jobs found in namespace {namespace}'
-        assert len(jobs_specs) > 0, f'No jobs found in namespace {namespace}'
+        if len(jobs_specs) == 0:
+            click.secho(f'No jobs found in namespace {namespace}', fg='yellow')
+            return
         jobs = [job['metadata']['name'] for job in jobs_specs]
     elif jobs:
         # Get all available jobs to match against patterns
@@ -1481,12 +1492,21 @@ def create(kind, from_file, from_directory, inline, name):
     data = {}
     if from_directory:
         click.echo(f'Creating secret from directory: {from_directory}')
-        base_path = pathlib.Path(from_directory)
-        for path in base_path.rglob('*'):
-            if path.is_file():
-                rel_path = path.relative_to(base_path)
-                with open(path, 'rb') as f:
-                    data[str(rel_path)] = b64encode(f.read()).decode()
+        # Use ABSOLUTE directory path so the top-level folder name is preserved
+        base_dir_abs = os.path.abspath(os.path.expanduser(from_directory))
+        if not os.path.isdir(base_dir_abs):
+            raise click.BadParameter(
+                f"--from-directory {from_directory} doesn't exist or is not a directory"
+            )
+        # Ensure there is at least one file inside
+        if not any(p.is_file() for p in pathlib.Path(base_dir_abs).rglob('*')):
+            raise click.BadParameter(f'--from-directory {from_directory} is empty.')
+
+        # Zip + base64 the WHOLE directory (this preserves the inner structure)
+        archive_b64 = base64_utils.zip_base64encode([base_dir_abs])
+
+        # Store as a single key; pod will unzip to the expanded path
+        data = {'payload.zip': archive_b64}
     elif from_file:
         click.echo(f'Creating secret from file: {from_file}')
         key = os.path.basename(from_file)
@@ -1630,7 +1650,7 @@ def list_secrets(all_users: bool):
 
 @cli.group(cls=_NaturalOrderGroup)
 def serve():
-    """Manage LLM serving with Konduktor.
+    """Manage deployment serving with Konduktor.
 
     USAGE: konduktor serve COMMAND
 
@@ -1692,6 +1712,13 @@ def serve():
     # pylint: disable=bad-docstring-quotes
     help='Skip confirmation prompt.',
 )
+@click.option(
+    '--skip-image-check',
+    '-s',
+    is_flag=True,
+    default=False,
+    help='Skip Docker image validation checks for faster startup.',
+)
 def serve_launch(
     entrypoint: Tuple[str, ...],
     dryrun: bool,
@@ -1714,6 +1741,7 @@ def serve_launch(
     ports: Optional[int],
     probe: Optional[str],
     yes: bool,
+    skip_image_check: bool = False,
 ):
     """Launch a deployment to serve.
 
@@ -1723,6 +1751,9 @@ def serve_launch(
     # NOTE(dev): Keep the docstring consistent between the Python API and CLI.
     env = _merge_env_vars(env_file, env)
 
+    if skip_image_check:
+        os.environ['KONDUKTOR_SKIP_IMAGE_CHECK'] = '1'
+
     task = _make_task_with_overrides(
         entrypoint=entrypoint,
         name=name,
@@ -1737,6 +1768,7 @@ def serve_launch(
         image_id=image_id,
         env=env,
         disk_size=disk_size,
+        # serving stuff
         min_replicas=min_replicas,
         max_replicas=max_replicas,
         ports=ports,