konduktor-nightly 0.1.0.dev20251107104752__py3-none-any.whl → 0.1.0.dev20251215105431__py3-none-any.whl

konduktor/__init__.py

@@ -11,7 +11,7 @@ from konduktor.task import Task
 __all__ = ['launch', 'Resources', 'Task', 'Serving']
 
 # Replaced with the current commit when building the wheels.
-_KONDUKTOR_COMMIT_SHA = '5ceef9b8f579ac23f7a2bd863820aaa2341055e3'
+_KONDUKTOR_COMMIT_SHA = '421390595e3a1b9f263e790323deae61d94da231'
 os.makedirs(os.path.expanduser('~/.konduktor'), exist_ok=True)
 
 
@@ -45,5 +45,5 @@ def _get_git_commit():
 
 
 __commit__ = _get_git_commit()
-__version__ = '1.0.0.dev0.1.0.dev20251107104752'
+__version__ = '1.0.0.dev0.1.0.dev20251215105431'
 __root_dir__ = os.path.dirname(os.path.abspath(__file__))

konduktor/backends/pod_utils.py

@@ -28,6 +28,8 @@ if typing.TYPE_CHECKING:
 logger = logging.get_logger(__name__)
 
 _RUN_DURATION_ANNOTATION_KEY = 'kueue.x-k8s.io/maxRunDurationSeconds'
+# Use a large default (7 days) to mimic "infinite" runtime.
+_DEFAULT_MAX_RUN_DURATION_SECONDS = 604800
 
 
 def create_pod_spec(task: 'konduktor.Task') -> Dict[str, Any]:
@@ -471,18 +473,21 @@ def inject_jobset_metadata(jobset_spec: Dict[str, Any], task: 'konduktor.Task')
         jobset_spec: The JobSet spec dictionary to modify
         task: The task object containing resource information
     """
-    # Add max run duration annotation
-    assert task.resources is not None and task.resources.labels is not None
-    maxRunDurationSeconds = task.resources.labels.get('maxRunDurationSeconds', None)
-    if not maxRunDurationSeconds:
-        raise ValueError('maxRunDurationSeconds is required')
-    jobset_spec['jobset']['metadata']['annotations'][_RUN_DURATION_ANNOTATION_KEY] = (
-        str(maxRunDurationSeconds)
+    assert task.resources is not None, 'Task resources are required'
+    labels = task.resources.labels or {}
+
+    # Add max run duration annotation, defaulting to a practically infinite value.
+    maxRunDurationSeconds = labels.get('maxRunDurationSeconds')
+    metadata = jobset_spec['jobset']['metadata']
+    metadata.setdefault('annotations', {})[_RUN_DURATION_ANNOTATION_KEY] = str(
+        maxRunDurationSeconds
+        if maxRunDurationSeconds is not None
+        else _DEFAULT_MAX_RUN_DURATION_SECONDS
     )
 
     # Inject resource labels into JobSet metadata.
-    if task.resources and task.resources.labels:
-        jobset_spec['jobset']['metadata']['labels'].update(task.resources.labels)
+    if labels:
+        jobset_spec['jobset']['metadata']['labels'].update(labels)
 
 
 def merge_pod_into_jobset_template(

konduktor/cli.py

@@ -34,6 +34,7 @@ listed in "konduktor --help".  Take care to put logically connected commands clo
 each other.
 """
 
+import difflib
 import fnmatch
 import os
 import pathlib
@@ -273,22 +274,20 @@ _TASK_OPTIONS = [
         '--env-file',
         required=False,
         type=dotenv.dotenv_values,
-        help="""\
-        Path to a dotenv file with environment variables to set on the remote
-        node.
-
-        If any values from ``--env-file`` conflict with values set by
-        ``--env``, the ``--env`` value will be preferred.""",
+        help=(
+            'Path to a dotenv file with environment variables to set on the '
+            'remote node. If any values from ``--env-file`` conflict '
+            'with values set by ``--env``, the ``--env`` value will '
+            'be preferred.'
+        ),
     ),
     click.option(
         '--env',
         required=False,
         type=_parse_env_var,
         multiple=True,
-        help="""\
-        Environment variable to set on the remote node.
-        It can be specified multiple times.
-        Examples:
+        help="""\\
+        Environment variable to set on the remote node. It can be specified multiple times:
 
         \b
         1. ``--env MY_ENV=1``: set ``$MY_ENV`` on the cluster to be 1.
@@ -298,7 +297,7 @@ _TASK_OPTIONS = [
         is run.
 
         3. ``--env MY_ENV3``: set ``$MY_ENV3`` on the cluster to be the
-        same value of ``$MY_ENV3`` in the local environment.""",
+        same value of ``$MY_ENV3`` in the local environment.""",  # noqa: E501,
     ),
 ]
 _TASK_OPTIONS_WITH_NAME = [
@@ -320,10 +319,10 @@ _EXTRA_RESOURCES_OPTIONS = [
         type=str,
         help=(
             'Type and number of GPUs to use. Example values: '
-            '"V100:8", "V100" (short for a count of 1)'
+            '"V100:8", "V100" (short for a count of 1) '
             'If a new cluster is being launched by this command, this is the '
-            'resources to provision. If an existing cluster is being reused, this'
-            " is seen as the task demand, which must fit the cluster's total "
+            'resources to provision. If an existing cluster is being reused, this '
+            "is seen as the task demand, which must fit the cluster's total "
             'resources and is used for scheduling the task. '
             'Overrides the "accelerators" '
             'config in the YAML if both are supplied. '
@@ -624,7 +623,7 @@ def cli():
     default=False,
     is_flag=True,
     required=False,
-    help='Show all clusters, including those not owned by the ' 'current user.',
+    help='Show all jobs, including those not owned by the current user.',
 )
 @click.option(
     '--limit',
@@ -660,19 +659,19 @@ def status(
 
     \b
     Examples:
-      konduktor status --limit 10
-      konduktor status --before "08/06/25 03:53PM"
-      konduktor status --all-users --limit 10 --after "08/06/25 03:53PM"
+        konduktor status --limit 10
+        konduktor status --before "08/06/25 03:53PM"
+        konduktor status --all-users --limit 10 --after "08/06/25 03:53PM"
 
     \b
     Notes:
-      • When using --before or --after timestamps, "08/06/25"
-        is equivalent to "08/06/25 00:00".
-      • "03:53PM" is equivalent to "03:53:00PM".
-      • Timestamps shown in "konduktor status" are truncated
-        and are in the local timezone.
-        Example: "03:53:55PM" → "03:53PM" — would show up in
-        --after "03:53PM" but not in --before "03:53PM".
+        • When using --before or --after timestamps, "08/06/25" is
+        equivalent to "08/06/25 00:00".
+        • "03:53PM" is equivalent to "03:53:00PM".
+        • Timestamps shown in "konduktor status" are truncated and are in
+        the local timezone.
+        Example: "03:53:55PM" → "03:53PM" would show up in --after "03:53PM"
+        but not in --before "03:53PM".
     """
     context = kubernetes_utils.get_current_kube_config_context_name()
     namespace = kubernetes_utils.get_kube_config_context_namespace(context)
@@ -690,8 +689,8 @@ def status(
     is_flag=True,
     default=False,
     help=(
-        'If specified, do not show logs but exit with a status code for the '
-        "job's status: 0 for succeeded, or 1 for all other statuses."
+        '[DEPRECATED] If specified, do not show logs but exit with a status code '
+        "for the job's status: 0 for succeeded, or 1 for all other statuses."
     ),
 )
 @click.option(
@@ -701,12 +700,13 @@ def status(
     help=(
         'Follow the logs of a job. '
         'If --no-follow is specified, print the log so far and exit. '
-        '[default: --follow]'
+        '(default: --follow)'
     ),
 )
 @click.option(
     '--num-lines',
-    '--num_lines' '-n',
+    '--num_lines',
+    '-n',
     default=-1,
     type=int,
     help=(
@@ -722,6 +722,19 @@ def status(
     type=int,
     help='The node rank to tail logs from.',
 )
+@click.option(
+    '--start-offset',
+    '--start_offset',
+    type=str,
+    required=False,
+    default='1h',
+    help=(
+        'Choose how much time from now to look back in logs. '
+        'Examples: 30s, 5m, 2h, 1d. Default is 1h. '
+        'Note: currently only applies when streaming (default --follow). '
+        'With --no-follow, all available logs are returned.'
+    ),
+)
 @click.argument('job_id', type=str, nargs=1)
 # TODO(zhwu): support logs by job name
 def logs(
@@ -730,11 +743,12 @@ def logs(
     follow: bool,
     num_lines: int,
     node_rank: int,
+    start_offset: str,
 ):
     # NOTE(dev): Keep the docstring consistent between the Python API and CLI.
     """Retrieve/tail the log of a job."""
     if status:
-        raise click.UsageError('`--status` is being deprecated)')
+        raise click.UsageError('`--status` is being deprecated')
 
     # Check if the job exists
     if not job_id:
@@ -748,20 +762,41 @@ def logs(
     try:
         _ = jobset_utils.get_jobset(namespace, job_id)
     except jobset_utils.JobNotFoundError:
-        click.secho(
+        message = (
             f"Job '{job_id}' not found in namespace '{namespace}'. "
-            f'Job may have been `konduktor down`. '
+            f'This may be due to a typo, `konduktor down`, or garbage collected. '
             f'Check your jobs with '
             f'{colorama.Style.BRIGHT}`konduktor status`'
-            f'{colorama.Style.RESET_ALL}.',
-            fg='yellow',
+            f'{colorama.Style.RESET_ALL}.'
         )
 
+        # Try to find near string matches to help with typos.
+        try:
+            job_specs = jobset_utils.list_jobset(namespace)
+            job_names = [
+                item['metadata']['name'] for item in (job_specs or {}).get('items', [])
+            ]
+            close_matches = difflib.get_close_matches(
+                job_id, job_names, n=3, cutoff=0.4
+            )
+        except Exception:
+            close_matches = []
+
+        if close_matches:
+            suggestions = ', '.join(
+                f'{colorama.Fore.YELLOW}{colorama.Style.BRIGHT}{name}{colorama.Style.NORMAL}'
+                for name in close_matches
+            )
+            message += f'{colorama.Fore.YELLOW} Did you mean: {suggestions}?'
+
+        click.secho(message, fg='yellow')
+
     log_utils.tail_logs(
         job_id,
         worker_id=node_rank,
         follow=follow,
         num_logs=num_lines,
+        start_offset=start_offset,
     )
 
 
@@ -829,8 +864,10 @@ def launch(
 ):
     """Launch a task.
 
-    If ENTRYPOINT points to a valid YAML file, it is read in as the task
-    specification. Otherwise, it is interpreted as a bash command.
+    \b
+    Notes:
+        • If ENTRYPOINT points to a valid YAML file, it is read in as the task
+        specification. Otherwise, it is interpreted as a bash command.
     """
     # NOTE(dev): Keep the docstring consistent between the Python API and CLI.
     env = _merge_env_vars(env_file, env)
@@ -902,96 +939,50 @@ def launch(
     )
 
 
-@cli.command(cls=_DocumentedCodeCommand)
-@click.argument(
-    'jobs',
-    nargs=-1,
-    required=False,
-)
-@click.option('--all', '-a', default=None, is_flag=True, help='Tear down all jobs.')
-@click.option(
-    '--all-users',
-    '--all_users',
-    default=False,
-    is_flag=True,
-    help='Include other users for teardown',
-)
-@click.option(
-    '--yes',
-    '-y',
-    is_flag=True,
-    default=False,
-    required=False,
-    help='Skip confirmation prompt.',
-)
-def down(
+def _find_matching_jobs(
     jobs: List[str],
-    all: Optional[bool],
+    jobs_response: Dict[str, Any],
+    namespace: str,
     all_users: Optional[bool],
-    yes: bool,
+    all_flag: Optional[bool] = None,
 ):
-    # NOTE(dev): Keep the docstring consistent between the Python API and CLI.
-    """Tear down job(s).
-
-    JOB is the name of the job to tear down.  If both
-    JOB and ``--all`` are supplied, the latter takes precedence.
-
-    Tearing down a job will delete all associated containers (all billing
-    stops), and any data on the containers disks will be lost.  Accelerators
-    (e.g., GPUs) that are part of the job will be deleted too.
-
-    Wildcard patterns are supported using * characters.
-    Examples: "test-*" matches all jobs starting with "test-",
-    "*-gpu" matches all jobs ending with "-gpu".
-
-    Examples:
-
-    .. code-block:: bash
-
-      # Tear down a specific job.
-      konduktor down cluster_name
-      \b
-      # Tear down multiple jobs.
-      konduktor down job1 job2
-      \b
-      # Tear down all jobs matching a pattern.
-      konduktor down "test-*"
-      \b
-      # Tear down all of this users jobs.
-      konduktor down -a
-      konduktor down --all
-
-      # Tear down all jobs across all users
-      konduktor down --all --all-users
+    """
+    Find all jobs matching against the user specified pattern.
+    In use in `konduktor down` and `konduktor stop`
 
+    Note(asaiacai): `jobs_response` should be the list of
+    all jobsets in this namespace, not necessarily belonging
+    to this user.
     """
 
-    context = kubernetes_utils.get_current_kube_config_context_name()
-    namespace = kubernetes_utils.get_kube_config_context_namespace(context)
-    jobs_response = jobset_utils.list_jobset(namespace)
-    assert jobs_response
-    jobs_specs = [
-        job
-        for job in jobs_response['items']
-        if (
-            job['metadata']['labels'][jobset_utils.JOBSET_USERID_LABEL]
-            == common_utils.user_and_hostname_hash()
-            and not all_users
-        )
-    ]
+    jobs_specs = [job for job in jobs_response['items']]
 
-    if all:
+    if all_flag:
         assert jobs_specs is not None, f'No jobs found in namespace {namespace}'
-        if len(jobs_specs) == 0:
-            click.secho(f'No jobs found in namespace {namespace}', fg='yellow')
-            return
-        jobs = [job['metadata']['name'] for job in jobs_specs]
+        assert len(jobs_specs) > 0, f'No jobs found in namespace {namespace}'
+        if all_users:
+            # --all with --all-users = all jobs of all users
+            jobs = [job['metadata']['name'] for job in jobs_specs]
+        else:
+            # --all without --all-users = all jobs of current user
+            jobs = [
+                job['metadata']['name']
+                for job in jobs_specs
+                if job['metadata']['labels'][backend_constants.USER_LABEL]
+                == common_utils.get_cleaned_username()
+            ]
+        return jobs
     elif jobs:
         # Get all available jobs to match against patterns
         if len(jobs_specs) == 0:
             raise click.ClickException(f'No jobs found in namespace {namespace}')
 
-        all_job_names = [job['metadata']['name'] for job in jobs_specs]
+        all_job_names = {
+            job['metadata']['name']: job['metadata']['labels'][
+                backend_constants.USER_LABEL
+            ]
+            for job in jobs_specs
+        }
         matched_jobs = []
 
         for job_pattern in jobs:
@@ -1003,6 +994,30 @@ def down(
                     fg='yellow',
                     err=True,
                 )
+            for matched_name in pattern_matches:
+                if all_job_names[matched_name] != common_utils.get_cleaned_username():
+                    warning_label = (
+                        f'{colorama.Style.BRIGHT}{colorama.Fore.RED}Warning'
+                        f'{colorama.Style.RESET_ALL}'
+                    )
+                    job_name = (
+                        f'{colorama.Style.BRIGHT}{colorama.Fore.WHITE}{matched_name}'
+                        f'{colorama.Style.RESET_ALL}'
+                    )
+                    launched_user = (
+                        f'{colorama.Style.BRIGHT}{colorama.Fore.CYAN}'
+                        f'{all_job_names[matched_name]}{colorama.Style.RESET_ALL}'
+                    )
+                    current_user = (
+                        f'{colorama.Style.BRIGHT}{colorama.Fore.GREEN}'
+                        f'{common_utils.get_cleaned_username()}'
+                        f'{colorama.Style.RESET_ALL}'
+                    )
+                    logger.info(
+                        f'{warning_label}: job {job_name} was launched by '
+                        f'{launched_user}, while the current user is {current_user}',
+                    )
+
             matched_jobs.extend(pattern_matches)
 
         # Remove duplicates while preserving order
@@ -1020,22 +1035,92 @@ def down(
             )
     else:
         raise click.ClickException(
-            'No jobs specified. Use --all to tear down '
-            'all jobs or specify job names/patterns.'
+            'No jobs specified. Use --all to specify '
+            'all jobs belonging to a user '
+            'or specify job names/patterns.'
         )
+    return jobs
+
+
+@cli.command(cls=_DocumentedCodeCommand)
+@click.argument(
+    'jobs',
+    nargs=-1,
+    required=False,
+)
+@click.option('--all', '-a', default=None, is_flag=True, help='Tear down all jobs.')
+@click.option(
+    '--all-users',
+    '--all_users',
+    default=False,
+    is_flag=True,
+    help='Include other users for teardown',
+)
+@click.option(
+    '--yes',
+    '-y',
+    is_flag=True,
+    default=False,
+    required=False,
+    help='Skip confirmation prompt.',
+)
+def down(
+    jobs: List[str],
+    all: Optional[bool],
+    all_users: Optional[bool],
+    yes: bool,
+):
+    # NOTE(dev): Keep the docstring consistent between the Python API and CLI.
+    """Tear down job(s).
+
+    \b
+    Examples:
+        # Tear down a specific job.
+        konduktor down my_job
+        \b
+        # Tear down multiple jobs.
+        konduktor down my_job1 my_job2
+        \b
+        # Tear down all jobs matching a pattern.
+        konduktor down "my_job-*"
+        \b
+        # Tear down all of this users jobs.
+        konduktor down -a
+        konduktor down --all
+        \b
+        # Tear down all jobs across all users
+        konduktor down --all --all-users
+
+    \b
+    Notes:
+        • If both JOB and ``--all`` are supplied, the latter takes precedence.
+        • Tearing down a job will delete all associated containers (all billing
+        stops), and any data on the containers disks will be lost.  Accelerators
+        (e.g., GPUs) that are part of the job will be deleted too.
+        • Wildcard patterns are supported using * characters.
+        Ex: "test-*" matches all jobs starting with "test-",
+        "*-gpu" matches all jobs ending with "-gpu".
+
+    """
+
+    context = kubernetes_utils.get_current_kube_config_context_name()
+    namespace = kubernetes_utils.get_kube_config_context_namespace(context)
+    jobs_response = jobset_utils.list_jobset(namespace)
+    assert jobs_response
+    filtered_jobs = _find_matching_jobs(jobs, jobs_response, namespace, all_users, all)
 
     if not yes:
         # Prompt if (1) --cluster is None, or (2) cluster doesn't exist, or (3)
         # it exists but is STOPPED.
         prompt = (
             f'Tearing down job(s) {colorama.Style.BRIGHT} '
-            f'{colorama.Fore.GREEN}{jobs}{colorama.Style.RESET_ALL}. '
+            f'{colorama.Fore.GREEN}{filtered_jobs}{colorama.Style.RESET_ALL}. '
             'Proceed?'
         )
         if prompt is not None:
             click.confirm(prompt, default=True, abort=True, show_default=True)
 
-    for job in track(jobs, description='Tearing down job(s)...'):
+    for job in track(filtered_jobs, description='Tearing down job(s)...'):
         jobset_utils.delete_jobset(namespace, job)
 
 
@@ -1069,112 +1154,59 @@ def stop(
 ):
     """Suspend job(s) (manual/user-initiated).
 
-    JOB is the name of the job to suspend. If both
-    JOB and ``--all`` are supplied, the latter takes precedence.
-
-    Suspending a job will pause execution and mark the job as SUSPENDED (by user).
-    The job can be resumed later with `konduktor start`.
-
-    If a job is suspended by the system (e.g., due to queueing),
-    it will show as SUSPENDED (by system).
-
-    Wildcard patterns are supported using * characters.
-    Examples: "my_job-*" matches all jobs starting with "my_job-",
-    "*-gpu" matches all jobs ending with "-gpu".
-
+    \b
     Examples:
+        # Suspend a specific job.
+        konduktor stop my_job
+        \b
+        # Suspend multiple jobs.
+        konduktor stop my_job1 my_job2
+        \b
+        # Suspend all jobs matching a pattern.
+        konduktor stop "my_job-*"
+        \b
+        # Suspend all of this users jobs.
+        konduktor stop -a
+        konduktor stop --all
+        \b
+        # Suspend all jobs across all users
+        konduktor stop --all --all-users
 
-    .. code-block:: bash
-
-      # Suspend a specific job.
-      konduktor stop my_job
-      \b
-      # Suspend multiple jobs.
-      konduktor stop my_job1 my_job2
-      \b
-      # Suspend all jobs matching a pattern.
-      konduktor stop "my_job-*"
-      \b
-      # Suspend all of this users jobs.
-      konduktor stop -a
-      konduktor stop --all
-
-      # Suspend all jobs across all users
-      konduktor stop --all --all-users
-
+    \b
+    Notes:
+        • If both JOB and ``--all`` are supplied, the latter takes precedence.
+        • Suspending a job will pause execution and mark the job as SUSPENDED (by user).
+        The job can be resumed later with `konduktor start`.
+        • If a job is suspended by the system (e.g., due to queueing), it
+        will show as SUSPENDED (by system).
+        • Wildcard patterns are supported using * characters.
+        Ex: "test-*" matches all jobs starting with "test-",
+        "*-gpu" matches all jobs ending with "-gpu".
     """
 
     context = kubernetes_utils.get_current_kube_config_context_name()
     namespace = kubernetes_utils.get_kube_config_context_namespace(context)
     jobs_response = jobset_utils.list_jobset(namespace)
     assert jobs_response
-    jobs_specs = [
-        job
-        for job in jobs_response['items']
-        if (
-            job['metadata']['labels'][jobset_utils.JOBSET_USERID_LABEL]
-            == common_utils.user_and_hostname_hash()
-            and not all_users
-        )
-    ]
-
-    if all:
-        assert jobs_specs is not None, f'No jobs found in namespace {namespace}'
-        assert len(jobs_specs) > 0, f'No jobs found in namespace {namespace}'
-        jobs = [job['metadata']['name'] for job in jobs_specs]
-    elif jobs:
-        # Get all available jobs to match against patterns
-        if len(jobs_specs) == 0:
-            raise click.ClickException(f'No jobs found in namespace {namespace}')
-
-        all_job_names = [job['metadata']['name'] for job in jobs_specs]
-        matched_jobs = []
-
-        for job_pattern in jobs:
-            # Use fnmatch for both wildcard and exact pattern matching
-            pattern_matches = fnmatch.filter(all_job_names, job_pattern)
-            if not pattern_matches:
-                click.secho(
-                    f'Warning: No jobs found matching pattern "{job_pattern}"',
-                    fg='yellow',
-                    err=True,
-                )
-            matched_jobs.extend(pattern_matches)
-
-        # Remove duplicates while preserving order
-        seen = set()
-        jobs = []
-        for job in matched_jobs:
-            if job not in seen:
-                seen.add(job)
-                jobs.append(job)
-
-        if not jobs:
-            raise click.ClickException(
-                f'No matching jobs found check status with '
-                f'{colorama.Style.BRIGHT}konduktor status{colorama.Style.RESET_ALL}'
-            )
-    else:
-        raise click.ClickException(
-            'No jobs specified. Use --all to suspend '
-            'all jobs or specify job names/patterns.'
-        )
+    filtered_jobs = _find_matching_jobs(jobs, jobs_response, namespace, all_users, all)
 
     if not yes:
         # Prompt for confirmation
         prompt = (
             f'Suspending job(s) {colorama.Style.BRIGHT} '
-            f'{colorama.Fore.GREEN}{jobs}{colorama.Style.RESET_ALL}. '
+            f'{colorama.Fore.GREEN}{filtered_jobs}{colorama.Style.RESET_ALL}. '
             'Proceed?'
         )
         if prompt is not None:
             click.confirm(prompt, default=True, abort=True, show_default=True)
 
-    for job in track(jobs, description='Suspending job(s)...'):
+    for job in track(filtered_jobs, description='Suspending job(s)...'):
         jobset_utils.stop_jobset(namespace, job)
 
     click.secho(
-        ux_utils.command_hint_messages(ux_utils.CommandHintType.JOB_STOP, jobs),
+        ux_utils.command_hint_messages(
+            ux_utils.CommandHintType.JOB_STOP, filtered_jobs
+        ),
         fg='green',
         bold=True,
     )
@@ -1212,54 +1244,41 @@ def start(
 ):
     """Resume suspended job(s) (manual/user-initiated).
 
-    JOB is the name of the job to resume. If both
-    JOB and ``--all`` are supplied, the latter takes precedence.
-
-    Resuming a job will restart execution from where it was suspended.
-    Only suspended jobs can be resumed.
-
-    This command works for both manually suspended jobs (SUSPENDED by user)
-    and system-suspended jobs (SUSPENDED by system).
-
-    Wildcard patterns are supported using * characters.
-    Examples: "my_job-*" matches all jobs starting with "my_job-",
-    "*-gpu" matches all jobs ending with "-gpu".
-
+    \b
     Examples:
+        # Resume a specific job.
+        konduktor start my_job
+        \b
+        # Resume multiple jobs.
+        konduktor start my_job1 my_job2
+        \b
+        # Resume all jobs matching a pattern.
+        konduktor start "my_job-*"
+        \b
+        # Resume all of this users suspended jobs.
+        konduktor start -a
+        konduktor start --all
+        \b
+        # Resume all suspended jobs across all users
+        konduktor start --all --all-users
 
-    .. code-block:: bash
-
-      # Resume a specific job.
-      konduktor start my_job
-      \b
-      # Resume multiple jobs.
-      konduktor start my_job1 my_job2
-      \b
-      # Resume all jobs matching a pattern.
-      konduktor start "my_job-*"
-      \b
-      # Resume all of this users suspended jobs.
-      konduktor start -a
-      konduktor start --all
-
-      # Resume all suspended jobs across all users
-      konduktor start --all --all-users
-
+    \b
+    Notes:
+        • If both JOB and ``--all`` are supplied, the latter takes precedence.
+        • Resuming a job will restart execution from where it was suspended.
+        Only suspended jobs can be resumed.
+        • This command works for both manually suspended jobs (SUSPENDED by user)
+        and system-suspended jobs (SUSPENDED by system).
+        • Wildcard patterns are supported using * characters.
+        Ex: "test-*" matches all jobs starting with "test-",
+        "*-gpu" matches all jobs ending with "-gpu".
     """
 
     context = kubernetes_utils.get_current_kube_config_context_name()
     namespace = kubernetes_utils.get_kube_config_context_namespace(context)
     jobs_response = jobset_utils.list_jobset(namespace)
     assert jobs_response
-    jobs_specs = [
-        job
-        for job in jobs_response['items']
-        if (
-            job['metadata']['labels'][jobset_utils.JOBSET_USERID_LABEL]
-            == common_utils.user_and_hostname_hash()
-            and not all_users
-        )
-    ]
+    jobs_specs = [job for job in jobs_response['items']]
 
     if all:
         # Only get suspended jobs when using --all
@@ -1335,24 +1354,22 @@ def start(
     nargs=-1,
 )
 def check(clouds: Tuple[str]):
-    """Check which clouds are available to use for storage
-
-    This checks storage credentials for a cloud supported by konduktor. If a
-    cloud is detected to be inaccessible, the reason and correction steps will
-    be shown.
-
-    If CLOUDS are specified, checks credentials for only those clouds.
-
-    The enabled clouds are cached and form the "search space" to be considered
-    for each task.
+    """Check which clouds are available to use for storage with Konduktor
 
+    \b
     Examples:
+        # Check only specific clouds - gs, s3.
+        konduktor check gs
+        konduktor check s3
 
-    .. code-block:: bash
-
-      # Check only specific clouds - gs, s3.
-      konduktor check gs
-      konduktor check s3
+    \b
+    Notes:
+        • This checks storage credentials for a cloud supported by konduktor.
+        If a cloud is detected to be inaccessible, the reason and correction
+        steps will be shown.
+        • If CLOUDS are specified, checks credentials for only those clouds.
+        • The enabled clouds are cached and form the "search space" to
+        be considered for each task.
     """
     clouds_arg = clouds if len(clouds) > 0 else None
     konduktor_check.check(clouds=clouds_arg)
@@ -1401,23 +1418,12 @@ def secret():
 
     USAGE: konduktor secret COMMAND
 
-    \b
-    Use one of the following COMMANDS:
-      create [FLAGS] [NAME]
-      delete [NAME]
-      list [FLAGS]
-
     \b
     Examples:
-      konduktor secret create --kind git-ssh --from-file=~/.ssh/id_rsa my-ssh-name
-      konduktor secret create --kind env --inline FOO=bar my-env-name
-      konduktor delete my-ssh-name
-      konduktor secret list
-
-    \b
-    For details on COMMAND ARGS:
-      konduktor secret create -h
-      konduktor secret list -h
+        konduktor secret create --kind git-ssh --from-file ~/.ssh/id_rsa my-ssh-name
+        konduktor secret create --kind env --inline FOO=bar my-env-name
+        konduktor secret delete my-ssh-name
+        konduktor secret list
     """
 
 
@@ -1608,8 +1614,7 @@ def delete(name):
     help='Show all secrets, including those not owned by the current user.',
 )
 def list_secrets(all_users: bool):
-    """List secrets in the namespace.
-    Defaults to only your secrets unless --all-users is set."""
+    """List secrets in the namespace."""
 
     context = kubernetes_utils.get_current_kube_config_context_name()
     namespace = kubernetes_utils.get_kube_config_context_namespace(context)
@@ -1654,23 +1659,11 @@ def serve():
 
     USAGE: konduktor serve COMMAND
 
-    \b
-    Use one of the following COMMANDS:
-      launch
-      down
-      status
-
     \b
     Examples:
       konduktor serve launch my-deployment
       konduktor serve down my-deployment
       konduktor serve status
-
-    \b
-    For details on COMMAND ARGS:
-      konduktor serve launch -h
-      konduktor serve down -h
-      konduktor serve status -h
     """
     pass
 
@@ -1745,8 +1738,10 @@ def serve_launch(
 ):
     """Launch a deployment to serve.
 
-    If ENTRYPOINT points to a valid YAML file, it is read in as the task
-    specification. Otherwise, it is interpreted as a bash command.
+    \b
+    Notes:
+        • If ENTRYPOINT points to a valid YAML file, it is read in as the task
+        specification. Otherwise, it is interpreted as a bash command.
     """
     # NOTE(dev): Keep the docstring consistent between the Python API and CLI.
     env = _merge_env_vars(env_file, env)
@@ -1828,13 +1823,10 @@ def serve_down(
 ):
     """Tear down deployments (Deployment, Service, PodAutoscaler).
 
-    Use --all or -a to tear down all deployments.
-
-    Examples:
-
     \b
-      konduktor serve down my-deployment
-      konduktor serve down -a
+    Examples:
+        konduktor serve down my-deployment
+        konduktor serve down -a
     """
     context = kubernetes_utils.get_current_kube_config_context_name()
     namespace = kubernetes_utils.get_kube_config_context_namespace(context)
@@ -1861,7 +1853,7 @@ def serve_down(
             )
     else:
         raise click.ClickException(
-            'No deployments specified. Use --all to tear down all deplotments '
+            'No deployments specified. Use --all to tear down all deployments '
             'or pass names/patterns.'
         )
 

konduktor/utils/log_utils.py

@@ -365,6 +365,7 @@ def tail_vicky_logs(
     worker_id: int = 0,
     num_logs: int = -1,
     follow: bool = True,
+    start_offset: str = '1h',
 ):
     context = kubernetes_utils.get_current_kube_config_context_name()
     namespace = kubernetes_utils.get_kube_config_context_namespace(context)
@@ -380,15 +381,16 @@ def tail_vicky_logs(
         assert num_logs > 0, f'num_logs must be greater than 0, got {num_logs}'
         query = {'limit': num_logs}
     if follow:
+        effective_offset = start_offset or '1h'
         logger.info(
-            'Tailing logs from 1 hour ago. '
+            f'Tailing logs from {effective_offset} ago. '
             'If logs come up empty, there might be logs just earlier '
-            'than the past hour, check Grafana or use:\n'
+            'than that window, check Grafana or use:\n'
             f'{colorama.Style.BRIGHT}{colorama.Fore.YELLOW}'
             f'`konduktor logs --no-follow {job_name}`'
             f'{colorama.Style.RESET_ALL}'
         )
-        query['start_offset'] = '1h'
+        query['start_offset'] = effective_offset
     query['query'] = (
         f'k8s.namespace.name: "{namespace}" AND '
         f'batch.kubernetes.io/job-name: "{job_name}-workers-0" AND '
@@ -453,12 +455,13 @@ def tail_logs(
     worker_id: int = 0,
     num_logs: int = 1000,
     follow: bool = True,
+    start_offset: str = '1h',
 ):
     logs_backend = config.get_nested(('logs', 'backend'), None)
     if logs_backend == LogBackend.VICTORIA:
-        tail_vicky_logs(job_name, worker_id, num_logs, follow)
+        tail_vicky_logs(job_name, worker_id, num_logs, follow, start_offset)
     elif logs_backend == LogBackend.LOKI:
         tail_loki_logs_ws(job_name, worker_id, num_logs, follow)
     else:
         logger.info('Defaulting to VictoriaLogs')
-        tail_vicky_logs(job_name, worker_id, num_logs, follow)
+        tail_vicky_logs(job_name, worker_id, num_logs, follow, start_offset)

{konduktor_nightly-0.1.0.dev20251107104752.dist-info → konduktor_nightly-0.1.0.dev20251215105431.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: konduktor-nightly
-Version: 0.1.0.dev20251107104752
+Version: 0.1.0.dev20251215105431
 Summary: GPU Cluster Health Management
 Author: Andrew Aikawa
 Author-email: asai@berkeley.edu
@@ -29,6 +29,7 @@ Requires-Dist: prettytable (>=3.12.0,<4.0.0)
 Requires-Dist: psutil (>=7.0.0,<8.0.0)
 Requires-Dist: python-dotenv (>=1.0.1,<2.0.0)
 Requires-Dist: rich (>=13.9.4,<14.0.0)
+Requires-Dist: sniffio (>=1.3,<2.0)
 Requires-Dist: websockets (>=15.0.1,<16.0.0)
 Description-Content-Type: text/markdown
 

{konduktor_nightly-0.1.0.dev20251107104752.dist-info → konduktor_nightly-0.1.0.dev20251215105431.dist-info}/RECORD

@@ -1,4 +1,4 @@
-konduktor/__init__.py,sha256=mHmTi0owXeaxTt6NwGboUKlwfKWw6xwzbdcUjq9-1DM,1574
+konduktor/__init__.py,sha256=A8k1HK8UyBfw1hk53UvZzpp6khOyWkEDT4EAypu6Osc,1574
 konduktor/adaptors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 konduktor/adaptors/aws.py,sha256=s47Ra-GaqCQibzVfmD0pmwEWHif1EGO5opMbwkLxTCU,8244
 konduktor/adaptors/common.py,sha256=ZIqzjx77PIHUwpjfAQ1uX8B2aX78YMuGj4Bppd-MdyM,4183
@@ -11,9 +11,9 @@ konduktor/backends/deployment.py,sha256=d0a3F7dxDbnRKIt4ZO_kQ0_vet0pZvg4bWYzVZ8D
 konduktor/backends/deployment_utils.py,sha256=9CmB9CYC_3wxIfIOmTSCN2hbURZ5MpEMTvPwYMUXBRM,49272
 konduktor/backends/jobset.py,sha256=drt8Gc0iYQx18JWXBU6XfhUvC2xCKd8szSJ2JC4O20Q,8640
 konduktor/backends/jobset_utils.py,sha256=g49NY8RFhL_NNd4c1adRLG_Bq3UTFtRURxcAzxnMEYw,26524
-konduktor/backends/pod_utils.py,sha256=kOi3cLbTI3abZFCNQswWrkrOiBBm3gW_9N4INjxeS-w,19276
+konduktor/backends/pod_utils.py,sha256=WL6b9_yBqBHjX84hE57uMTYL-rbGo_Ugf2L7-_8NpDc,19422
 konduktor/check.py,sha256=JennyWoaqSKhdyfUldd266KwVXTPJpcYQa4EED4a_BA,7569
-konduktor/cli.py,sha256=B3Pp3RCwkGj8r9YgH-TgC85XU4zcc3eema1kpcDTQ3I,58452
+konduktor/cli.py,sha256=ORnFQub6aSGeZytETn39Dafl1gjH-yihP2r5FnF3EeQ,59591
 konduktor/config.py,sha256=9upqgCCYvcu6fKw7tovEYC1MWTkAAir0_WHPdayylbI,15536
 konduktor/constants.py,sha256=T3AeXXxuQHINW_bAWyztvDeS8r4g8kXBGIwIq13cys0,1814
 konduktor/controller/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -93,15 +93,15 @@ konduktor/utils/env_options.py,sha256=T41Slzf4Mzl-n45CGXXqdy2fCrYhPNZQ7RP5vmnN4x
 konduktor/utils/exceptions.py,sha256=5IFnN5bIUSBJv4KRRrCepk5jyY9EG5vWWQqbjCmP3NU,6682
 konduktor/utils/kubernetes_enums.py,sha256=SabUueF6Bpzbpa57gyH5VB65xla2N9l8CZmAeYTfGmM,176
 konduktor/utils/kubernetes_utils.py,sha256=XleYxzG64hciZb-CjzBDjX8BOMhFATIIHZlXD2bqN0Q,27186
-konduktor/utils/log_utils.py,sha256=VUyTtN819BJnSwm33-73-h8aaD51Y5Gawt6ek2kU1tk,18181
+konduktor/utils/log_utils.py,sha256=EPDDNu7WxfbP0T5WBHRNA1Pplr7mIeqFIQf9JTOQlss,18340
 konduktor/utils/loki_utils.py,sha256=eOGiD7dZNuwzmyXKiifyqz00EVh2nwcUPFSiPkac9y0,4050
 konduktor/utils/rich_utils.py,sha256=ycADW6Ij3wX3uT8ou7T8qxX519RxlkJivsLvUahQaJo,3583
 konduktor/utils/schemas.py,sha256=cr39nEAgjluhXoUYnvIwCwLBH8rLds37MBsF1uQv1rw,19067
 konduktor/utils/subprocess_utils.py,sha256=WoFkoFhGecPR8-rF8WJxbIe-YtV94LXz9UG64SDhCY4,9448
 konduktor/utils/ux_utils.py,sha256=LSH4b5lckD157qDF4keThxtkGdxNrAfGKmH1ewhZkm4,8646
 konduktor/utils/validator.py,sha256=UcLvZCk9Cpbbhw8r_ZJtTpMSTfY1NKqcyciKsPzRPZM,17222
-konduktor_nightly-0.1.0.dev20251107104752.dist-info/LICENSE,sha256=MuuqTZbHvmqXR_aNKAXzggdV45ANd3wQ5YI7tnpZhm0,6586
-konduktor_nightly-0.1.0.dev20251107104752.dist-info/METADATA,sha256=EEA9KjVBKhzBk4hO1-mWEacCmBul0d5GqMbB_VUKWbQ,4247
-konduktor_nightly-0.1.0.dev20251107104752.dist-info/WHEEL,sha256=XbeZDeTWKc1w7CSIyre5aMDU_-PohRwTQceYnisIYYY,88
-konduktor_nightly-0.1.0.dev20251107104752.dist-info/entry_points.txt,sha256=k3nG5wDFIJhNqsZWrHk4d0irIB2Ns9s47cjRWYsTCT8,48
-konduktor_nightly-0.1.0.dev20251107104752.dist-info/RECORD,,
+konduktor_nightly-0.1.0.dev20251215105431.dist-info/LICENSE,sha256=MuuqTZbHvmqXR_aNKAXzggdV45ANd3wQ5YI7tnpZhm0,6586
+konduktor_nightly-0.1.0.dev20251215105431.dist-info/METADATA,sha256=WWbQsIiHdKINH1YFtDbNxi8ZW3K_G9P7FyasPy3IWdM,4283
+konduktor_nightly-0.1.0.dev20251215105431.dist-info/WHEEL,sha256=XbeZDeTWKc1w7CSIyre5aMDU_-PohRwTQceYnisIYYY,88
+konduktor_nightly-0.1.0.dev20251215105431.dist-info/entry_points.txt,sha256=k3nG5wDFIJhNqsZWrHk4d0irIB2Ns9s47cjRWYsTCT8,48
+konduktor_nightly-0.1.0.dev20251215105431.dist-info/RECORD,,