konduktor-nightly 0.1.0.dev20251124105105__tar.gz → 0.1.0.dev20251211105235__tar.gz

{konduktor_nightly-0.1.0.dev20251124105105 → konduktor_nightly-0.1.0.dev20251211105235}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: konduktor-nightly
-Version: 0.1.0.dev20251124105105
+Version: 0.1.0.dev20251211105235
 Summary: GPU Cluster Health Management
 Author: Andrew Aikawa
 Author-email: asai@berkeley.edu
@@ -29,6 +29,7 @@ Requires-Dist: prettytable (>=3.12.0,<4.0.0)
 Requires-Dist: psutil (>=7.0.0,<8.0.0)
 Requires-Dist: python-dotenv (>=1.0.1,<2.0.0)
 Requires-Dist: rich (>=13.9.4,<14.0.0)
+Requires-Dist: sniffio (>=1.3,<2.0)
 Requires-Dist: websockets (>=15.0.1,<16.0.0)
 Description-Content-Type: text/markdown
 

{konduktor_nightly-0.1.0.dev20251124105105 → konduktor_nightly-0.1.0.dev20251211105235}/konduktor/__init__.py

@@ -11,7 +11,7 @@ from konduktor.task import Task
 __all__ = ['launch', 'Resources', 'Task', 'Serving']
 
 # Replaced with the current commit when building the wheels.
-_KONDUKTOR_COMMIT_SHA = '4837817a46660c45c449eec1cf69ac90ba5b8390'
+_KONDUKTOR_COMMIT_SHA = '421390595e3a1b9f263e790323deae61d94da231'
 os.makedirs(os.path.expanduser('~/.konduktor'), exist_ok=True)
 
 
@@ -45,5 +45,5 @@ def _get_git_commit():
 
 
 __commit__ = _get_git_commit()
-__version__ = '1.0.0.dev0.1.0.dev20251124105105'
+__version__ = '1.0.0.dev0.1.0.dev20251211105235'
 __root_dir__ = os.path.dirname(os.path.abspath(__file__))

{konduktor_nightly-0.1.0.dev20251124105105 → konduktor_nightly-0.1.0.dev20251211105235}/konduktor/backends/pod_utils.py

@@ -28,6 +28,8 @@ if typing.TYPE_CHECKING:
 logger = logging.get_logger(__name__)
 
 _RUN_DURATION_ANNOTATION_KEY = 'kueue.x-k8s.io/maxRunDurationSeconds'
+# Use a large default (7 days) to mimic "infinite" runtime.
+_DEFAULT_MAX_RUN_DURATION_SECONDS = 604800
 
 
 def create_pod_spec(task: 'konduktor.Task') -> Dict[str, Any]:
@@ -471,20 +473,21 @@ def inject_jobset_metadata(jobset_spec: Dict[str, Any], task: 'konduktor.Task')
         jobset_spec: The JobSet spec dictionary to modify
         task: The task object containing resource information
     """
-    # Add max run duration annotation
-    assert (
-        task.resources is not None and task.resources.labels is not None
-    ), 'Task resources and task.resources.labels are required'
-    maxRunDurationSeconds = task.resources.labels.get('maxRunDurationSeconds', None)
-    if not maxRunDurationSeconds:
-        raise ValueError('maxRunDurationSeconds is required')
-    jobset_spec['jobset']['metadata']['annotations'][_RUN_DURATION_ANNOTATION_KEY] = (
-        str(maxRunDurationSeconds)
+    assert task.resources is not None, 'Task resources are required'
+    labels = task.resources.labels or {}
+
+    # Add max run duration annotation, defaulting to a practically infinite value.
+    maxRunDurationSeconds = labels.get('maxRunDurationSeconds')
+    metadata = jobset_spec['jobset']['metadata']
+    metadata.setdefault('annotations', {})[_RUN_DURATION_ANNOTATION_KEY] = str(
+        maxRunDurationSeconds
+        if maxRunDurationSeconds is not None
+        else _DEFAULT_MAX_RUN_DURATION_SECONDS
     )
 
     # Inject resource labels into JobSet metadata.
-    if task.resources and task.resources.labels:
-        jobset_spec['jobset']['metadata']['labels'].update(task.resources.labels)
+    if labels:
+        jobset_spec['jobset']['metadata']['labels'].update(labels)
 
 
 def merge_pod_into_jobset_template(

{konduktor_nightly-0.1.0.dev20251124105105 → konduktor_nightly-0.1.0.dev20251211105235}/konduktor/cli.py

@@ -274,22 +274,20 @@ _TASK_OPTIONS = [
         '--env-file',
         required=False,
         type=dotenv.dotenv_values,
-        help="""\
-        Path to a dotenv file with environment variables to set on the remote
-        node.
-
-        If any values from ``--env-file`` conflict with values set by
-        ``--env``, the ``--env`` value will be preferred.""",
+        help=(
+            'Path to a dotenv file with environment variables to set on the '
+            'remote node. If any values from ``--env-file`` conflict '
+            'with values set by ``--env``, the ``--env`` value will '
+            'be preferred.'
+        ),
     ),
     click.option(
         '--env',
         required=False,
         type=_parse_env_var,
         multiple=True,
-        help="""\
-        Environment variable to set on the remote node.
-        It can be specified multiple times.
-        Examples:
+        help="""\\
+        Environment variable to set on the remote node. It can be specified multiple times:
 
         \b
         1. ``--env MY_ENV=1``: set ``$MY_ENV`` on the cluster to be 1.
@@ -299,7 +297,7 @@ _TASK_OPTIONS = [
         is run.
 
         3. ``--env MY_ENV3``: set ``$MY_ENV3`` on the cluster to be the
-        same value of ``$MY_ENV3`` in the local environment.""",
+        same value of ``$MY_ENV3`` in the local environment.""",  # noqa: E501,
     ),
 ]
 _TASK_OPTIONS_WITH_NAME = [
@@ -321,10 +319,10 @@ _EXTRA_RESOURCES_OPTIONS = [
         type=str,
         help=(
             'Type and number of GPUs to use. Example values: '
-            '"V100:8", "V100" (short for a count of 1)'
+            '"V100:8", "V100" (short for a count of 1) '
             'If a new cluster is being launched by this command, this is the '
-            'resources to provision. If an existing cluster is being reused, this'
-            " is seen as the task demand, which must fit the cluster's total "
+            'resources to provision. If an existing cluster is being reused, this '
+            "is seen as the task demand, which must fit the cluster's total "
             'resources and is used for scheduling the task. '
             'Overrides the "accelerators" '
             'config in the YAML if both are supplied. '
@@ -661,19 +659,19 @@ def status(
 
     \b
     Examples:
-      konduktor status --limit 10
-      konduktor status --before "08/06/25 03:53PM"
-      konduktor status --all-users --limit 10 --after "08/06/25 03:53PM"
+        konduktor status --limit 10
+        konduktor status --before "08/06/25 03:53PM"
+        konduktor status --all-users --limit 10 --after "08/06/25 03:53PM"
 
     \b
     Notes:
-      • When using --before or --after timestamps, "08/06/25"
-        is equivalent to "08/06/25 00:00".
-      • "03:53PM" is equivalent to "03:53:00PM".
-      • Timestamps shown in "konduktor status" are truncated
-        and are in the local timezone.
-        Example: "03:53:55PM" → "03:53PM" — would show up in
-        --after "03:53PM" but not in --before "03:53PM".
+        • When using --before or --after timestamps, "08/06/25" is
+        equivalent to "08/06/25 00:00".
+        • "03:53PM" is equivalent to "03:53:00PM".
+        • Timestamps shown in "konduktor status" are truncated and are in
+        the local timezone.
+        Example: "03:53:55PM" → "03:53PM" would show up in --after "03:53PM"
+        but not in --before "03:53PM".
     """
     context = kubernetes_utils.get_current_kube_config_context_name()
     namespace = kubernetes_utils.get_kube_config_context_namespace(context)
@@ -691,8 +689,8 @@ def status(
     is_flag=True,
     default=False,
     help=(
-        'If specified, do not show logs but exit with a status code for the '
-        "job's status: 0 for succeeded, or 1 for all other statuses."
+        '[DEPRECATED] If specified, do not show logs but exit with a status code '
+        "for the job's status: 0 for succeeded, or 1 for all other statuses."
     ),
 )
 @click.option(
@@ -702,7 +700,7 @@ def status(
     help=(
         'Follow the logs of a job. '
         'If --no-follow is specified, print the log so far and exit. '
-        '[default: --follow]'
+        '(default: --follow)'
     ),
 )
 @click.option(
@@ -732,7 +730,7 @@ def status(
     default='1h',
     help=(
         'Choose how much time from now to look back in logs. '
-        'Examples: 30s, 5m, 2h, 1d. Default is 1h.'
+        'Examples: 30s, 5m, 2h, 1d. Default is 1h. '
         'Note: currently only applies when streaming (default --follow). '
         'With --no-follow, all available logs are returned.'
     ),
@@ -866,8 +864,10 @@ def launch(
 ):
     """Launch a task.
 
-    If ENTRYPOINT points to a valid YAML file, it is read in as the task
-    specification. Otherwise, it is interpreted as a bash command.
+    \b
+    Notes:
+        • If ENTRYPOINT points to a valid YAML file, it is read in as the task
+        specification. Otherwise, it is interpreted as a bash command.
     """
     # NOTE(dev): Keep the docstring consistent between the Python API and CLI.
     env = _merge_env_vars(env_file, env)
@@ -944,6 +944,7 @@ def _find_matching_jobs(
     jobs_response: Dict[str, Any],
     namespace: str,
     all_users: Optional[bool],
+    all_flag: Optional[bool] = None,
 ):
     """
     Find all jobs matching against the user specified pattern.
@@ -956,19 +957,21 @@ def _find_matching_jobs(
 
     jobs_specs = [job for job in jobs_response['items']]
 
-    if all_users:
+    if all_flag:
         assert jobs_specs is not None, f'No jobs found in namespace {namespace}'
         assert len(jobs_specs) > 0, f'No jobs found in namespace {namespace}'
-        jobs = [
-            job['metadata']['name']
-            for job in jobs_specs
-            if job['metadata']['labels'][backend_constants.USER_LABEL]
-            == common_utils.get_cleaned_username()
-        ]
-        logger.debug(
-            f'Jobs found for user {colorama.Style.BRIGHT}{colorama.Fore.CYAN}'
-            f'{common_utils.get_cleaned_username()}{colorama.Style.RESET_ALL}: {jobs}'
-        )
+        if all_users:
+            # --all with --all-users = all jobs of all users
+            jobs = [job['metadata']['name'] for job in jobs_specs]
+        else:
+            # --all without --all-users = all jobs of current user
+            jobs = [
+                job['metadata']['name']
+                for job in jobs_specs
+                if job['metadata']['labels'][backend_constants.USER_LABEL]
+                == common_utils.get_cleaned_username()
+            ]
+        return jobs
     elif jobs:
         # Get all available jobs to match against patterns
         if len(jobs_specs) == 0:
@@ -1070,36 +1073,33 @@ def down(
     # NOTE(dev): Keep the docstring consistent between the Python API and CLI.
     """Tear down job(s).
 
-    JOB is the name of the job to tear down.  If both
-    JOB and ``--all`` are supplied, the latter takes precedence.
-
-    Tearing down a job will delete all associated containers (all billing
-    stops), and any data on the containers disks will be lost.  Accelerators
-    (e.g., GPUs) that are part of the job will be deleted too.
-
-    Wildcard patterns are supported using * characters.
-    Examples: "test-*" matches all jobs starting with "test-",
-    "*-gpu" matches all jobs ending with "-gpu".
-
+    \b
     Examples:
+        # Tear down a specific job.
+        konduktor down my_job
+        \b
+        # Tear down multiple jobs.
+        konduktor down my_job1 my_job2
+        \b
+        # Tear down all jobs matching a pattern.
+        konduktor down "my_job-*"
+        \b
+        # Tear down all of this users jobs.
+        konduktor down -a
+        konduktor down --all
+        \b
+        # Tear down all jobs across all users
+        konduktor down --all --all-users
 
-    .. code-block:: bash
-
-      # Tear down a specific job.
-      konduktor down cluster_name
-      \b
-      # Tear down multiple jobs.
-      konduktor down job1 job2
-      \b
-      # Tear down all jobs matching a pattern.
-      konduktor down "test-*"
-      \b
-      # Tear down all of this users jobs.
-      konduktor down -a
-      konduktor down --all
-
-      # Tear down all jobs across all users
-      konduktor down --all --all-users
+    \b
+    Notes:
+        • If both JOB and ``--all`` are supplied, the latter takes precedence.
+        • Tearing down a job will delete all associated containers (all billing
+        stops), and any data on the containers disks will be lost.  Accelerators
+        (e.g., GPUs) that are part of the job will be deleted too.
+        • Wildcard patterns are supported using * characters.
+        Ex: "test-*" matches all jobs starting with "test-",
+        "*-gpu" matches all jobs ending with "-gpu".
 
     """
 
@@ -1107,7 +1107,7 @@ def down(
     namespace = kubernetes_utils.get_kube_config_context_namespace(context)
     jobs_response = jobset_utils.list_jobset(namespace)
     assert jobs_response
-    filtered_jobs = _find_matching_jobs(jobs, jobs_response, namespace, all_users)
+    filtered_jobs = _find_matching_jobs(jobs, jobs_response, namespace, all_users, all)
 
     if not yes:
         # Prompt if (1) --cluster is None, or (2) cluster doesn't exist, or (3)
@@ -1154,46 +1154,41 @@ def stop(
 ):
     """Suspend job(s) (manual/user-initiated).
 
-    JOB is the name of the job to suspend. If both
-    JOB and ``--all`` are supplied, the latter takes precedence.
-
-    Suspending a job will pause execution and mark the job as SUSPENDED (by user).
-    The job can be resumed later with `konduktor start`.
-
-    If a job is suspended by the system (e.g., due to queueing),
-    it will show as SUSPENDED (by system).
-
-    Wildcard patterns are supported using * characters.
-    Examples: "my_job-*" matches all jobs starting with "my_job-",
-    "*-gpu" matches all jobs ending with "-gpu".
-
+    \b
     Examples:
+        # Suspend a specific job.
+        konduktor stop my_job
+        \b
+        # Suspend multiple jobs.
+        konduktor stop my_job1 my_job2
+        \b
+        # Suspend all jobs matching a pattern.
+        konduktor stop "my_job-*"
+        \b
+        # Suspend all of this users jobs.
+        konduktor stop -a
+        konduktor stop --all
+        \b
+        # Suspend all jobs across all users
+        konduktor stop --all --all-users
 
-    .. code-block:: bash
-
-      # Suspend a specific job.
-      konduktor stop my_job
-      \b
-      # Suspend multiple jobs.
-      konduktor stop my_job1 my_job2
-      \b
-      # Suspend all jobs matching a pattern.
-      konduktor stop "my_job-*"
-      \b
-      # Suspend all of this users jobs.
-      konduktor stop -a
-      konduktor stop --all
-
-      # Suspend all jobs across all users
-      konduktor stop --all --all-users
-
+    \b
+    Notes:
+        • If both JOB and ``--all`` are supplied, the latter takes precedence.
+        • Suspending a job will pause execution and mark the job as SUSPENDED (by user).
+        The job can be resumed later with `konduktor start`.
+        • If a job is suspended by the system (e.g., due to queueing), it
+        will show as SUSPENDED (by system).
+        • Wildcard patterns are supported using * characters.
+        Ex: "test-*" matches all jobs starting with "test-",
+        "*-gpu" matches all jobs ending with "-gpu".
     """
 
     context = kubernetes_utils.get_current_kube_config_context_name()
     namespace = kubernetes_utils.get_kube_config_context_namespace(context)
     jobs_response = jobset_utils.list_jobset(namespace)
     assert jobs_response
-    filtered_jobs = _find_matching_jobs(jobs, jobs_response, namespace, all_users)
+    filtered_jobs = _find_matching_jobs(jobs, jobs_response, namespace, all_users, all)
 
     if not yes:
         # Prompt for confirmation
@@ -1249,39 +1244,34 @@ def start(
 ):
     """Resume suspended job(s) (manual/user-initiated).
 
-    JOB is the name of the job to resume. If both
-    JOB and ``--all`` are supplied, the latter takes precedence.
-
-    Resuming a job will restart execution from where it was suspended.
-    Only suspended jobs can be resumed.
-
-    This command works for both manually suspended jobs (SUSPENDED by user)
-    and system-suspended jobs (SUSPENDED by system).
-
-    Wildcard patterns are supported using * characters.
-    Examples: "my_job-*" matches all jobs starting with "my_job-",
-    "*-gpu" matches all jobs ending with "-gpu".
-
+    \b
     Examples:
+        # Resume a specific job.
+        konduktor start my_job
+        \b
+        # Resume multiple jobs.
+        konduktor start my_job1 my_job2
+        \b
+        # Resume all jobs matching a pattern.
+        konduktor start "my_job-*"
+        \b
+        # Resume all of this users suspended jobs.
+        konduktor start -a
+        konduktor start --all
+        \b
+        # Resume all suspended jobs across all users
+        konduktor start --all --all-users
 
-    .. code-block:: bash
-
-      # Resume a specific job.
-      konduktor start my_job
-      \b
-      # Resume multiple jobs.
-      konduktor start my_job1 my_job2
-      \b
-      # Resume all jobs matching a pattern.
-      konduktor start "my_job-*"
-      \b
-      # Resume all of this users suspended jobs.
-      konduktor start -a
-      konduktor start --all
-
-      # Resume all suspended jobs across all users
-      konduktor start --all --all-users
-
+    \b
+    Notes:
+        • If both JOB and ``--all`` are supplied, the latter takes precedence.
+        • Resuming a job will restart execution from where it was suspended.
+        Only suspended jobs can be resumed.
+        • This command works for both manually suspended jobs (SUSPENDED by user)
+        and system-suspended jobs (SUSPENDED by system).
+        • Wildcard patterns are supported using * characters.
+        Ex: "test-*" matches all jobs starting with "test-",
+        "*-gpu" matches all jobs ending with "-gpu".
     """
 
     context = kubernetes_utils.get_current_kube_config_context_name()
@@ -1364,24 +1354,22 @@ def start(
     nargs=-1,
 )
 def check(clouds: Tuple[str]):
-    """Check which clouds are available to use for storage
-
-    This checks storage credentials for a cloud supported by konduktor. If a
-    cloud is detected to be inaccessible, the reason and correction steps will
-    be shown.
-
-    If CLOUDS are specified, checks credentials for only those clouds.
-
-    The enabled clouds are cached and form the "search space" to be considered
-    for each task.
+    """Check which clouds are available to use for storage with Konduktor
 
+    \b
     Examples:
+        # Check only specific clouds - gs, s3.
+        konduktor check gs
+        konduktor check s3
 
-    .. code-block:: bash
-
-      # Check only specific clouds - gs, s3.
-      konduktor check gs
-      konduktor check s3
+    \b
+    Notes:
+        • This checks storage credentials for a cloud supported by konduktor.
+        If a cloud is detected to be inaccessible, the reason and correction
+        steps will be shown.
+        • If CLOUDS are specified, checks credentials for only those clouds.
+        • The enabled clouds are cached and form the "search space" to
+        be considered for each task.
     """
     clouds_arg = clouds if len(clouds) > 0 else None
     konduktor_check.check(clouds=clouds_arg)
@@ -1430,23 +1418,12 @@ def secret():
 
     USAGE: konduktor secret COMMAND
 
-    \b
-    Use one of the following COMMANDS:
-      create [FLAGS] [NAME]
-      delete [NAME]
-      list [FLAGS]
-
     \b
     Examples:
-      konduktor secret create --kind git-ssh --from-file=~/.ssh/id_rsa my-ssh-name
-      konduktor secret create --kind env --inline FOO=bar my-env-name
-      konduktor delete my-ssh-name
-      konduktor secret list
-
-    \b
-    For details on COMMAND ARGS:
-      konduktor secret create -h
-      konduktor secret list -h
+        konduktor secret create --kind git-ssh --from-file ~/.ssh/id_rsa my-ssh-name
+        konduktor secret create --kind env --inline FOO=bar my-env-name
+        konduktor secret delete my-ssh-name
+        konduktor secret list
     """
 
 
@@ -1637,8 +1614,7 @@ def delete(name):
     help='Show all secrets, including those not owned by the current user.',
 )
 def list_secrets(all_users: bool):
-    """List secrets in the namespace.
-    Defaults to only your secrets unless --all-users is set."""
+    """List secrets in the namespace."""
 
     context = kubernetes_utils.get_current_kube_config_context_name()
     namespace = kubernetes_utils.get_kube_config_context_namespace(context)
@@ -1683,23 +1659,11 @@ def serve():
 
     USAGE: konduktor serve COMMAND
 
-    \b
-    Use one of the following COMMANDS:
-      launch
-      down
-      status
-
     \b
     Examples:
       konduktor serve launch my-deployment
       konduktor serve down my-deployment
       konduktor serve status
-
-    \b
-    For details on COMMAND ARGS:
-      konduktor serve launch -h
-      konduktor serve down -h
-      konduktor serve status -h
     """
     pass
 
@@ -1774,8 +1738,10 @@ def serve_launch(
 ):
     """Launch a deployment to serve.
 
-    If ENTRYPOINT points to a valid YAML file, it is read in as the task
-    specification. Otherwise, it is interpreted as a bash command.
+    \b
+    Notes:
+        • If ENTRYPOINT points to a valid YAML file, it is read in as the task
+        specification. Otherwise, it is interpreted as a bash command.
     """
     # NOTE(dev): Keep the docstring consistent between the Python API and CLI.
     env = _merge_env_vars(env_file, env)
@@ -1857,13 +1823,10 @@ def serve_down(
 ):
     """Tear down deployments (Deployment, Service, PodAutoscaler).
 
-    Use --all or -a to tear down all deployments.
-
-    Examples:
-
     \b
-      konduktor serve down my-deployment
-      konduktor serve down -a
+    Examples:
+        konduktor serve down my-deployment
+        konduktor serve down -a
     """
     context = kubernetes_utils.get_current_kube_config_context_name()
     namespace = kubernetes_utils.get_kube_config_context_namespace(context)

{konduktor_nightly-0.1.0.dev20251124105105 → konduktor_nightly-0.1.0.dev20251211105235}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "konduktor-nightly"
-version = "0.1.0.dev20251124105105"
+version = "0.1.0.dev20251211105235"
 description = "GPU Cluster Health Management"
 packages = [
     {include = "konduktor"}
@@ -28,6 +28,7 @@ filelock = "^3.18.0"
 boto3 = { version = "^1.34.84", optional = true, extras = ["s3"] }
 botocore = { version = "^1.34.84", optional = true, extras = ["s3"] }
 awscli = { version = "^1.32.84", optional = true, extras = ["s3"] }
+sniffio = "^1.3"
 
 [tool.poetry.extras]
 s3 = ["boto3", "botocore", "awscli"]