xpk 0.11.0__py3-none-any.whl → 0.12.0__py3-none-any.whl

xpk/commands/cluster.py

@@ -92,7 +92,7 @@ def cluster_adapt(args) -> None:
 
   system, return_code = get_system_characteristics(args)
 
-  if return_code > 0:
+  if return_code > 0 or system is None:
     xpk_print('Fetching system characteristics failed!')
     xpk_exit(return_code)
 
@@ -141,8 +141,6 @@ def cluster_adapt(args) -> None:
     if not tensorboard_config:
       xpk_exit(1)
 
-  # Provision node pools dynamically based on incoming workloads:
-  # Currently autoprovisioning is not supported with Pathways.
   autoprovisioning_config = None
   if args.enable_autoprovisioning:
     xpk_print('Enabling Autoprovisioning')
@@ -201,7 +199,7 @@ def cluster_create(args) -> None:
   """
   system, return_code = get_system_characteristics(args)
 
-  if return_code > 0:
+  if return_code > 0 or system is None:
     xpk_print('Fetching system characteristics failed!')
     xpk_exit(return_code)
 
@@ -217,13 +215,13 @@ def cluster_create(args) -> None:
     xpk_exit(0)
 
   return_code, gke_server_config = get_gke_server_config(args)
-  if return_code != 0:
+  if return_code != 0 or gke_server_config is None:
     xpk_exit(return_code)
 
   return_code, gke_control_plane_version = get_gke_control_plane_version(
       args, gke_server_config
   )
-  if return_code != 0:
+  if return_code != 0 or gke_control_plane_version is None:
     xpk_exit(return_code)
 
   create_cluster_command_code = create_cluster_if_necessary(
@@ -294,7 +292,7 @@ def cluster_create(args) -> None:
   # Provision node pools dynamically based on incoming workloads:
   # Currently autoprovisioning is not supported with Pathways.
   autoprovisioning_config = None
-  if not args.enable_pathways and args.enable_autoprovisioning:
+  if args.enable_autoprovisioning:
     xpk_print('Enabling Autoprovisioning')
     autoprovisioning_config, return_code = enable_autoprovisioning_on_cluster(
         args, system
@@ -398,7 +396,7 @@ def cluster_cacheimage(args) -> None:
   get_cluster_credentials(args)
   system, return_code = get_system_characteristics(args)
 
-  if return_code > 0:
+  if return_code > 0 or system is None:
     xpk_print('Fetching system characteristics failed!')
     xpk_exit(return_code)
 
@@ -808,6 +806,7 @@ def scale_up_coredns(args, replicas: int = 15, namespace: str = 'kube-system'):
 
 def check_deployment_exists(args, deployment_name: str, namespace: str) -> bool:
   """Check for the existence of a specific Deployment in a given namespace."""
+  # TODO: rewrite this to be more obvious, check if it is correct
   command = (
       f'kubectl get deployment {deployment_name} -n'
       f' {namespace} --ignore-not-found'
@@ -815,11 +814,11 @@ def check_deployment_exists(args, deployment_name: str, namespace: str) -> bool:
   result = run_command_with_updates(
       command, 'Waiting for kubeDNS to be checked.', args
   )
-  return result
+  return result != 0
 
 
 def verify_coredns_readiness(
-    args, timeout: int = 120, namespace: str = 'kube-system'
+    args, timeout: int = 240, namespace: str = 'kube-system'
 ):
   """Verifies CoreDNS readiness using kubectl wait commands."""
   xpk_print('Now verifying CoreDNS readiness...')
@@ -874,7 +873,7 @@ def cleanup_coredns_repo(coredns_repo_full_path: str):
     xpk_print(f'Error deleting directory {coredns_repo_full_path}: {e}')
 
 
-def update_coredns(args):
+def update_coredns(args) -> int:
   """Updates and deploys CoreDNS within a cluster.
 
   Args:

xpk/commands/cluster_gcluster.py

@@ -310,4 +310,5 @@ def generate_blueprint(
           system_node_pool_machine_type=args.default_pool_cpu_machine_type,
           system_node_pool_min_node_count=args.default_pool_cpu_num_nodes,
       )
-  return None
+  xpk_print('Device type is not supported.')
+  xpk_exit(1)

xpk/commands/common.py

@@ -50,8 +50,8 @@ def set_cluster_command(args) -> int:
 
 
 def is_TAS_possible(
-    system_characteristics: SystemCharacteristics,
-    capacity_type: CapacityType,
+    system_characteristics: SystemCharacteristics | None,
+    capacity_type: CapacityType | None,
     flex: bool,
 ) -> bool:
   """Check cluster's machine_type and capacity type to determine if Kueue TAS is possible
@@ -71,7 +71,7 @@ def is_TAS_possible(
     xpk_print('capacity_type data was not found in configmaps.')
     xpk_exit(1)
 
-  if flex:
+  if not flex:
     return False
 
   if (

xpk/commands/info.py

@@ -51,19 +51,19 @@ def info(args: Namespace) -> None:
   cqs = run_kueuectl_list_clusterqueue(args)
   quotas = get_nominal_quotas(cqs)
 
-  if lq:
+  if lq and lqs is not None:
     print_formatted_lqs(lqs, quotas)
 
   if cq:
     print_formatted_cqs(cqs, quotas)
 
 
-def get_nominal_quotas(cqs: list[dict]) -> dict[str, dict[str, str]]:
+def get_nominal_quotas(cqs: str) -> dict[str, dict[str, str]]:
   """Get quotas from clusterqueues.
   This function retrieves how much of resource in each flavor is assigned to cluster queue.
   It parses flavors of passed cluster queues.
   Args:
-    - cqs - list of cluster queues.
+    - cqs - string containing a list of cluster queues in JSON format.
   Returns:
     - dictionary of cluster queues resources quotas in format:
     {cq_name:{"flavorName:resourceName":quota}}
@@ -75,7 +75,7 @@ def get_nominal_quotas(cqs: list[dict]) -> dict[str, dict[str, str]]:
     xpk_print(cqs)
     xpk_exit(1)
 
-  quotas = {}
+  quotas: dict[str, dict] = {}
   for cq in cq_list:
     spec = cq['spec']
     cq_name = cq['metadata']['name']
@@ -89,7 +89,7 @@ def get_nominal_quotas(cqs: list[dict]) -> dict[str, dict[str, str]]:
   return quotas
 
 
-def print_formatted_cqs(cqs: list[dict], nominalQuotas) -> None:
+def print_formatted_cqs(cqs: str, nominalQuotas) -> None:
   try:
     cq_list = json.loads(cqs)['items']
   except ValueError:
@@ -105,7 +105,7 @@ def print_formatted_cqs(cqs: list[dict], nominalQuotas) -> None:
   )
 
 
-def print_formatted_lqs(lqs: list[dict], nominalQuotas) -> None:
+def print_formatted_lqs(lqs: str, nominalQuotas) -> None:
   try:
     lq_list = json.loads(lqs)['items']
   except ValueError:
@@ -143,18 +143,18 @@ def parse_queue_lists(
 
 
 def get_flavors_resources_reservations(
-    cq_name: str, flavors_res: list[dict]
+    cq_name: str, flavors_res: dict
 ) -> dict[str, dict[str, str]]:
   """Get usage of flavors resources.
   This function parser flavorsReservation section of clusterQueue of LocalQueue.
   Args:
     - cq_name - name of ClusterQueue to which flavors belong.
-    - flavors_res - list of reservations made by flavors
+    - flavors_res - dict of reservations made by flavors
   Returns:
     Dict containing usage of each resource in flavor for each flavor in cluster or local queue.
     Dict format: {cq_name: {{flavor:resource}:reservation}}
   """
-  reservations = {}
+  reservations: dict[str, dict] = {}
   reservations[cq_name] = {}
   for flavor_name, flavor_resources_reservation_list in flavors_res.items():
     for resource in flavor_resources_reservation_list:
@@ -167,15 +167,15 @@ def get_flavors_resources_reservations(
 
 def get_flavors_usage(
     q_entry: dict, res_field: str, flavor_resource_quotas: dict
-) -> list[dict]:
+) -> dict[str, str]:
   """Parse q_entry to retrieve list of each resource usage in flavour.
   Args:
     q_entry - single entry into either LocalQueue or ClusterQueue structured as json
     flavor_resource_quotas - nominalQuota of flavors resource usage for each clusterqueue
   Returns:
-    list of dicts where each list entry is in format (key, entry) where:
+    Dict where for each (key, value):
     - key is flavorName:resourceName
-    - entry is flavorResourceReservation/flavorResourceQuota
+    - value is string formatted as 'flavorResourceReservation/flavorResourceQuota'
   """
   status = q_entry['status']
   flavors_res = status[res_field]

xpk/commands/job.py

@@ -18,6 +18,7 @@ import re
 import sys
 
 from ruamel.yaml import YAML
+from typing import cast
 
 from ..core.commands import run_command_for_value, run_command_with_updates
 from ..core.cluster import get_cluster_credentials
@@ -84,7 +85,7 @@ def job_info(args):
 
 
 def get_profile(job_yaml: dict) -> str:
-  containers = (
+  containers: list[dict] = (
       job_yaml.get('spec', {})
       .get('template', {})
       .get('spec', {})
@@ -96,13 +97,13 @@ def get_profile(job_yaml: dict) -> str:
 
 
 def get_mounts(job_yaml: dict) -> list[dict]:
-  containers = (
+  containers: list[dict] = (
       job_yaml.get('spec', {})
       .get('template', {})
       .get('spec', {})
       .get('containers', [])
   )
-  mounts = next(iter(containers), {}).get('volumeMounts', [])
+  mounts: list[dict] = next(iter(containers), {}).get('volumeMounts', [])
   return mounts
 
 
@@ -112,23 +113,24 @@ def get_kjob_env_vars(job_desc_text: str) -> list[tuple[str, str]]:
   return search_res
 
 
-def get_pods(pods_text: str) -> list[str]:
+def get_pods(pods_text: str) -> list[dict[str, str]]:
   pods_lines = pods_text.strip().split('\n')
-  pods_lines = [line.split() for line in pods_lines]
+  pods_lines_tokenized = [line.split() for line in pods_lines]
   return [
       {
-          'Name': line[0],
-          'Status': line[2],
+          'Name': tokens[0],
+          'Status': tokens[2],
       }
-      for line in pods_lines
+      for tokens in pods_lines_tokenized
   ]
 
 
 def get_script_name(job_yaml: dict) -> str | None:
-  return (
+  return cast(
+      str | None,
       job_yaml.get('metadata', {})
       .get('annotations', {})
-      .get('kjobctl.x-k8s.io/script', '')
+      .get('kjobctl.x-k8s.io/script', ''),
   )
 
 

xpk/commands/kjob_common.py

@@ -33,6 +33,7 @@ from ..core.resources import get_cluster_capacity_type, get_cluster_system_chara
 def add_gpu_networking_annotations_to_command(args, cmd: str) -> str:
   gpu_type = get_gpu_type_from_cluster(args)
 
+  annotations: tuple
   if gpu_type == H100_MEGA_DEVICE_TYPE:
     annotations = get_a3mega_pod_template_annotations(args)
   elif gpu_type == H200_DEVICE_TYPE:
@@ -40,7 +41,7 @@ def add_gpu_networking_annotations_to_command(args, cmd: str) -> str:
   elif gpu_type == B200_DEVICE_TYPE:
     annotations = get_a4_pod_template_annotations(args)
   else:
-    annotations = []
+    annotations = tuple()
 
   flags = [
       f" --pod-template-annotation {annotation} " for annotation in annotations

xpk/commands/storage.py

@@ -141,7 +141,7 @@ def storage_delete(args: Namespace) -> None:
 
 def storage_attach(args: Namespace) -> None:
   add_zone_and_project(args)
-  manifest = [{}]
+  manifest: list[dict] = [{}]
   if args.type == GCP_FILESTORE_TYPE:
     if args.instance is None:
       args.instance = args.name

xpk/commands/workload.py

@@ -84,6 +84,7 @@ from ..core.system_characteristics import (
 from ..core.vertex import create_vertex_experiment
 from ..core.workload import (
     check_if_workload_exists,
+    get_jobsets_list_gcp_link,
     get_workload_list,
     wait_for_job_completion,
     zone_to_region,
@@ -226,7 +227,8 @@ spec:
             metadata:
               labels:
                 xpk.google.com/workload: {args.workload}
-              annotations: {annotations}
+              annotations:
+                {annotations}
             spec:
               priorityClassName: {args.priority}
               restartPolicy: Never
@@ -319,7 +321,7 @@ def workload_create(args) -> None:
   xpk_print('Starting workload create', flush=True)
   system, return_code = get_system_characteristics(args)
 
-  if return_code > 0:
+  if return_code > 0 or system is None:
     xpk_print('Fetching system characteristics failed!')
     xpk_exit(return_code)
 
@@ -345,7 +347,7 @@ def workload_create(args) -> None:
   ):
     xpk_print(
         'Warning: Cluster has been created using XPK version:'
-        f' {cluster_config_map["xpk_version"]} but the XPK version you are'
+        f' {cluster_xpk_version} but the XPK version you are'
         f' using to schedule workload is: {XPK_CURRENT_VERSION}. Some features'
         ' might not be available for this cluster. We recommend to'
         ' upgrade/downgrade your XPK version or cluster by running `xpk'
@@ -354,7 +356,7 @@ def workload_create(args) -> None:
 
   debugging_dashboard_id = None
 
-  tensorboard_config = {}
+  tensorboard_config: dict | None = {}
   if VERTEX_TENSORBOARD_FEATURE_FLAG and args.use_vertex_tensorboard:
     tensorboard_config = create_vertex_experiment(args)
     # exit if failed to create Experiment in Vertex AI
@@ -450,8 +452,8 @@ def workload_create(args) -> None:
       - action: FailJobSet
         onJobFailureReasons:
         - PodFailurePolicy"""
-    restart_on_exit_codes = get_restart_exit_codes(args)
-    restart_on_exit_codes = ','.join(map(str, restart_on_exit_codes))
+    restart_on_exit_codes_list = get_restart_exit_codes(args)
+    restart_on_exit_codes = ','.join(map(str, restart_on_exit_codes_list))
     pod_failure_policy = f"""
           podFailurePolicy:
             rules:
@@ -760,4 +762,8 @@ def workload_list(args) -> None:
     xpk_print(f'List Job request returned ERROR {return_code}')
     xpk_exit(return_code)
   xpk_print(f'Workload List Output:\n{return_value}')
+
+  workload_list_gcp_link = get_jobsets_list_gcp_link(project=args.project)
+  xpk_print(f'See your workloads in Cloud Console: {workload_list_gcp_link}')
+
   xpk_exit(0)

xpk/core/blueprint/blueprint_generator.py

@@ -34,7 +34,7 @@ from ..system_characteristics import get_system_characteristics_by_device_type
 from .blueprint_definitions import Blueprint, DeploymentGroup, DeploymentModule
 from ..kueue import KUEUE_VERSION
 
-yaml = yaml.YAML()
+yaml_parser = yaml.YAML()
 
 a3high_device_type = H100_DEVICE_TYPE
 a3mega_device_type = H100_MEGA_DEVICE_TYPE
@@ -52,7 +52,7 @@ blueprint_dependencies_dir = {
 }
 
 cluster_toolkit_url = "github.com/GoogleCloudPlatform/cluster-toolkit"
-cluster_toolkit_version = "v1.57.1"
+cluster_toolkit_version = "v1.62.2"
 
 
 class BlueprintGeneratorOutput:
@@ -1019,7 +1019,7 @@ class BlueprintGenerator:
   ) -> str:
     blueprint_path = self._get_blueprint_path(blueprint_name, prefix)
     with open(blueprint_path, "w+", encoding="utf-8") as blueprint_file:
-      yaml.dump(xpk_blueprint, blueprint_file)
+      yaml_parser.dump(xpk_blueprint, blueprint_file)
     return blueprint_path
 
   def _get_blueprint_path(self, blueprint_name, prefix: str = ""):
@@ -1033,7 +1033,7 @@ class BlueprintGenerator:
     ensure_directory_exists(storage_path_with_prefix)
     return storage_path_with_prefix
 
-  def blueprint_exists(self, blueprint_name, prefix: str = ""):
+  def blueprint_exists(self, blueprint_name, prefix: str = "") -> bool:
     blueprint_path = self._get_blueprint_path(blueprint_name, prefix)
     return os.path.exists(blueprint_path)
 
@@ -1061,6 +1061,6 @@ class BlueprintGenerator:
     }
 
 
-yaml.register_class(Blueprint)
-yaml.register_class(DeploymentGroup)
-yaml.register_class(DeploymentModule)
+yaml_parser.register_class(Blueprint)
+yaml_parser.register_class(DeploymentGroup)
+yaml_parser.register_class(DeploymentModule)

xpk/core/blueprint/blueprint_test.py

@@ -0,0 +1,218 @@
+"""
+Copyright 2024 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import os
+import shutil
+
+import ruamel.yaml
+
+from xpk.core.blueprint.blueprint_definitions import Blueprint
+from xpk.core.blueprint.blueprint_generator import BlueprintGenerator
+from xpk.core.capacity import CapacityType
+
+yaml = ruamel.yaml.YAML()
+
+yaml.register_class(Blueprint)
+
+a3_yaml_test_path = "src/xpk/core/blueprint/testing/data/a3_mega.yaml"
+a3_spot_yaml_test_path = "src/xpk/core/blueprint/testing/data/a3_mega_spot.yaml"
+a3_ultra_yaml_test_path = "src/xpk/core/blueprint/testing/data/a3_ultra.yaml"
+a4_yaml_test_path = "src/xpk/core/blueprint/testing/data/a4.yaml"
+config_map_filename = "config-map.yaml.tftpl"
+kueue_conf_filename = "kueue-xpk-configuration.yaml.tftpl"
+tmp_test_dir = "/tmp/xpk_test"
+
+
+def prepare_test():
+  if os.path.exists(tmp_test_dir):
+    shutil.rmtree(tmp_test_dir)
+  os.mkdir(tmp_test_dir)
+
+
+def test_generate_a3_mega_blueprint():
+  prepare_test()
+  blueprint_name = "xpk-gke-a3-megagpu"
+  bp_generator = BlueprintGenerator(tmp_test_dir)
+  bp = bp_generator.generate_a3_mega_blueprint(
+      project_id="foo",
+      cluster_name="bar",
+      blueprint_name=blueprint_name,
+      prefix="prefix",
+      region="us-central1",
+      zone="us-central1-c",
+      auth_cidr="10.0.0.0/32",
+      reservation_placement_policy={
+          "type": "COMPACT",
+          "name": "test-reservation-placement",
+      },
+      reservation="test-reservation",
+      capacity_type=CapacityType.RESERVATION,
+      system_node_pool_min_node_count=5,
+  )
+
+  assert bp.blueprint_file.endswith("/prefix/xpk-gke-a3-megagpu.yaml")
+
+  with open(a3_yaml_test_path, encoding="utf-8") as stream:
+    ctk_yaml = yaml.load(stream)
+    with open(bp.blueprint_file, encoding="utf-8") as generated_blueprint:
+      ctk_test = yaml.load(generated_blueprint)
+      assert ctk_yaml.blueprint_name == ctk_test.blueprint_name
+      assert ctk_test.terraform_backend_defaults is None
+      assert ctk_yaml.toolkit_modules_url == ctk_test.toolkit_modules_url
+      assert (
+          ctk_yaml.toolkit_modules_version == ctk_test.toolkit_modules_version
+      )
+      assert ctk_yaml.vars == ctk_test.vars
+      assert ctk_test.deployment_groups == ctk_yaml.deployment_groups
+      assert os.path.exists(
+          os.path.join(
+              tmp_test_dir, "prefix", blueprint_name, config_map_filename
+          )
+      )
+      assert os.path.exists(
+          os.path.join(
+              tmp_test_dir, "prefix", blueprint_name, kueue_conf_filename
+          )
+      )
+
+  shutil.rmtree(tmp_test_dir)
+
+
+def test_generate_a3_mega_spot_blueprint():
+  prepare_test()
+  blueprint_name = "xpk-gke-a3-megagpu"
+  bp_generator = BlueprintGenerator(tmp_test_dir)
+  bp = bp_generator.generate_a3_mega_blueprint(
+      project_id="foo",
+      cluster_name="bar",
+      blueprint_name=blueprint_name,
+      prefix="prefix",
+      region="us-central1",
+      zone="us-central1-c",
+      auth_cidr="10.0.0.0/32",
+      capacity_type=CapacityType.SPOT,
+      system_node_pool_min_node_count=5,
+  )
+
+  assert bp.blueprint_file.endswith("/prefix/xpk-gke-a3-megagpu.yaml")
+
+  with open(a3_spot_yaml_test_path, encoding="utf-8") as stream:
+    ctk_yaml = yaml.load(stream)
+    with open(bp.blueprint_file, encoding="utf-8") as generated_blueprint:
+      ctk_test = yaml.load(generated_blueprint)
+      assert ctk_yaml.blueprint_name == ctk_test.blueprint_name
+      assert ctk_test.terraform_backend_defaults is None
+      assert ctk_yaml.toolkit_modules_url == ctk_test.toolkit_modules_url
+      assert (
+          ctk_yaml.toolkit_modules_version == ctk_test.toolkit_modules_version
+      )
+      assert ctk_yaml.vars == ctk_test.vars
+      assert ctk_test.deployment_groups == ctk_yaml.deployment_groups
+
+  shutil.rmtree(tmp_test_dir)
+
+
+def test_generate_a3_ultra_blueprint():
+  prepare_test()
+  blueprint_name = "xpk-gke-a3-ultra"
+  bp_generator = BlueprintGenerator(tmp_test_dir)
+  bp = bp_generator.generate_a3_ultra_blueprint(
+      project_id="foo",
+      cluster_name="gke-a3-ultra",
+      blueprint_name=blueprint_name,
+      region="us-central1",
+      zone="us-central1-c",
+      auth_cidr="10.0.0.0/32",
+      reservation="test-reservation",
+      system_node_pool_machine_type="e2-standard-16",
+      capacity_type=CapacityType.RESERVATION,
+      gcs_bucket="test-bucket",
+      prefix="testdir",
+  )
+  with open(a3_ultra_yaml_test_path, encoding="utf-8") as stream:
+    ctk_yaml = yaml.load(stream)
+    with open(bp.blueprint_file, encoding="utf-8") as generated_blueprint:
+      ctk_test = yaml.load(generated_blueprint)
+      assert ctk_yaml.blueprint_name == ctk_test.blueprint_name
+      assert (
+          ctk_yaml.terraform_backend_defaults
+          == ctk_test.terraform_backend_defaults
+      )
+      assert ctk_yaml.toolkit_modules_url == ctk_test.toolkit_modules_url
+      assert (
+          ctk_yaml.toolkit_modules_version == ctk_test.toolkit_modules_version
+      )
+      assert ctk_test.deployment_groups == ctk_yaml.deployment_groups
+      assert os.path.exists(
+          os.path.join(
+              tmp_test_dir, "testdir", blueprint_name, "mlgru-disable.yaml"
+          )
+      )
+      assert os.path.exists(
+          os.path.join(
+              tmp_test_dir, "testdir", blueprint_name, "nccl-installer.yaml"
+          )
+      )
+
+  shutil.rmtree(tmp_test_dir)
+
+
+def test_generate_a4_blueprint():
+  prepare_test()
+  blueprint_name = "xpk-gke-a4"
+  bp_generator = BlueprintGenerator(tmp_test_dir)
+  bp = bp_generator.generate_a4_blueprint(
+      project_id="foo",
+      cluster_name="gke-a4",
+      blueprint_name=blueprint_name,
+      region="us-central1",
+      zone="us-central1-c",
+      auth_cidr="10.0.0.0/32",
+      reservation="test-reservation",
+      system_node_pool_machine_type="e2-standard-16",
+      capacity_type=CapacityType.RESERVATION,
+      gcs_bucket="test-bucket",
+      prefix="testdir",
+  )
+  with open(a4_yaml_test_path, encoding="utf-8") as stream:
+    ctk_yaml = yaml.load(stream)
+    with open(bp.blueprint_file, encoding="utf-8") as generated_blueprint:
+      ctk_test = yaml.load(generated_blueprint)
+      assert ctk_yaml.blueprint_name == ctk_test.blueprint_name
+      assert (
+          ctk_yaml.terraform_backend_defaults
+          == ctk_test.terraform_backend_defaults
+      )
+      assert ctk_yaml.toolkit_modules_url == ctk_test.toolkit_modules_url
+      assert (
+          ctk_yaml.toolkit_modules_version == ctk_test.toolkit_modules_version
+      )
+      assert ctk_test.deployment_groups == ctk_yaml.deployment_groups
+      assert os.path.exists(
+          os.path.join(
+              tmp_test_dir, "testdir", blueprint_name, "storage_crd.yaml"
+          )
+      )
+      assert os.path.exists(
+          os.path.join(
+              tmp_test_dir,
+              "testdir",
+              blueprint_name,
+              "nccl-rdma-installer-a4.yaml",
+          )
+      )
+
+  shutil.rmtree(tmp_test_dir)

xpk/core/capacity.py

@@ -195,10 +195,12 @@ def get_capacity_arguments_from_capacity_type(
       capacity_args = '--spot'
     case CapacityType.FLEX_START:
       capacity_args = (
-          ' --flex-start --enable-queued-provisioning --enable-autoscaling'
+          ' --flex-start --enable-autoscaling'
           ' --location-policy=ANY --reservation-affinity=none'
           f' --no-enable-autorepair --max-nodes={max_nodes}'
       )
+      if args.num_slices <= 1:
+        capacity_args += ' --enable-queued-provisioning'
     case CapacityType.RESERVATION:
       capacity_args = (
           f'--reservation-affinity=specific --reservation={args.reservation}'