xpk 1.1.1__tar.gz → 1.2.0__tar.gz

{xpk-1.1.1 → xpk-1.2.0}/Makefile

@@ -6,6 +6,7 @@ KUEUECTL_URL = "https://github.com/kubernetes-sigs/kueue/releases/download/$(KUE
 
 PROJECT_DIR := $(realpath $(shell dirname $(firstword $(MAKEFILE_LIST))))
 BIN_PATH=$(PROJECT_DIR)/bin
+PIP_OPTS ?=
 
 .PHONY: install
 install: check-python check-gcloud install-gcloud-auth-plugin install-kueuectl pip-install
@@ -15,11 +16,11 @@ install-dev: check-python check-gcloud mkdir-bin install-kueuectl pip-install pi
 
 .PHONY: pip-install-dev
 pip-install-dev:
-	pip install -e ".[dev]"
+	pip install $(PIP_OPTS) -e ".[dev]"
 
 .PHONY: pip-install
 pip-install:
-	pip install -e .
+	pip install $(PIP_OPTS) -e .
 
 .PHONY: install-pytest
 install-pytest:

{xpk-1.1.1/src/xpk.egg-info → xpk-1.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: xpk
-Version: 1.1.1
+Version: 1.2.0
 Summary: xpk helps Cloud developers to orchestrate training jobs on accelerators on GKE.
 Author-email: XPK team <xpk-code-reviewers@google.com>
 License: Apache-2.0

{xpk-1.1.1 → xpk-1.2.0}/recipes/Basic_cluster_adapt.md

@@ -134,8 +134,8 @@ description: "Very High"
 kubectl apply -f 6083d72fc3ba2ac7d243c1269dd67717abd4086bf64e397e3a1737de415dd133
 [XPK] Task: `Count total nodes` is implemented by the following command not running since it is a dry run. 
 kubectl get node --no-headers | wc -l
-[XPK] Try 1: Updating Kueue Controller Manager resources
-[XPK] Task: `Updating Kueue Controller Manager resources` is implemented by the following command not running since it is a dry run. 
+[XPK] Try 1: Updating Controller Manager resources
+[XPK] Task: `Updating Controller Manager resources` is implemented by the following command not running since it is a dry run. 
 kubectl patch deployment kueue-controller-manager -n kueue-system --type='strategic' --patch='{"spec": {"template": {"spec": {"containers": [{"name": "manager", "resources": {"limits": {"memory": "4096Mi"}}}]}}}}'
 [XPK] GKE commands done! Resources are created.
 [XPK] See your GKE Cluster here: https://console.cloud.google.com/kubernetes/clusters/details/us-central1/golden-cluster/details?project=golden-project

{xpk-1.1.1 → xpk-1.2.0}/recipes/Basic_cluster_create.md

@@ -266,8 +266,8 @@ description: "Very High"
 kubectl apply -f 6083d72fc3ba2ac7d243c1269dd67717abd4086bf64e397e3a1737de415dd133
 [XPK] Task: `Count total nodes` is implemented by the following command not running since it is a dry run. 
 kubectl get node --no-headers | wc -l
-[XPK] Try 1: Updating Kueue Controller Manager resources
-[XPK] Task: `Updating Kueue Controller Manager resources` is implemented by the following command not running since it is a dry run. 
+[XPK] Try 1: Updating Controller Manager resources
+[XPK] Task: `Updating Controller Manager resources` is implemented by the following command not running since it is a dry run. 
 kubectl patch deployment kueue-controller-manager -n kueue-system --type='strategic' --patch='{"spec": {"template": {"spec": {"containers": [{"name": "manager", "resources": {"limits": {"memory": "4096Mi"}}}]}}}}'
 [XPK] GKE commands done! Resources are created.
 [XPK] See your GKE Cluster here: https://console.cloud.google.com/kubernetes/clusters/details/us-central1/golden-cluster/details?project=golden-project

{xpk-1.1.1 → xpk-1.2.0}/recipes/Cluster_create_RayCluster.md

@@ -273,8 +273,8 @@ description: "Very High"
 kubectl apply -f 6083d72fc3ba2ac7d243c1269dd67717abd4086bf64e397e3a1737de415dd133
 [XPK] Task: `Count total nodes` is implemented by the following command not running since it is a dry run. 
 kubectl get node --no-headers | wc -l
-[XPK] Try 1: Updating Kueue Controller Manager resources
-[XPK] Task: `Updating Kueue Controller Manager resources` is implemented by the following command not running since it is a dry run. 
+[XPK] Try 1: Updating Controller Manager resources
+[XPK] Task: `Updating Controller Manager resources` is implemented by the following command not running since it is a dry run. 
 kubectl patch deployment kueue-controller-manager -n kueue-system --type='strategic' --patch='{"spec": {"template": {"spec": {"containers": [{"name": "manager", "resources": {"limits": {"memory": "4096Mi"}}}]}}}}'
 [XPK] Try 1: Deleting old RayCluster
 [XPK] Task: `Deleting old RayCluster` is implemented by the following command not running since it is a dry run. 

{xpk-1.1.1 → xpk-1.2.0}/recipes/Cluster_create_for_multi-host_nodepool.md

@@ -268,8 +268,8 @@ description: "Very High"
 kubectl apply -f b58f50dd88cb1211d51276b9b445f6bca02f0e97fa984656d47992aecd9322cc
 [XPK] Task: `Count total nodes` is implemented by the following command not running since it is a dry run. 
 kubectl get node --no-headers | wc -l
-[XPK] Try 1: Updating Kueue Controller Manager resources
-[XPK] Task: `Updating Kueue Controller Manager resources` is implemented by the following command not running since it is a dry run. 
+[XPK] Try 1: Updating Controller Manager resources
+[XPK] Task: `Updating Controller Manager resources` is implemented by the following command not running since it is a dry run. 
 kubectl patch deployment kueue-controller-manager -n kueue-system --type='strategic' --patch='{"spec": {"template": {"spec": {"containers": [{"name": "manager", "resources": {"limits": {"memory": "4096Mi"}}}]}}}}'
 [XPK] GKE commands done! Resources are created.
 [XPK] See your GKE Cluster here: https://console.cloud.google.com/kubernetes/clusters/details/us-central1/golden-cluster/details?project=golden-project

{xpk-1.1.1 → xpk-1.2.0}/recipes/Cluster_create_for_single-host_nodepool.md

@@ -266,8 +266,8 @@ description: "Very High"
 kubectl apply -f f228edecda8022002fe1876e83ebf4c0c280eb4aeb0f72da3a5d746b5dfb1c91
 [XPK] Task: `Count total nodes` is implemented by the following command not running since it is a dry run. 
 kubectl get node --no-headers | wc -l
-[XPK] Try 1: Updating Kueue Controller Manager resources
-[XPK] Task: `Updating Kueue Controller Manager resources` is implemented by the following command not running since it is a dry run. 
+[XPK] Try 1: Updating Controller Manager resources
+[XPK] Task: `Updating Controller Manager resources` is implemented by the following command not running since it is a dry run. 
 kubectl patch deployment kueue-controller-manager -n kueue-system --type='strategic' --patch='{"spec": {"template": {"spec": {"containers": [{"name": "manager", "resources": {"limits": {"memory": "4096Mi"}}}]}}}}'
 [XPK] GKE commands done! Resources are created.
 [XPK] See your GKE Cluster here: https://console.cloud.google.com/kubernetes/clusters/details/us-central1/golden-cluster/details?project=golden-project

{xpk-1.1.1 → xpk-1.2.0}/recipes/Cluster_create_private.md

@@ -285,8 +285,8 @@ description: "Very High"
 kubectl apply -f 2e0015f210b664c3b767ae4e11af51387b01d4d6b36e20fecbdee137d3d2700b
 [XPK] Task: `Count total nodes` is implemented by the following command not running since it is a dry run. 
 kubectl get node --no-headers | wc -l
-[XPK] Try 1: Updating Kueue Controller Manager resources
-[XPK] Task: `Updating Kueue Controller Manager resources` is implemented by the following command not running since it is a dry run. 
+[XPK] Try 1: Updating Controller Manager resources
+[XPK] Task: `Updating Controller Manager resources` is implemented by the following command not running since it is a dry run. 
 kubectl patch deployment kueue-controller-manager -n kueue-system --type='strategic' --patch='{"spec": {"template": {"spec": {"containers": [{"name": "manager", "resources": {"limits": {"memory": "4096Mi"}}}]}}}}'
 [XPK] GKE commands done! Resources are created.
 [XPK] See your GKE Cluster here: https://console.cloud.google.com/kubernetes/clusters/details/us-central1/golden-cluster-private/details?project=golden-project

{xpk-1.1.1 → xpk-1.2.0}/recipes/Cluster_create_sub-slicing.md

@@ -290,8 +290,8 @@ spec:
 kubectl apply -f 2f2b4591858b4bc50348c575cd2cc048c79d1e4ffb67e0a6d6e1eafad21c5002
 [XPK] Task: `Count total nodes` is implemented by the following command not running since it is a dry run. 
 kubectl get node --no-headers | wc -l
-[XPK] Try 1: Updating Kueue Controller Manager resources
-[XPK] Task: `Updating Kueue Controller Manager resources` is implemented by the following command not running since it is a dry run. 
+[XPK] Try 1: Updating Controller Manager resources
+[XPK] Task: `Updating Controller Manager resources` is implemented by the following command not running since it is a dry run. 
 kubectl patch deployment kueue-controller-manager -n kueue-system --type='strategic' --patch='{"spec": {"template": {"spec": {"containers": [{"name": "manager", "resources": {"limits": {"memory": "4096Mi"}}}]}}}}'
 [XPK] GKE commands done! Resources are created.
 [XPK] See your GKE Cluster here: https://console.cloud.google.com/kubernetes/clusters/details/us-central1/golden-cluster/details?project=golden-project

{xpk-1.1.1 → xpk-1.2.0}/recipes/Cluster_create_super-slicing.md

@@ -98,6 +98,9 @@ data:
 [XPK] Try 1: Install Jobset on golden-cluster
 [XPK] Task: `Install Jobset on golden-cluster` is implemented by the following command not running since it is a dry run. 
 kubectl apply --server-side --force-conflicts -f https://github.com/kubernetes-sigs/jobset/releases/download/v0.8.0/manifests.yaml
+[XPK] Try 1: Updating Controller Manager resources
+[XPK] Task: `Updating Controller Manager resources` is implemented by the following command not running since it is a dry run. 
+kubectl patch deployment jobset-controller-manager -n jobset-system --type='strategic' --patch='{"spec": {"template": {"spec": {"containers": [{"name": "manager", "resources": {"requests": {"cpu": "4", "memory": "16Gi"}, "limits": {"cpu": "4", "memory": "16Gi"}}}]}}}}'
 [XPK] Task: `Count total nodes` is implemented by the following command not running since it is a dry run. 
 kubectl get node --no-headers | wc -l
 [XPK] Temp file (1b31e624e490f9c8c4ef4e369f08d3fa467990af5a261e4405bd045265d70e95) content: 
@@ -299,11 +302,9 @@ spec:
   - nodeLabel: kubernetes.io/hostname
 [XPK] Task: `Applying Kueue Custom Resources` is implemented by the following command not running since it is a dry run. 
 kubectl apply -f 6df31e8df3d8970d7ed3bf3aa948ae7cea9487c15ed6cfb1577ca6c948cf5525
-[XPK] Task: `Count total nodes` is implemented by the following command not running since it is a dry run. 
-kubectl get node --no-headers | wc -l
-[XPK] Try 1: Updating Kueue Controller Manager resources
-[XPK] Task: `Updating Kueue Controller Manager resources` is implemented by the following command not running since it is a dry run. 
-kubectl patch deployment kueue-controller-manager -n kueue-system --type='strategic' --patch='{"spec": {"template": {"spec": {"containers": [{"name": "manager", "resources": {"limits": {"memory": "4096Mi"}}}]}}}}'
+[XPK] Try 1: Updating Controller Manager resources
+[XPK] Task: `Updating Controller Manager resources` is implemented by the following command not running since it is a dry run. 
+kubectl patch deployment kueue-controller-manager -n kueue-system --type='strategic' --patch='{"spec": {"replicas": 3, "template": {"spec": {"containers": [{"name": "manager", "resources": {"requests": {"cpu": "16", "memory": "64Gi"}, "limits": {"cpu": "16", "memory": "64Gi"}}}]}}}}'
 [XPK] GKE commands done! Resources are created.
 [XPK] See your GKE Cluster here: https://console.cloud.google.com/kubernetes/clusters/details/us-central1/golden-cluster/details?project=golden-project
 [XPK] Exiting XPK cleanly

{xpk-1.1.1 → xpk-1.2.0}/recipes/Cluster_create_with_CPU_and_memory_limits_above_capacity.md

@@ -270,8 +270,8 @@ description: "Very High"
 kubectl apply -f 1ce6c42efe0834ff0519978ad09539c725a5d6f22267c5f1b41b6e458668e45f
 [XPK] Task: `Count total nodes` is implemented by the following command not running since it is a dry run. 
 kubectl get node --no-headers | wc -l
-[XPK] Try 1: Updating Kueue Controller Manager resources
-[XPK] Task: `Updating Kueue Controller Manager resources` is implemented by the following command not running since it is a dry run. 
+[XPK] Try 1: Updating Controller Manager resources
+[XPK] Task: `Updating Controller Manager resources` is implemented by the following command not running since it is a dry run. 
 kubectl patch deployment kueue-controller-manager -n kueue-system --type='strategic' --patch='{"spec": {"template": {"spec": {"containers": [{"name": "manager", "resources": {"limits": {"memory": "4096Mi"}}}]}}}}'
 [XPK] GKE commands done! Resources are created.
 [XPK] See your GKE Cluster here: https://console.cloud.google.com/kubernetes/clusters/details/us-central1/golden-cluster/details?project=golden-project

{xpk-1.1.1 → xpk-1.2.0}/recipes/Cluster_create_with_CPU_and_memory_limits_below_capacity.md

@@ -270,8 +270,8 @@ description: "Very High"
 kubectl apply -f 1ce6c42efe0834ff0519978ad09539c725a5d6f22267c5f1b41b6e458668e45f
 [XPK] Task: `Count total nodes` is implemented by the following command not running since it is a dry run. 
 kubectl get node --no-headers | wc -l
-[XPK] Try 1: Updating Kueue Controller Manager resources
-[XPK] Task: `Updating Kueue Controller Manager resources` is implemented by the following command not running since it is a dry run. 
+[XPK] Try 1: Updating Controller Manager resources
+[XPK] Task: `Updating Controller Manager resources` is implemented by the following command not running since it is a dry run. 
 kubectl patch deployment kueue-controller-manager -n kueue-system --type='strategic' --patch='{"spec": {"template": {"spec": {"containers": [{"name": "manager", "resources": {"limits": {"memory": "4096Mi"}}}]}}}}'
 [XPK] GKE commands done! Resources are created.
 [XPK] See your GKE Cluster here: https://console.cloud.google.com/kubernetes/clusters/details/us-central1/golden-cluster/details?project=golden-project

{xpk-1.1.1 → xpk-1.2.0}/recipes/Cluster_create_with_Managed_Lustre_driver.md

@@ -47,6 +47,12 @@ gcloud container clusters describe golden-cluster --project=golden-project --loc
 [XPK] Updating GKE cluster to enable Lustre CSI driver, may take a while!
 [XPK] Task: `GKE Cluster Update to enable Lustre CSI driver` is implemented by the following command not running since it is a dry run. 
 gcloud container clusters update golden-cluster --project=golden-project --location=us-central1 --quiet --update-addons=LustreCsiDriver=ENABLED
+[XPK] Recreating existing nodes (if any) to complete the Lustre CSI driver installation.
+[XPK] Task: `Get All Node Pools` is implemented by the following command not running since it is a dry run. 
+gcloud beta container node-pools list --cluster golden-cluster --project=golden-project --location=us-central1 --format="csv[no-heading](name)"
+[XPK] To complete NodesRecreate-0 we are executing gcloud container clusters upgrade golden-cluster --project=golden-project --node-pool=0 --location=us-central1 --quiet
+[XPK] Breaking up a total of 1 commands into 1 batches
+[XPK] Pretending all the jobs succeeded
 [XPK] Task: `Determine current gke master version` is implemented by the following command not running since it is a dry run. 
 gcloud beta container clusters describe golden-cluster --location us-central1 --project golden-project --format="value(currentMasterVersion)"
 [XPK] Creating 1 node pool or pools of tpu7x-8
@@ -271,8 +277,8 @@ description: "Very High"
 kubectl apply -f 6083d72fc3ba2ac7d243c1269dd67717abd4086bf64e397e3a1737de415dd133
 [XPK] Task: `Count total nodes` is implemented by the following command not running since it is a dry run. 
 kubectl get node --no-headers | wc -l
-[XPK] Try 1: Updating Kueue Controller Manager resources
-[XPK] Task: `Updating Kueue Controller Manager resources` is implemented by the following command not running since it is a dry run. 
+[XPK] Try 1: Updating Controller Manager resources
+[XPK] Task: `Updating Controller Manager resources` is implemented by the following command not running since it is a dry run. 
 kubectl patch deployment kueue-controller-manager -n kueue-system --type='strategic' --patch='{"spec": {"template": {"spec": {"containers": [{"name": "manager", "resources": {"limits": {"memory": "4096Mi"}}}]}}}}'
 [XPK] GKE commands done! Resources are created.
 [XPK] See your GKE Cluster here: https://console.cloud.google.com/kubernetes/clusters/details/us-central1/golden-cluster/details?project=golden-project

{xpk-1.1.1 → xpk-1.2.0}/recipes/Cluster_create_with_Managed_Lustre_driver_and_legacy_port.md

@@ -47,6 +47,12 @@ gcloud container clusters describe golden-cluster --project=golden-project --loc
 [XPK] Updating GKE cluster to enable Lustre CSI driver, may take a while!
 [XPK] Task: `GKE Cluster Update to enable Lustre CSI driver` is implemented by the following command not running since it is a dry run. 
 gcloud container clusters update golden-cluster --project=golden-project --location=us-central1 --quiet --enable-legacy-lustre-port
+[XPK] Recreating existing nodes (if any) to complete the Lustre CSI driver installation.
+[XPK] Task: `Get All Node Pools` is implemented by the following command not running since it is a dry run. 
+gcloud beta container node-pools list --cluster golden-cluster --project=golden-project --location=us-central1 --format="csv[no-heading](name)"
+[XPK] To complete NodesRecreate-0 we are executing gcloud container clusters upgrade golden-cluster --project=golden-project --node-pool=0 --location=us-central1 --quiet
+[XPK] Breaking up a total of 1 commands into 1 batches
+[XPK] Pretending all the jobs succeeded
 [XPK] Task: `Determine current gke master version` is implemented by the following command not running since it is a dry run. 
 gcloud beta container clusters describe golden-cluster --location us-central1 --project golden-project --format="value(currentMasterVersion)"
 [XPK] Creating 1 node pool or pools of tpu7x-8
@@ -271,8 +277,8 @@ description: "Very High"
 kubectl apply -f 6083d72fc3ba2ac7d243c1269dd67717abd4086bf64e397e3a1737de415dd133
 [XPK] Task: `Count total nodes` is implemented by the following command not running since it is a dry run. 
 kubectl get node --no-headers | wc -l
-[XPK] Try 1: Updating Kueue Controller Manager resources
-[XPK] Task: `Updating Kueue Controller Manager resources` is implemented by the following command not running since it is a dry run. 
+[XPK] Try 1: Updating Controller Manager resources
+[XPK] Task: `Updating Controller Manager resources` is implemented by the following command not running since it is a dry run. 
 kubectl patch deployment kueue-controller-manager -n kueue-system --type='strategic' --patch='{"spec": {"template": {"spec": {"containers": [{"name": "manager", "resources": {"limits": {"memory": "4096Mi"}}}]}}}}'
 [XPK] GKE commands done! Resources are created.
 [XPK] See your GKE Cluster here: https://console.cloud.google.com/kubernetes/clusters/details/us-central1/golden-cluster/details?project=golden-project

{xpk-1.1.1 → xpk-1.2.0}/recipes/Cluster_create_with_gb200-4.md

@@ -287,8 +287,8 @@ spec:
 kubectl apply -f c177e643775bb8e3462648245162a984934b0e09a13b0e3bfb62adf8585442b0
 [XPK] Task: `Count total nodes` is implemented by the following command not running since it is a dry run. 
 kubectl get node --no-headers | wc -l
-[XPK] Try 1: Updating Kueue Controller Manager resources
-[XPK] Task: `Updating Kueue Controller Manager resources` is implemented by the following command not running since it is a dry run. 
+[XPK] Try 1: Updating Controller Manager resources
+[XPK] Task: `Updating Controller Manager resources` is implemented by the following command not running since it is a dry run. 
 kubectl patch deployment kueue-controller-manager -n kueue-system --type='strategic' --patch='{"spec": {"template": {"spec": {"containers": [{"name": "manager", "resources": {"limits": {"memory": "4096Mi"}}}]}}}}'
 [XPK] Installing NCCL Plugin for cluster
 [XPK] Task: `Install NCCL Plugin On Cluster` is implemented by the following command not running since it is a dry run. 

{xpk-1.1.1 → xpk-1.2.0}/recipes/Cluster_create_with_shared_reservation.md

@@ -273,8 +273,8 @@ description: "Very High"
 kubectl apply -f 6083d72fc3ba2ac7d243c1269dd67717abd4086bf64e397e3a1737de415dd133
 [XPK] Task: `Count total nodes` is implemented by the following command not running since it is a dry run. 
 kubectl get node --no-headers | wc -l
-[XPK] Try 1: Updating Kueue Controller Manager resources
-[XPK] Task: `Updating Kueue Controller Manager resources` is implemented by the following command not running since it is a dry run. 
+[XPK] Try 1: Updating Controller Manager resources
+[XPK] Task: `Updating Controller Manager resources` is implemented by the following command not running since it is a dry run. 
 kubectl patch deployment kueue-controller-manager -n kueue-system --type='strategic' --patch='{"spec": {"template": {"spec": {"containers": [{"name": "manager", "resources": {"limits": {"memory": "4096Mi"}}}]}}}}'
 [XPK] GKE commands done! Resources are created.
 [XPK] See your GKE Cluster here: https://console.cloud.google.com/kubernetes/clusters/details/us-central1/golden-cluster/details?project=golden-project

{xpk-1.1.1 → xpk-1.2.0}/recipes/NAP_cluster-create.md

@@ -307,8 +307,8 @@ description: "Very High"
 kubectl apply -f ff0e8bb58b2038c4b29f1bce1aabe9f02ac0757ae2e80ad3657f704542371839
 [XPK] Task: `Count total nodes` is implemented by the following command not running since it is a dry run. 
 kubectl get node --no-headers | wc -l
-[XPK] Try 1: Updating Kueue Controller Manager resources
-[XPK] Task: `Updating Kueue Controller Manager resources` is implemented by the following command not running since it is a dry run. 
+[XPK] Try 1: Updating Controller Manager resources
+[XPK] Task: `Updating Controller Manager resources` is implemented by the following command not running since it is a dry run. 
 kubectl patch deployment kueue-controller-manager -n kueue-system --type='strategic' --patch='{"spec": {"template": {"spec": {"containers": [{"name": "manager", "resources": {"limits": {"memory": "4096Mi"}}}]}}}}'
 [XPK] GKE commands done! Resources are created.
 [XPK] See your GKE Cluster here: https://console.cloud.google.com/kubernetes/clusters/details/us-central1/golden-cluster/details?project=golden-project

{xpk-1.1.1 → xpk-1.2.0}/recipes/NAP_cluster-create_with_pathways.md

@@ -317,8 +317,8 @@ description: "Very High"
 kubectl apply -f fc46093b5c0d291fe7c53c15aebd624b485d767cabf99a73500e95952c70b6f6
 [XPK] Task: `Count total nodes` is implemented by the following command not running since it is a dry run. 
 kubectl get node --no-headers | wc -l
-[XPK] Try 1: Updating Kueue Controller Manager resources
-[XPK] Task: `Updating Kueue Controller Manager resources` is implemented by the following command not running since it is a dry run. 
+[XPK] Try 1: Updating Controller Manager resources
+[XPK] Task: `Updating Controller Manager resources` is implemented by the following command not running since it is a dry run. 
 kubectl patch deployment kueue-controller-manager -n kueue-system --type='strategic' --patch='{"spec": {"template": {"spec": {"containers": [{"name": "manager", "resources": {"limits": {"memory": "4096Mi"}}}]}}}}'
 [XPK] GKE commands done! Resources are created.
 [XPK] See your GKE Cluster here: https://console.cloud.google.com/kubernetes/clusters/details/us-central1/golden-cluster/details?project=golden-project

{xpk-1.1.1 → xpk-1.2.0}/recipes/Workload_create_super-slicing.md

@@ -47,7 +47,7 @@ docker buildx build --platform=linux/amd64 -f 4b6736a12db8ea0f78ce793fd0d4ee0c94
 docker tag dry-run-runner gcr.io/golden-project/dry-run-runner:prefix-current
 [XPK] Task: `Upload Docker Image` is implemented by the following command not running since it is a dry run. 
 docker push gcr.io/golden-project/dry-run-runner:prefix-current
-[XPK] Temp file (2eed164577b237a7b764f6adf46b9c249551c2200e440ef0f696c53dfb2d1cb5) content: 
+[XPK] Temp file (2c5ab381c0d643f8512a07d296d411413080ec652c15e8c676fd58435de5a327) content: 
 apiVersion: jobset.x-k8s.io/v1alpha2
 kind: JobSet
 metadata:
@@ -136,7 +136,7 @@ spec:
                   exit $EXIT_CODE
                 resources:
                   limits:
-                    google.com/tpu: 2
+                    google.com/tpu: 4
 
                 volumeMounts:
                 - mountPath: /dev/shm
@@ -156,7 +156,7 @@ spec:
               
 
 [XPK] Task: `Creating Workload` is implemented by the following command not running since it is a dry run. 
-kubectl apply -f 2eed164577b237a7b764f6adf46b9c249551c2200e440ef0f696c53dfb2d1cb5
+kubectl apply -f 2c5ab381c0d643f8512a07d296d411413080ec652c15e8c676fd58435de5a327
 [XPK] Task: `GKE Dashboard List` is implemented by the following command not running since it is a dry run. 
 gcloud monitoring dashboards list --project=golden-project --filter="displayName:'GKE - TPU Monitoring Dashboard'" --format="value(name)" --verbosity=error
 [XPK] Check statistics and outlier mode of GKE metrics here: https://console.cloud.google.com/monitoring/dashboards/builder/0?project=golden-project&f.rlabel.cluster_name.ClusterName=golden-cluster. To view the metric data for your workload, select golden-workload from the JobName filter on the dashboard.

{xpk-1.1.1 → xpk-1.2.0}/src/xpk/commands/cluster.py

@@ -369,7 +369,7 @@ def cluster_create(args) -> None:
 
   get_cluster_credentials(args)
 
-  update_coredns_command_code = update_coredns_if_necessary()
+  update_coredns_command_code = update_coredns_if_necessary(args)
   if update_coredns_command_code != 0:
     xpk_exit(update_coredns_command_code)
 
@@ -927,7 +927,7 @@ def scale_down_deployment(deployment_name: str, namespace: str = 'kube-system'):
   xpk_print(f'{deployment_name} has been scaled down.')
 
 
-def scale_up_coredns(replicas: int = 15, namespace: str = 'kube-system'):
+def scale_up_coredns(replicas: int, namespace: str = 'kube-system'):
   """Scales up the CoreDNS deployment to a specified number of replicas."""
   command_coredns_scale = (
       f'kubectl scale deployment coredns --replicas={replicas} -n {namespace}'
@@ -1008,7 +1008,14 @@ def cleanup_coredns_repo(coredns_repo_full_path: str):
     xpk_print(f'Error deleting directory {coredns_repo_full_path}: {e}')
 
 
-def update_coredns() -> int:
+def _get_coredns_replica_count(args) -> int:
+  # XPK large scale guide recommends 15 coreDNS replicas for clusters with 5000 VMs.
+  # Otherwise, limit the replica count to the desired number of default pool nodes.
+  default_pool_node_count: int = args.default_pool_cpu_num_nodes
+  return min(15, default_pool_node_count)
+
+
+def update_coredns(args) -> int:
   """Updates and deploys CoreDNS within a cluster.
 
   Returns:
@@ -1018,6 +1025,8 @@ def update_coredns() -> int:
   coredns_repo_dir_name = 'deployment'
   coredns_repo_full_path = os.path.join(coredns_repo_dir, coredns_repo_dir_name)
   coredns_k8s_path = os.path.join(coredns_repo_full_path, 'kubernetes')
+  coredns_replica_count = _get_coredns_replica_count(args)
+
   # 1. Install jq
   install_jq()
 
@@ -1034,7 +1043,7 @@ def update_coredns() -> int:
   scale_down_deployment('kube-dns')
 
   # 6. Scale up coredns and verify readiness
-  scale_up_coredns(replicas=15)
+  scale_up_coredns(coredns_replica_count)
   verify_coredns_readiness()
 
   xpk_print('The CoreDNS setup process has been completed.')
@@ -1074,7 +1083,7 @@ def coredns_deployment_exists(namespace: str = 'kube-system') -> bool:
     return False
 
 
-def update_coredns_if_necessary() -> int:
+def update_coredns_if_necessary(args) -> int:
   """Updates and deploys CoreDNS within the cluster if it's not already present.
 
   This function checks for the existence of the CoreDNS deployment.
@@ -1089,7 +1098,7 @@ def update_coredns_if_necessary() -> int:
     return 0
   else:
     xpk_print('CoreDNS deployment not found. Proceeding with CoreDNS setup.')
-    return update_coredns()
+    return update_coredns(args)
 
 
 def create_cluster_if_necessary(

{xpk-1.1.1 → xpk-1.2.0}/src/xpk/commands/cluster_test.py

@@ -22,7 +22,7 @@ from unittest.mock import MagicMock, patch
 import pytest
 
 from xpk.core.telemetry import MetricsCollector
-from xpk.commands.cluster import _install_kueue, _validate_cluster_create_args, run_gke_cluster_create_command, cluster_create, _log_cluster_create_telemetry
+from xpk.commands.cluster import _install_kueue, _validate_cluster_create_args, _get_coredns_replica_count, run_gke_cluster_create_command, cluster_create, _log_cluster_create_telemetry
 from xpk.core.capacity import CapacityType
 from xpk.core.system_characteristics import SystemCharacteristics, UserFacingNameToSystemCharacteristics
 from xpk.core.testing.commands_tester import CommandsTester
@@ -787,3 +787,18 @@ def test_validate_cluster_create_args_sets_correct_num_slices(
   _validate_cluster_create_args(args, SUPER_SLICING_SYSTEM)
 
   assert args.num_slices == expected
+
+
+def test_get_coredns_replica_count_lower_limit_is_number_of_nodes():
+  args = construct_args(
+      default_pool_cpu_num_nodes=7,
+  )
+
+  assert _get_coredns_replica_count(args) == 7
+
+
+def test_get_coredns_replica_count_upper_limit_is_15():
+  args = construct_args(
+      default_pool_cpu_num_nodes=20,
+  )
+  assert _get_coredns_replica_count(args) == 15

{xpk-1.1.1 → xpk-1.2.0}/src/xpk/core/cluster.py

@@ -21,6 +21,8 @@ from kubernetes import client as k8s_client
 from kubernetes import config
 from kubernetes.client.exceptions import ApiException
 
+from .kubectl_common import PatchResources, patch_controller_manager_resources
+from ..utils.feature_flags import FeatureFlags
 from ..utils.console import xpk_exit, xpk_print
 from .capacity import H200_DEVICE_TYPE
 from .commands import (
@@ -33,6 +35,7 @@ from .gcloud_context import (
     get_cluster_location,
     zone_to_region,
 )
+from .nodepool import recreate_nodes_in_existing_node_pools
 from .resources import get_cluster_system_characteristics
 from .system_characteristics import INSTALLER_NCCL_TCPXO, SystemCharacteristics
 
@@ -72,7 +75,21 @@ def set_jobset_on_cluster(args) -> int:
         ' https://github.com/google/xpk/blob/main/README.md#troubleshooting for'
         ' instructions on how to fix these permissions.'
     )
-  return return_code
+    return return_code
+
+  if FeatureFlags.SUPER_SLICING_ENABLED and args.super_slicing:
+    return patch_controller_manager_resources(
+        name='jobset-controller-manager',
+        namespace='jobset-system',
+        patch_resources=PatchResources(
+            cpu_request=4,
+            cpu_limit=4,
+            memory_request='16Gi',
+            memory_limit='16Gi',
+        ),
+    )
+
+  return 0
 
 
 def set_pathways_job_on_cluster(args) -> int:
@@ -605,6 +622,19 @@ def update_gke_cluster_with_lustre_driver_enabled(args) -> int:
   if return_code != 0:
     xpk_print(f'GKE Cluster Update request returned ERROR {return_code}')
     return 1
+
+  xpk_print(
+      'Recreating existing nodes (if any) to complete the Lustre CSI driver'
+      ' installation.'
+  )
+  return_code = recreate_nodes_in_existing_node_pools(args)
+  if return_code != 0:
+    xpk_print(
+        f'Node recreation failed with ERROR {return_code}. You must recreate'
+        ' the nodes manually in order to access Lustre storage from your'
+        ' workloads.'
+    )
+    return 1
   return 0
 
 

{xpk-1.1.1 → xpk-1.2.0}/src/xpk/core/cluster_test.py

@@ -14,10 +14,12 @@ See the License for the specific language governing permissions and
 limitations under the License.
 """
 
+from unittest.mock import MagicMock
 import pytest
 from .testing.commands_tester import CommandsTester
-from .cluster import get_cluster_credentials, update_gke_cluster_with_lustre_driver_enabled, update_cluster_with_lustre_driver_if_necessary
+from .cluster import get_cluster_credentials, update_gke_cluster_with_lustre_driver_enabled, update_cluster_with_lustre_driver_if_necessary, set_jobset_on_cluster
 from pytest_mock import MockerFixture
+from ..utils.feature_flags import FeatureFlags
 
 
 @pytest.fixture(autouse=True)
@@ -26,6 +28,9 @@ def commands_tester(mocker: MockerFixture) -> CommandsTester:
       mocker=mocker,
       run_command_for_value_path="xpk.core.cluster.run_command_for_value",
       run_command_with_updates_path="xpk.core.cluster.run_command_with_updates",
+      run_command_with_updates_retry_path=(
+          "xpk.core.cluster.run_command_with_updates_retry"
+      ),
   )
 
 
@@ -38,7 +43,17 @@ def mock_location(mocker: MockerFixture):
 
 @pytest.fixture(autouse=True)
 def command_args(mocker: MockerFixture):
-  return mocker.Mock(cluster="cluster", project="project", zone="zone")
+  return mocker.Mock(
+      cluster="cluster", project="project", zone="zone", super_slicing=False
+  )
+
+
+@pytest.fixture(autouse=True)
+def mock_patch_controller_manager_resources(mocker: MockerFixture) -> MagicMock:
+  return mocker.patch(
+      "xpk.core.cluster.patch_controller_manager_resources",
+      return_value=0,
+  )
 
 
 def test_get_cluster_credentials_returns_1_when_retrieval_commands_fail(
@@ -166,11 +181,14 @@ def test_update_cluster_with_lustre_driver_if_necessary_with_legacy_port_runs_co
 
 
 def test_update_gke_cluster_with_lustre_driver_enabled_default_port(
-    commands_tester: CommandsTester, command_args
+    commands_tester: CommandsTester, command_args, mocker: MockerFixture
 ):
   commands_tester.set_result_for_command(
       (0, ""), "gcloud container clusters update"
   )
+  mocker.patch(
+      "xpk.core.cluster.recreate_nodes_in_existing_node_pools", return_value=0
+  )
   command_args.enable_legacy_lustre_port = None
   update_gke_cluster_with_lustre_driver_enabled(command_args)
 
@@ -181,12 +199,30 @@ def test_update_gke_cluster_with_lustre_driver_enabled_default_port(
   ]
 
 
+def test_update_gke_cluster_with_lustre_driver_enabled_fails_if_node_recreation_failed(
+    commands_tester: CommandsTester, command_args, mocker: MockerFixture
+):
+  commands_tester.set_result_for_command(
+      (0, ""), "gcloud container clusters update"
+  )
+  mocker.patch(
+      "xpk.core.cluster.recreate_nodes_in_existing_node_pools", return_value=123
+  )
+  command_args.enable_legacy_lustre_port = None
+  return_code = update_gke_cluster_with_lustre_driver_enabled(command_args)
+
+  assert return_code != 0
+
+
 def test_update_gke_cluster_with_lustre_driver_enabled_legacy_port(
-    commands_tester: CommandsTester, command_args
+    commands_tester: CommandsTester, command_args, mocker: MockerFixture
 ):
   commands_tester.set_result_for_command(
       (0, ""), "gcloud container clusters update"
   )
+  mocker.patch(
+      "xpk.core.cluster.recreate_nodes_in_existing_node_pools", return_value=0
+  )
   command_args.enable_legacy_lustre_port = True
   update_gke_cluster_with_lustre_driver_enabled(command_args)
 
@@ -195,3 +231,24 @@ def test_update_gke_cluster_with_lustre_driver_enabled_legacy_port(
       "gcloud container clusters update cluster --project=project"
       " --location=us-central1 --quiet --enable-legacy-lustre-port"
   ]
+
+
+def test_set_jobset_on_cluster_not_setting_resources_by_default(
+    mock_patch_controller_manager_resources: MagicMock, command_args
+):
+  result = set_jobset_on_cluster(command_args)
+
+  assert result == 0
+  mock_patch_controller_manager_resources.assert_not_called()
+
+
+def test_set_jobset_on_cluster_super_slicing_resources(
+    mock_patch_controller_manager_resources: MagicMock, command_args
+):
+  FeatureFlags.SUPER_SLICING_ENABLED = True
+  command_args.super_slicing = True
+
+  result = set_jobset_on_cluster(command_args)
+
+  assert result == 0
+  mock_patch_controller_manager_resources.assert_called()

{xpk-1.1.1 → xpk-1.2.0}/src/xpk/core/docker_container.py

@@ -181,7 +181,9 @@ def get_main_container(
             tpu_stacktrace_terminate_command=tpu_stacktrace_terminate_command,
             gpu_workload_terminate_command=gpu_workload_terminate_command,
             xpk_internal_commands=xpk_internal_commands,
-            resources=get_main_container_resources(args, system, resource_type),
+            resources=get_main_container_resources(
+                args, system, resource_type, parallel_containers
+            ),
             volume_mounts=volume_mounts,
         )
     )

{xpk-1.1.1 → xpk-1.2.0}/src/xpk/core/docker_resources.py

@@ -23,7 +23,10 @@ from ..utils.execution_context import is_dry_run
 
 
 def get_main_container_resources(
-    args, system: SystemCharacteristics, resource_type
+    args,
+    system: SystemCharacteristics,
+    resource_type: str,
+    parallel_containers: int,
 ) -> str:
   """Resources for the main container.
   Args:
@@ -53,10 +56,7 @@ def get_main_container_resources(
     offset_vCPUs = int(system.chips_per_vm) * 0.95
     return f'{resource_type}: {offset_vCPUs}'
 
-  return (
-      f'{resource_type}:'
-      f' {int(system.chips_per_vm / system.parallel_containers)}'
-  )
+  return f'{resource_type}: {int(system.chips_per_vm / parallel_containers)}'
 
 
 def get_env_container(args, system: SystemCharacteristics) -> str: