xpk 1.0.0__tar.gz → 1.1.1__tar.gz

{xpk-1.0.0 → xpk-1.1.1}/.github/workflows/integration_basic_cluster_create.yaml

@@ -31,7 +31,7 @@ jobs:
           group: nightly-test-cluster-group-empty
           cancel-in-progress: false
       env:
-          EMPTY_CLUSTER_NAME: nightly-xpk-zero-nodepools
+          EMPTY_CLUSTER_NAME: nightly-xpk-zero
       steps:
           - uses: actions/download-artifact@v4
             with:
@@ -59,7 +59,7 @@ jobs:
           group: nightly-test-cluster-group-private
           cancel-in-progress: false
       env:
-          PRIVATE_CLUSTER_NAME: nightly-xpk-private-2-v4-8-nodepools
+          PRIVATE_CLUSTER_NAME: nightly-xpk-private-2-v4-8
       steps:
           - uses: actions/download-artifact@v4
             with:
@@ -83,38 +83,6 @@ jobs:
             with:
                 name: empty-private-cluster-nodepool-log-${{github.run_id}}
                 path: /tmp/NodepoolCreate-${{ env.PRIVATE_CLUSTER_NAME }}-np-*
-    dws_flex_cluster:
-      runs-on: [ubuntu-22.04]
-      concurrency: # We support one build test to run at a time currently.
-          group: nightly-test-cluster-group-flex
-          cancel-in-progress: false
-      env:
-          DWS_FLEX_CLUSTER_NAME: xpk-dws-nightly-test-2-v4-8
-      steps:
-          - uses: actions/download-artifact@v4
-            with:
-                name: custom-scripts
-          - name: Setup environment
-            uses: ./.github/actions/setup-test-env
-            with:
-                credentials_json: "${{ secrets.GCP_SA_KEY }}"
-          - name: Check xpk installation
-            run: xpk version
-          - name: Create a DWS flex queued xpk cluster
-            run: xpk cluster create --cluster ${DWS_FLEX_CLUSTER_NAME} --tpu-type=v5p-8  --num-slices=1 --zone=us-east5-a --default-pool-cpu-num-nodes=2 --flex --custom-cluster-arguments="${CLUSTER_NETWORK_ARGUMENTS_DWS}"
-          - name: Run dws flex queued TPU workload
-            run: xpk workload create --workload xpktest-build-${{ github.run_attempt }}-dws --cluster ${DWS_FLEX_CLUSTER_NAME} --zone=us-east5-a --tpu-type=v5p-8  --flex --command "echo foo" --num-slices=1
-          - name: Wait for workload completion and confirm it succeeded
-            run: xpk workload list --cluster ${DWS_FLEX_CLUSTER_NAME} --zone=us-east5-a --wait-for-job-completion xpktest-build-${{ github.run_attempt }}-dws --timeout 1000
-          - name: Delete the DWS flex queued cluster
-            if: always()
-            run: xpk cluster delete --cluster ${DWS_FLEX_CLUSTER_NAME} --zone=us-east5-a --force
-          - name: Upload DWS cluster nodepool creation log
-            if: always()
-            uses: actions/upload-artifact@v4
-            with:
-                name: empty-dws-cluster-nodepool-log-${{github.run_id}}
-                path: /tmp/NodepoolCreate-${{ env.DWS_FLEX_CLUSTER_NAME }}-np-*
 
     cluster-create-and-delete:
         runs-on: [ubuntu-22.04]
@@ -122,7 +90,7 @@ jobs:
             group: nightly-test-cluster-group-tpu
             cancel-in-progress: false
         env:
-            TPU_CLUSTER_NAME: nightly-xpk-2-v5p-8-nodepools
+            TPU_CLUSTER_NAME: nightly-xpk-2-v5p-8
             WORKLOAD_NAME: xpktest-nightly-${{ github.run_attempt }}
         steps:
             - uses: actions/download-artifact@v4

xpk-1.1.1/.github/workflows/integration_gpu_cluster_create.yaml

@@ -0,0 +1,78 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License
+
+name: Basic GPU cluster create
+
+on:
+    workflow_call:
+
+permissions:
+  contents: read
+
+jobs:
+    gpu-cluster-create-and-delete:
+        runs-on: [ubuntu-22.04]
+        concurrency:
+            group: nightly-test-cluster-group-gpu
+            cancel-in-progress: false
+        env:
+            GPU_CLUSTER_NAME: nightly-xpk-b200
+            WORKLOAD_NAME: xpktest-gpu-nightly-${{ github.run_attempt }}
+        steps:
+            - uses: actions/download-artifact@v4
+              with:
+                  name: custom-scripts
+            - name: Setup environment
+              uses: ./.github/actions/setup-test-env
+              with:
+                  credentials_json: "${{ secrets.GCP_SA_KEY }}"
+            - name: Check xpk installation
+              run: xpk version
+            - name: 'Setup Service Account for XPK'
+              run: |
+                # 1. Clear any existing WIF configurations to avoid conflicts
+                rm -rf $HOME/.config/gcloud
+                mkdir -p $HOME/.config/gcloud
+
+                # 2. Write the Key File
+                echo '${{ secrets.GCP_SA_KEY }}' > $HOME/.config/gcloud/application_default_credentials.json
+
+                # 3. Activate the Service Account
+                # This updates the internal config files to point to the key file.
+                # When Docker mounts the directory, it will now see "Active Account: Service Account"
+                gcloud auth activate-service-account --key-file=$HOME/.config/gcloud/application_default_credentials.json --project=cloud-tpu-multipod-dev
+
+                # 4. Set Env Var for the host (GitHub Runner)
+                echo "GOOGLE_APPLICATION_CREDENTIALS=$HOME/.config/gcloud/application_default_credentials.json" >> $GITHUB_ENV
+            - name: Create an XPK Cluster with 1 x b200 GPU
+              run: xpk cluster create --cluster $GPU_CLUSTER_NAME --device-type=b200-8 --zone=asia-northeast1-b --default-pool-cpu-machine-type=n1-standard-16 --spot
+            - name: Authenticate Docker
+              run: gcloud auth configure-docker --quiet
+            - name: Run a base-docker-image workload
+              run: xpk workload create --cluster $GPU_CLUSTER_NAME --workload $WORKLOAD_NAME --docker-image='nvidia/cuda:12.1.0-base-ubuntu22.04' --command "nvidia-smi" --zone=asia-northeast1-b --device-type=b200-8
+            - name: List out the workloads on the cluster
+              run: xpk workload list --cluster $GPU_CLUSTER_NAME --zone=asia-northeast1-b
+            - name: Wait for workload completion and confirm it succeeded
+              run: xpk workload list --cluster $GPU_CLUSTER_NAME --zone=asia-northeast1-b --wait-for-job-completion $WORKLOAD_NAME --timeout 600
+            - name: Delete the workload on the cluster
+              run: xpk workload delete --workload $WORKLOAD_NAME --cluster $GPU_CLUSTER_NAME --zone=asia-northeast1-b
+            - name: Delete the cluster created
+              if: always()
+              run: xpk cluster delete --cluster $GPU_CLUSTER_NAME --zone=asia-northeast1-b --force
+            - name: Upload cluster nodepool creation log
+              if: always()
+              uses: actions/upload-artifact@v4
+              with:
+                  name: gpu-cluster-nodepool-log-${{github.run_id}}
+                  path: /tmp/NodepoolCreate-${{ env.GPU_CLUSTER_NAME }}-np-*

{xpk-1.0.0 → xpk-1.1.1}/.github/workflows/nightly_tests.yaml

@@ -16,8 +16,8 @@ name: Nightly Tests
 
 on:
   workflow_dispatch:
-  schedule: # Schedule the job run at 12AM PST daily.
-    - cron: "0 8 * * *"
+  schedule: # Schedule the job run at 6AM UTC daily.
+    - cron: "0 6 * * *"
 
 permissions:
   contents: read
@@ -32,6 +32,11 @@ jobs:
     uses: ./.github/workflows/integration_basic_cluster_create.yaml
     secrets: inherit
 
+  gpu_cluster_create:
+    needs: [build_actions, build_wheel]
+    uses: ./.github/workflows/integration_gpu_cluster_create.yaml
+    secrets: inherit
+
   pathways_cluster_create:
     needs: [build_actions, build_wheel]
     uses: ./.github/workflows/integration_pathways_cluster_create.yaml
@@ -41,10 +46,6 @@ jobs:
     needs: [build_actions, build_wheel]
     uses: ./.github/workflows/integration_ray_cluster_create.yaml
     secrets: inherit
-  legacy_integration:
-    needs: [build_actions, build_wheel]
-    uses: ./.github/workflows/integration_legacy_tests.yaml
-    secrets: inherit
   storage-tests:
     needs: [build_actions, build_wheel]
     uses: ./.github/workflows/integration_storage_tests.yaml

{xpk-1.0.0 → xpk-1.1.1}/.github/workflows/reusable_goldens.yaml

@@ -38,7 +38,7 @@ jobs:
         key: xpk-deps-3.10-${{github.run_id}}-${{github.run_attempt}}
         restore-keys: xpk-deps-3.10-
     - name: Verify goldens
-      run: ./golden_buddy.sh verify goldens.yaml goldens
+      run: python3 tools/recipes.py golden recipes/*.md
       env:
         UPDATE_GOLDEN_COMMAND: make goldens
         XPK_VERSION_OVERRIDE: v0.0.0

{xpk-1.0.0 → xpk-1.1.1}/.github/workflows/stale.yaml

@@ -12,11 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License
 
-
-name: 'Close stale issues and PRs'
+name: "Close stale issues and PRs"
 on:
   schedule:
-    - cron: '30 1 * * *'
+    - cron: "30 1 * * *"
 
 jobs:
   stale:
@@ -24,7 +23,8 @@ jobs:
     steps:
       - uses: actions/stale@5f858e3efba33a5ca4407a664cc011ad407f2008 # v10.1.0
         with:
-          stale-pr-message: 'This pull request is stale because it has been open 30 days with no activity. Remove stale label or comment or this will be closed in 7 days.'
+          days-before-issue-stale: -1
+          stale-pr-message: "This pull request is stale because it has been open 30 days with no activity. Remove stale label or comment or this will be closed in 7 days."
           days-before-pr-stale: 30
           days-before-pr-close: 7
           operations-per-run: 100

{xpk-1.0.0 → xpk-1.1.1}/Makefile

@@ -1,12 +1,7 @@
-KUEUE_REPO=https://github.com/kubernetes-sigs/kueue.git
-
-KUBECTL_VERSION := $(shell curl -L -s https://dl.k8s.io/release/stable.txt)
-KUEUE_VERSION=v0.14.3
-
 OS := $(shell uname -s | tr A-Z a-z)
 PLATFORM := $(shell uname -m | sed -e 's/aarch64/arm64/' | sed -e 's/x86_64/amd64/')
 
-KUBECTL_URL = "https://dl.k8s.io/release/$(KUBECTL_VERSION)/bin/$(OS)/$(PLATFORM)/kubectl"
+KUEUE_VERSION=v0.15.2
 KUEUECTL_URL = "https://github.com/kubernetes-sigs/kueue/releases/download/$(KUEUE_VERSION)/kubectl-kueue-$(OS)-$(PLATFORM)"
 
 PROJECT_DIR := $(realpath $(shell dirname $(firstword $(MAKEFILE_LIST))))
@@ -34,12 +29,9 @@ install-pytest:
 run-unittests:
 	XPK_TESTER=false XPK_VERSION_OVERRIDE=v0.0.0 pytest  -vv src/xpk/
 
-run-integrationtests:
-	XPK_TESTER=false XPK_VERSION_OVERRIDE=v0.0.0 pytest src/integration/
-
 .PHONY: goldens
 goldens:
-	XPK_TESTER=false XPK_VERSION_OVERRIDE=v0.0.0 ./golden_buddy.sh update goldens.yaml goldens
+	XPK_TESTER=false XPK_VERSION_OVERRIDE=v0.0.0 python3 tools/recipes.py update recipes/*.md
 
 .PHONY: mkdir-bin
 mkdir-bin:

{xpk-1.0.0/src/xpk.egg-info → xpk-1.1.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: xpk
-Version: 1.0.0
+Version: 1.1.1
 Summary: xpk helps Cloud developers to orchestrate training jobs on accelerators on GKE.
 Author-email: XPK team <xpk-code-reviewers@google.com>
 License: Apache-2.0
@@ -93,28 +93,41 @@ XPK supports a variety of hardware accelerators.
 
 XPK also supports the following [Google Cloud Storage solutions](./docs/usage/storage.md):
 
-| Storage Type                               | Documentation                                                                            |
-|--------------------------------------------|------------------------------------------------------------------------------------------|
-| Cloud Storage FUSE                         | [docs](./docs/usage/storage.md#fuse)                                                     |
-| Filestore                                  | [docs](./docs/usage/storage.md#filestore)                                                |
-| Parallelstore                              | [docs](./docs/usage/storage.md#parallelstore)                                            |
-| Block storage (Persistent Disk, Hyperdisk) | [docs](./docs/usage/storage.md#block-storage-persistent-disk-hyperdisk)                  |
+| Storage Type                               | Documentation                                                           |
+| ------------------------------------------ | ----------------------------------------------------------------------- |
+| Cloud Storage FUSE                         | [docs](./docs/usage/storage.md#fuse)                                    |
+| Filestore                                  | [docs](./docs/usage/storage.md#filestore)                               |
+| Parallelstore                              | [docs](./docs/usage/storage.md#parallelstore)                           |
+| Block storage (Persistent Disk, Hyperdisk) | [docs](./docs/usage/storage.md#block-storage-persistent-disk-hyperdisk) |
 
 # Documentation
 
-* [Permissions](./docs/permissions.md)
-* [Installation](./docs/installation.md)
-* Usage:
-  * [Clusters](./docs/usage/clusters.md)
-    * [GPU](./docs/usage/gpu.md)
-    * [CPU](./docs/usage/cpu.md)
-    * [Autoprovisioning](./docs/usage/autoprovisioning.md)
-  * [Workloads](./docs/usage/workloads.md)
-    * [Docker](./docs/usage/docker.md)
-  * [Storage](./docs/usage/storage.md)
-  * [Advanced](./docs/usage/advanced.md)
-  * [Inspector](./docs/usage/inspector.md)
-* [Troubleshooting](./docs/troubleshooting.md)
+- [Permissions](./docs/permissions.md)
+- [Installation](./docs/installation.md)
+- Usage:
+  - [Clusters](./docs/usage/clusters.md)
+    - [GPU](./docs/usage/gpu.md)
+    - [CPU](./docs/usage/cpu.md)
+    - [Autoprovisioning](./docs/usage/autoprovisioning.md)
+  - [Workloads](./docs/usage/workloads.md)
+    - [Docker](./docs/usage/docker.md)
+  - [Storage](./docs/usage/storage.md)
+  - [Advanced](./docs/usage/advanced.md)
+  - [Inspector](./docs/usage/inspector.md)
+- [Troubleshooting](./docs/troubleshooting.md)
+
+# Dependencies
+
+| Dependency                                                                                                   | When used                   |
+| ------------------------------------------------------------------------------------------------------------ | --------------------------- |
+| [Google Cloud SDK (gcloud)](https://cloud.google.com/sdk/docs/install)                                       | _always_                    |
+| [kubectl](https://cloud.google.com/kubernetes-engine/docs/how-to/cluster-access-for-kubectl#install_kubectl) | _always_                    |
+| [ClusterToolkit](https://github.com/GoogleCloudPlatform/cluster-toolkit)                                     | Provisioning GPU clusters   |
+| [Kueue](https://github.com/kubernetes-sigs/kueue)                                                            | Scheduling workloads        |
+| [JobSet](https://github.com/kubernetes-sigs/jobset)                                                          | Workload creation           |
+| [Docker](https://docs.docker.com/engine/install/)                                                            | Building workload container |
+| [CoreDNS](https://github.com/coredns/deployment/tree/master/kubernetes)                                      | Cluster set up              |
+| [PathwaysJob](https://github.com/google/pathways-job)                                                        | Running Pathways workloads  |
 
 # Privacy notice
 
@@ -129,11 +142,14 @@ XPK telemetry overall is handled in accordance with the [Google Privacy Policy](
 you use XPK to interact with or utilize GCP Services, your information is handled in accordance with the
 [Google Cloud Privacy Notice](https://cloud.google.com/terms/cloud-privacy-notice).
 
-
 # Contributing
 
 Please read [`contributing.md`](./docs/contributing.md) for details on our code of conduct, and the process for submitting pull requests to us.
 
+# Get involved
+
+We'd love to hear from you! If you have questions or want to discuss ideas, join us on [GitHub Discussions](https://github.com/AI-Hypercomputer/xpk/discussions). Found a bug or have a feature request? Please let us know on [GitHub Issues](https://github.com/AI-Hypercomputer/xpk/issues).
+
 # License
 
 This project is licensed under the Apache License 2.0 - see the [`LICENSE`](./LICENSE) file for details

{xpk-1.0.0 → xpk-1.1.1}/README.md

@@ -52,28 +52,41 @@ XPK supports a variety of hardware accelerators.
 
 XPK also supports the following [Google Cloud Storage solutions](./docs/usage/storage.md):
 
-| Storage Type                               | Documentation                                                                            |
-|--------------------------------------------|------------------------------------------------------------------------------------------|
-| Cloud Storage FUSE                         | [docs](./docs/usage/storage.md#fuse)                                                     |
-| Filestore                                  | [docs](./docs/usage/storage.md#filestore)                                                |
-| Parallelstore                              | [docs](./docs/usage/storage.md#parallelstore)                                            |
-| Block storage (Persistent Disk, Hyperdisk) | [docs](./docs/usage/storage.md#block-storage-persistent-disk-hyperdisk)                  |
+| Storage Type                               | Documentation                                                           |
+| ------------------------------------------ | ----------------------------------------------------------------------- |
+| Cloud Storage FUSE                         | [docs](./docs/usage/storage.md#fuse)                                    |
+| Filestore                                  | [docs](./docs/usage/storage.md#filestore)                               |
+| Parallelstore                              | [docs](./docs/usage/storage.md#parallelstore)                           |
+| Block storage (Persistent Disk, Hyperdisk) | [docs](./docs/usage/storage.md#block-storage-persistent-disk-hyperdisk) |
 
 # Documentation
 
-* [Permissions](./docs/permissions.md)
-* [Installation](./docs/installation.md)
-* Usage:
-  * [Clusters](./docs/usage/clusters.md)
-    * [GPU](./docs/usage/gpu.md)
-    * [CPU](./docs/usage/cpu.md)
-    * [Autoprovisioning](./docs/usage/autoprovisioning.md)
-  * [Workloads](./docs/usage/workloads.md)
-    * [Docker](./docs/usage/docker.md)
-  * [Storage](./docs/usage/storage.md)
-  * [Advanced](./docs/usage/advanced.md)
-  * [Inspector](./docs/usage/inspector.md)
-* [Troubleshooting](./docs/troubleshooting.md)
+- [Permissions](./docs/permissions.md)
+- [Installation](./docs/installation.md)
+- Usage:
+  - [Clusters](./docs/usage/clusters.md)
+    - [GPU](./docs/usage/gpu.md)
+    - [CPU](./docs/usage/cpu.md)
+    - [Autoprovisioning](./docs/usage/autoprovisioning.md)
+  - [Workloads](./docs/usage/workloads.md)
+    - [Docker](./docs/usage/docker.md)
+  - [Storage](./docs/usage/storage.md)
+  - [Advanced](./docs/usage/advanced.md)
+  - [Inspector](./docs/usage/inspector.md)
+- [Troubleshooting](./docs/troubleshooting.md)
+
+# Dependencies
+
+| Dependency                                                                                                   | When used                   |
+| ------------------------------------------------------------------------------------------------------------ | --------------------------- |
+| [Google Cloud SDK (gcloud)](https://cloud.google.com/sdk/docs/install)                                       | _always_                    |
+| [kubectl](https://cloud.google.com/kubernetes-engine/docs/how-to/cluster-access-for-kubectl#install_kubectl) | _always_                    |
+| [ClusterToolkit](https://github.com/GoogleCloudPlatform/cluster-toolkit)                                     | Provisioning GPU clusters   |
+| [Kueue](https://github.com/kubernetes-sigs/kueue)                                                            | Scheduling workloads        |
+| [JobSet](https://github.com/kubernetes-sigs/jobset)                                                          | Workload creation           |
+| [Docker](https://docs.docker.com/engine/install/)                                                            | Building workload container |
+| [CoreDNS](https://github.com/coredns/deployment/tree/master/kubernetes)                                      | Cluster set up              |
+| [PathwaysJob](https://github.com/google/pathways-job)                                                        | Running Pathways workloads  |
 
 # Privacy notice
 
@@ -88,11 +101,14 @@ XPK telemetry overall is handled in accordance with the [Google Privacy Policy](
 you use XPK to interact with or utilize GCP Services, your information is handled in accordance with the
 [Google Cloud Privacy Notice](https://cloud.google.com/terms/cloud-privacy-notice).
 
-
 # Contributing
 
 Please read [`contributing.md`](./docs/contributing.md) for details on our code of conduct, and the process for submitting pull requests to us.
 
+# Get involved
+
+We'd love to hear from you! If you have questions or want to discuss ideas, join us on [GitHub Discussions](https://github.com/AI-Hypercomputer/xpk/discussions). Found a bug or have a feature request? Please let us know on [GitHub Issues](https://github.com/AI-Hypercomputer/xpk/issues).
+
 # License
 
 This project is licensed under the Apache License 2.0 - see the [`LICENSE`](./LICENSE) file for details

{xpk-1.0.0 → xpk-1.1.1}/docs/testing.md

@@ -47,36 +47,57 @@ A crucial aspect of effective unit testing is isolation. A unit test should only
 
 A good, state-of-the-art sample of [code](https://github.com/AI-Hypercomputer/xpk/blob/0434cf6a023069522f90d5846c6d980b68382b66/src/xpk/core/nodepool.py#L614) that has been correctly covered with unit tests can be found [here](https://github.com/AI-Hypercomputer/xpk/blob/8464ce26cd0fd24c681e346b2c915ad918724e53/src/xpk/core/nodepool_test.py#L26). This provided example serves as a practical guide and "source of truth" for developers, demonstrating best practices in unit test structure like naming. Another sample, leveraging mocks could be found [here](https://github.com/AI-Hypercomputer/xpk/blob/8464ce26cd0fd24c681e346b2c915ad918724e53/src/xpk/core/nodepool_test.py#L86).
 
-## Golden Test
+## Golden Recipes
 
-Golden tests encompass a broad scope within XPK, effectively covering the entire execution of a command from a user's perspective. Their primary objective is to highlight the blast radius of a change by making developers aware of all user journeys that might be affected by the change. These tests are executed on feature branches and serve as the main tool for raising awareness, enabling developers to thoroughly double-check changes across various scenarios and understand their potential impact.
+Golden recipes encompass a broad scope within XPK, effectively covering entire user journeys. Their primary objective is to orchestrate multiple commands to achieve a high-level goal, simulating a real user interacting with the system. They also serve as regression tests by asserting on the output of each step, ensuring that the user experience remains consistent. These tests are executed on feature branches and serve as the main tool for raising awareness, enabling developers to thoroughly double-check changes across various complex scenarios and understand their potential impact.
 
 ### Naming Conventions
 
-Each Golden test name should refer to a potential use case or persona utilizing the system, explicitly including the command that is executed. This approach ensures that the test names clearly communicate the real-world scenarios and user interactions they validate, focusing on the actions taken. A good Golden test name should typically convey:
+Each Golden recipe file in the `recipes` directory corresponds to a specific use case or persona utilizing the system. The filename should clearly indicate the scenario, for example `NAP_cluster-create_with_pathways.md` or `Cluster_create_with_Managed_Lustre_driver.md`.
 
-* **Command name that is executed:** cluster create, cluster create-pathways, or workload list.
-* **Use case it is covering:** nap cluster creation, tpu cluster creation, workload status listing
+### Developer guide to Golden Recipes
+
+All golden recipes are located in the `recipes` directory. Each recipe is a Markdown file that describes the user journey and contains the sequence of commands to be executed.
+
+A sample structure of a recipe file is:
 
-For example, a good golden test name could be: "NAP cluster-create with pathways".
+```markdown
+# Recipe Title
 
-### Developer guide to Golden Tests
-All golden tests are registered in the `goldens.yaml` file in the root directory. Their reference output is stored in text files located in goldens directory in the root directory.
+Description of the recipe.
 
-A sample structure of `goldens.yaml` file is defined as:
+## Step 1: Create Cluster
+\`\`\`shell #golden
+xpk cluster create ...
+\`\`\`
+<!--
+Expected output block
+-->
 
-```yaml
-goldens:
-  "NAP cluster-create with pathways":
-    command: xpk cluster create-pathways --enable-autoprovisioning
-    description: "" # optional description allowing to better understand use-case
+## Step 2: Submit Workload
+\`\`\`shell #golden
+xpk workload create ...
+\`\`\`
+<!--
+Expected output block
+-->
 ```
 
-Goldens after change in the code, or registering a new one can be re-generated using `make goldens` command.
+Recipe files are self-contained, storing both the commands and their expected golden outputs in comment blocks. The recipe executor runs these commands in order, maintaining state between them (e.g. environment variables).
 
 ### Underlying execution mechanisms
 
-These tests are executed through the GoldenBuddy testing script located in the `golden_buddy.sh` file of the repository. The framework executes all registered commands in `dry_run` mode, then compares diffs between them with the reference output located in goldens directory.
+These tests are executed through the `tools/recipes.py` script. The framework executes the sequence of commands in `dry_run` mode (by injecting a mock `xpk` function) and compares the output of each step with the expected output stored in the recipe file.
+
+**Usage:**
+
+*   **Regenerate Goldens:** `make goldens`
+    *   This is the primary command for developers. It executes all recipes in `update` mode, regenerating the golden outputs. Run this after making changes to the code or adding new recipes.
+
+*   **Advanced:** *(These commands are primarily used by CI/CD pipelines or for debugging specific scenarios.)*
+    *   **Verification:** `python3 tools/recipes.py golden <file or files>` (Verifies outputs match without updating)
+    *   **Integration Run:** `python3 tools/recipes.py run <file or files>` (Executes commands for real)
+    *   **Selective Update:** `python3 tools/recipes.py update <file or files>` (Updates a specific recipe)
 
 ## Integration Test
 Integration tests sit at the apex of the testing pyramid, being the most expensive and slowest to execute. This is primarily because they rely on actual Google Cloud Platform (GCP) infrastructure, which introduces potential flakiness due to external factors and makes it challenging to write given capacity constraints. Consequently, these tests should be reserved for ultimate verification before release, ensuring all of XPK's components function seamlessly together within a real GCP environment. They are not run on feature branches; instead, they are executed on the mainline (`main`) branch nightly after code merges, and right before a release to validate a new XPK release candidate. This strategic placement ensures a final, comprehensive check of the entire system's functionality in its production-like setting.

{xpk-1.0.0 → xpk-1.1.1}/docs/usage/clusters.md

@@ -51,7 +51,7 @@ all zones.
     --num-slices=4 --on-demand
     ```
 
-*   Cluster Create (provision spot / preemptable capacity):
+*   Cluster Create (provision spot / preemptible capacity):
 
     ```shell
     xpk cluster create \
@@ -274,7 +274,7 @@ xpk cluster create-pathways \
 --managed-mldiagnostics
 ```
 
-* Cluster Create (provision spot / preemptable capacity) with flag **--managed-mldiagnostics**:
+* Cluster Create (provision spot / preemptible capacity) with flag **--managed-mldiagnostics**:
 
 ```shell
 xpk cluster create \

{xpk-1.0.0 → xpk-1.1.1}/docs/usage/tpu7x/recipes/flex_filestore_recipe.md

@@ -29,14 +29,10 @@ Before you start, complete the following steps:
 
 ### Create a single-NIC, single slice cluster
 
-Currently flex start provisioning for Ironwood works only in single slice and multi-host or multi-slice and single host setups. More options will be added soon
-
 1. Set the following environment variables:
 
     > **NOTE:** For multi-host provisioning use an ACCELERATOR_TYPE with any topology that results to more than 8 chips, e.g. `tpu7x-2x2x2` or `tpu7x-16`. For single-host provisioning use an ACCELERATOR_TYPE with any topology that results to 8 or less chips, e.g. `tpu7x-2x2x1` or `tpu7x-8`.
 
-    > **NOTE:** Single-host provisioning is not supported for single-slice. If you want to create a single-host cluster, you need to set `--num-slices` to 2 or higher on the `xpk cluster create` command.
-
     ```shell
     export PROJECT_ID=<project_id> # Your GCP project name
     export ZONE=<zone> # Example: us-central1-c

{xpk-1.0.0 → xpk-1.1.1}/docs/usage/tpu7x/recipes/flex_lustre_recipe.md

@@ -29,14 +29,10 @@ Before you start, complete the following steps:
 
 ### Create a single-NIC, single slice cluster
 
-Currently flex start provisioning for Ironwood works only in single slice and multi-host or multi-slice and single host setups. More options will be added soon
-
 1. Set the following environment variables:
 
     > **NOTE:** For multi-host provisioning use an ACCELERATOR_TYPE with any topology that results to more than 8 chips, e.g. `tpu7x-2x2x2` or `tpu7x-16`. For single-host provisioning use an ACCELERATOR_TYPE with any topology that results to 8 or less chips, e.g. `tpu7x-2x2x1` or `tpu7x-8`.
 
-    > **NOTE:** Single-host provisioning is not supported for single-slice. If you want to create a single-host cluster, you need to set `--num-slices` to 2 or higher on the `xpk cluster create` command.
-
     ```shell
     export PROJECT_ID=<project_id> # Your GCP project name
     export ZONE=<zone> # Example: us-central1-c

{xpk-1.0.0 → xpk-1.1.1}/docs/usage/workloads.md

@@ -15,6 +15,9 @@
  -->
  
 ## Workload Create
+
+_Note: `xpk workload create` works only on clusters created through XPK. See [docs](./clusters.md) on how to create a cluster via XPK._
+
 *   Workload Create (submit training job):
 
     ```shell