skypilot-nightly 1.0.0.dev20250905__py3-none-any.whl → 1.0.0.dev20251210__py3-none-any.whl

sky/utils/kubernetes/generate_kind_config.py

@@ -3,67 +3,8 @@
 Maps specified ports from host to cluster container.
 """
 import argparse
-import textwrap
-
-
-def generate_kind_config(path: str,
-                         port_start: int = 30000,
-                         port_end: int = 32768,
-                         num_nodes: int = 1,
-                         gpus: bool = False) -> None:
-    """Generate a kind cluster config with ports mapped from host to container
-
-    Args:
-        path: Path to generate the config file at
-        port_start: Port range start
-        port_end: Port range end
-        num_nodes: Number of nodes in the cluster
-        gpus: If true, initialize kind cluster with GPU support
-    """
-
-    preamble = textwrap.dedent(f"""
-    apiVersion: kind.x-k8s.io/v1alpha4
-    kind: Cluster
-    kubeadmConfigPatches:
-    - |
-      kind: ClusterConfiguration
-      apiServer:
-        extraArgs:
-          "service-node-port-range": {port_start}-{port_end}
-    nodes:
-    - role: control-plane
-      kubeadmConfigPatches:
-      - |
-        kind: InitConfiguration
-        nodeRegistration:
-          kubeletExtraArgs:
-            node-labels: "ingress-ready=true"
-    """)
-    if gpus:
-        preamble += textwrap.indent(
-            textwrap.dedent("""
-        extraMounts:
-        - hostPath: /dev/null
-          containerPath: /var/run/nvidia-container-devices/all"""), ' ' * 2)
-    preamble += textwrap.indent(
-        textwrap.dedent("""
-      extraPortMappings:"""), ' ' * 2)
-    suffix = ''
-    if num_nodes > 1:
-        for _ in range(1, num_nodes):
-            suffix += """- role: worker\n"""
-    with open(path, 'w', encoding='utf-8') as f:
-        f.write(preamble)
-        for port in range(port_start, port_end + 1):
-            f.write(f"""
-      - containerPort: {port}
-        hostPort: {port}
-        listenAddress: "0.0.0.0"
-        protocol: tcp""")
-        f.write('\n')
-        if suffix:
-            f.write(suffix)
 
+from sky.utils.kubernetes import kubernetes_deploy_utils
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(description='Generate a kind cluster '
@@ -77,10 +18,6 @@ if __name__ == '__main__':
                         type=int,
                         default=30000,
                         help='Port range start')
-    parser.add_argument('--port-end',
-                        type=int,
-                        default=32768,
-                        help='Port range end')
     parser.add_argument('--num-nodes',
                         type=int,
                         default=1,
@@ -90,5 +27,8 @@ if __name__ == '__main__':
                         action='store_true',
                         help='Initialize kind cluster with GPU support')
     args = parser.parse_args()
-    generate_kind_config(args.path, args.port_start, args.port_end,
-                         args.num_nodes, args.gpus)
+
+    with open(args.path, 'w', encoding='utf-8') as f:
+        f.write(
+            kubernetes_deploy_utils.generate_kind_config(
+                args.port_start, args.num_nodes, args.gpus))

sky/utils/kubernetes/gpu_labeler.py

@@ -40,7 +40,9 @@ def cleanup(context: Optional[str] = None) -> Tuple[bool, str]:
             success = True
         except subprocess.CalledProcessError as e:
             output = e.output.decode('utf-8')
-            reason = 'Error deleting existing GPU labeler resources: ' + output
+            stderr = e.stderr.decode('utf-8')
+            reason = ('Error deleting existing GPU labeler resources: ' +
+                      output + stderr)
         return success, reason
 
 
@@ -183,9 +185,17 @@ def wait_for_jobs_completion(jobs_to_node_names: Dict[str, str],
     batch_v1 = kubernetes.batch_api(context=context)
     w = kubernetes.watch()
     completed_jobs = []
+    # Use resource_version="0" to start from the oldest available version.
+    # In multi-replica API server environments, replicas may be at different
+    # resource versions due to replication lag. Without specifying this, the
+    # watch may get version X from one replica but connect to another replica
+    # that only has up to version Y < X, causing "Too large resource version"
+    # errors. Using "0" ensures all replicas can serve the request from their
+    # oldest available version, avoiding version mismatches.
     for event in w.stream(func=batch_v1.list_namespaced_job,
                           namespace=namespace,
-                          timeout_seconds=timeout):
+                          timeout_seconds=timeout,
+                          resource_version='0'):
         job = event['object']
         job_name = job.metadata.name
         if job_name in jobs_to_node_names:
@@ -212,7 +222,7 @@ def wait_for_jobs_completion(jobs_to_node_names: Dict[str, str],
         _format_string(
             f'Timed out after waiting {timeout} seconds '
             'for job to complete', colorama.Style.DIM))
-    return False  #Timed out
+    return False  # Timed out
 
 
 def main():

sky/utils/kubernetes/k8s_gpu_labeler_job.yaml

@@ -13,10 +13,11 @@ spec:
       serviceAccountName: gpu-labeler-sa
       containers:
       - name: gpu-labeler
-        image: us-central1-docker.pkg.dev/skypilot-375900/skypilotk8s/skypilot-gpu:latest # Using this image also serves as a way to "pre-pull" the image onto nodes
+        image: us-docker.pkg.dev/sky-dev-465/skypilotk8s/skypilot-gpu:latest # Using this image also serves as a way to "pre-pull" the image onto nodes
         command: ["/bin/bash", "-i", "-c"]
         args:
           - |
+            source ~/skypilot-runtime/bin/activate
             python /label_gpus.py
         env:
         - name: MY_NODE_NAME

sky/utils/kubernetes/k8s_gpu_labeler_setup.yaml

@@ -53,51 +53,51 @@ data:
     import os
     import subprocess
     from typing import Optional
-    
+
     from kubernetes import client
     from kubernetes import config
-    
+
     canonical_gpu_names = [
-        'A100-80GB', 'A100', 'A10G', 'H100', 'K80', 'M60', 'T4g', 'T4', 'V100', 
+        'A100-80GB', 'A100', 'A10G', 'H100', 'K80', 'M60', 'T4g', 'T4', 'V100',
         'A10', 'P100', 'P40', 'P4', 'L4'
     ]
-    
-    
+
+
     def get_gpu_name() -> Optional[str]:
         try:
             result = subprocess.run(
                 ['nvidia-smi', '--query-gpu=name', '--format=csv,noheader,nounits'],
                 stdout=subprocess.PIPE)
             gpu_name = result.stdout.decode('utf-8').strip()
-            # In the case of multi-gpu nodes, we assume the node is homogenous and 
+            # In the case of multi-gpu nodes, we assume the node is homogenous and
             # just use the first GPU name.
             gpu_name = gpu_name.split('\n')[0]
             return gpu_name.lower()
         except Exception as e:
             print(f'Error getting GPU name: {e}')
             return None
-    
-    
+
+
     def label_node(gpu_name: str) -> None:
         try:
             config.load_incluster_config()  # Load in-cluster configuration
             v1 = client.CoreV1Api()
-    
+
             # Fetch the current node's name from the environment variable
             node_name = os.environ.get('MY_NODE_NAME')
             if not node_name:
                 raise ValueError('Failed to get node name from environment')
-    
+
             # Label the node with the GPU name
             body = {'metadata': {'labels': {'skypilot.co/accelerator': gpu_name}}}
             v1.patch_node(node_name, body)
-    
+
             print(f'Labeled node {node_name} with GPU {gpu_name}')
-    
+
         except Exception as e:
             print(f'Error labeling node: {e}')
-    
-    
+
+
     def main():
         gpu_name = get_gpu_name()
         if gpu_name is not None:
@@ -119,7 +119,7 @@ data:
                 labelled = True
         else:
             print('No GPU detected. Try running nvidia-smi in the container.')
-    
-    
+
+
     if __name__ == '__main__':
         main()