skypilot-nightly 1.0.0.dev20250502__py3-none-any.whl → 1.0.0.dev20251203__py3-none-any.whl

sky/utils/kubernetes/exec_kubeconfig_converter.py

@@ -5,10 +5,19 @@ the 'command' field in the exec configuration, leaving only the executable name.
 This is useful when moving between different environments where auth plugin
 executables might be installed in different locations.
 
+For Nebius kubeconfigs, it also changes the --profile argument to 'sky' to
+ensure compatibility with SkyPilot's expected profile configuration.
+
 It assumes the target environment has the auth executable available in PATH.
 If not, you'll need to update your environment container to include the auth
 executable in PATH.
 
+When using LOCAL_CREDENTIALS (aka exec auth) with Kubernetes, though, SkyPilot
+will automatically inject a wrapper script for common exec auth providers like
+GKE and EKS. This wrapper script helps to resolve path issues that may arise
+from executables installed on non system-default paths. Thus, the kubeconfig
+file may look different on the sky jobs controller.
+
 Usage:
     python -m sky.utils.kubernetes.exec_kubeconfig_converter
 """
@@ -17,36 +26,7 @@ import os
 
 import yaml
 
-
-def strip_auth_plugin_paths(kubeconfig_path: str, output_path: str):
-    """Strip path information from exec plugin commands in a kubeconfig file.
-
-    Args:
-        kubeconfig_path (str): Path to the input kubeconfig file
-        output_path (str): Path where the modified kubeconfig will be saved
-    """
-    with open(kubeconfig_path, 'r', encoding='utf-8') as file:
-        config = yaml.safe_load(file)
-
-    updated = False
-    for user in config.get('users', []):
-        exec_info = user.get('user', {}).get('exec', {})
-        current_command = exec_info.get('command', '')
-
-        if current_command:
-            # Strip the path and keep only the executable name
-            executable = os.path.basename(current_command)
-            if executable != current_command:
-                exec_info['command'] = executable
-                updated = True
-
-    if updated:
-        with open(output_path, 'w', encoding='utf-8') as file:
-            yaml.safe_dump(config, file)
-        print('Kubeconfig updated with path-less exec auth. '
-              f'Saved to {output_path}')
-    else:
-        print('No updates made. No exec-based auth commands paths found.')
+from sky.provision.kubernetes import utils as kubernetes_utils
 
 
 def main():
@@ -66,7 +46,18 @@ def main():
         help='Output kubeconfig file path (default: %(default)s)')
 
     args = parser.parse_args()
-    strip_auth_plugin_paths(args.input, args.output)
+
+    with open(args.input, 'r', encoding='utf-8') as file:
+        config = yaml.safe_load(file)
+
+    updated = kubernetes_utils.format_kubeconfig_exec_auth(
+        config, args.output, False)
+
+    if updated:
+        print('Kubeconfig updated with path-less exec auth. '
+              f'Saved to {args.output}')
+    else:
+        print('No updates made.')
 
 
 if __name__ == '__main__':

sky/utils/kubernetes/generate_kind_config.py

@@ -3,67 +3,8 @@
 Maps specified ports from host to cluster container.
 """
 import argparse
-import textwrap
-
-
-def generate_kind_config(path: str,
-                         port_start: int = 30000,
-                         port_end: int = 32768,
-                         num_nodes: int = 1,
-                         gpus: bool = False) -> None:
-    """Generate a kind cluster config with ports mapped from host to container
-
-    Args:
-        path: Path to generate the config file at
-        port_start: Port range start
-        port_end: Port range end
-        num_nodes: Number of nodes in the cluster
-        gpus: If true, initialize kind cluster with GPU support
-    """
-
-    preamble = textwrap.dedent(f"""
-    apiVersion: kind.x-k8s.io/v1alpha4
-    kind: Cluster
-    kubeadmConfigPatches:
-    - |
-      kind: ClusterConfiguration
-      apiServer:
-        extraArgs:
-          "service-node-port-range": {port_start}-{port_end}
-    nodes:
-    - role: control-plane
-      kubeadmConfigPatches:
-      - |
-        kind: InitConfiguration
-        nodeRegistration:
-          kubeletExtraArgs:
-            node-labels: "ingress-ready=true"
-    """)
-    if gpus:
-        preamble += textwrap.indent(
-            textwrap.dedent("""
-        extraMounts:
-        - hostPath: /dev/null
-          containerPath: /var/run/nvidia-container-devices/all"""), ' ' * 2)
-    preamble += textwrap.indent(
-        textwrap.dedent("""
-      extraPortMappings:"""), ' ' * 2)
-    suffix = ''
-    if num_nodes > 1:
-        for _ in range(1, num_nodes):
-            suffix += """- role: worker\n"""
-    with open(path, 'w', encoding='utf-8') as f:
-        f.write(preamble)
-        for port in range(port_start, port_end + 1):
-            f.write(f"""
-      - containerPort: {port}
-        hostPort: {port}
-        listenAddress: "0.0.0.0"
-        protocol: tcp""")
-        f.write('\n')
-        if suffix:
-            f.write(suffix)
 
+from sky.utils.kubernetes import kubernetes_deploy_utils
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(description='Generate a kind cluster '
@@ -77,10 +18,6 @@ if __name__ == '__main__':
                         type=int,
                         default=30000,
                         help='Port range start')
-    parser.add_argument('--port-end',
-                        type=int,
-                        default=32768,
-                        help='Port range end')
     parser.add_argument('--num-nodes',
                         type=int,
                         default=1,
@@ -90,5 +27,8 @@ if __name__ == '__main__':
                         action='store_true',
                         help='Initialize kind cluster with GPU support')
     args = parser.parse_args()
-    generate_kind_config(args.path, args.port_start, args.port_end,
-                         args.num_nodes, args.gpus)
+
+    with open(args.path, 'w', encoding='utf-8') as f:
+        f.write(
+            kubernetes_deploy_utils.generate_kind_config(
+                args.port_start, args.num_nodes, args.gpus))

sky/utils/kubernetes/generate_kubeconfig.sh

@@ -156,6 +156,9 @@ rules:
   - apiGroups: [""]                 # Required for sky show-gpus command
     resources: ["pods"]
     verbs: ["get", "list"]
+  - apiGroups: ["rbac.authorization.k8s.io"]   # Required for SkyPilot to inspect its own permissions
+    resources: ["clusterroles", "clusterrolebindings", "roles", "rolebindings"]
+    verbs: ["get", "list", "watch"]
 ---
 # ClusterRoleBinding for the service account
 apiVersion: rbac.authorization.k8s.io/v1
@@ -316,7 +319,7 @@ EOF
 echo "---
 Done!
 
-Kubeconfig using service acccount '${SKYPILOT_SA}' in namespace '${NAMESPACE}' written at $(pwd)/kubeconfig
+Kubeconfig using service account '${SKYPILOT_SA}' in namespace '${NAMESPACE}' written at $(pwd)/kubeconfig
 
 Copy the generated kubeconfig file to your ~/.kube/ directory to use it with
 kubectl and skypilot:

sky/utils/kubernetes/gpu_labeler.py

@@ -8,9 +8,9 @@ from typing import Dict, Optional, Tuple
 import colorama
 import yaml
 
-import sky
 from sky.adaptors import kubernetes
 from sky.provision.kubernetes import utils as kubernetes_utils
+from sky.utils import directory_utils
 from sky.utils import rich_utils
 
 
@@ -40,7 +40,9 @@ def cleanup(context: Optional[str] = None) -> Tuple[bool, str]:
             success = True
         except subprocess.CalledProcessError as e:
             output = e.output.decode('utf-8')
-            reason = 'Error deleting existing GPU labeler resources: ' + output
+            stderr = e.stderr.decode('utf-8')
+            reason = ('Error deleting existing GPU labeler resources: ' +
+                      output + stderr)
         return success, reason
 
 
@@ -62,8 +64,8 @@ def label(context: Optional[str] = None, wait_for_completion: bool = True):
     if not unlabeled_gpu_nodes:
         print('No unlabeled GPU nodes found in the cluster. If you have '
               'unlabeled GPU nodes, please ensure that they have the resource '
-              f'`{kubernetes_utils.get_gpu_resource_key()}: <number of GPUs>` '
-              'in their capacity.')
+              f'`{kubernetes_utils.get_gpu_resource_key(context)}: '
+              '<number of GPUs>` in their capacity.')
         return
 
     print(
@@ -71,8 +73,8 @@ def label(context: Optional[str] = None, wait_for_completion: bool = True):
             f'Found {len(unlabeled_gpu_nodes)} '
             'unlabeled GPU nodes in the cluster', colorama.Fore.YELLOW))
 
-    sky_dir = os.path.dirname(sky.__file__)
-    manifest_dir = os.path.join(sky_dir, 'utils/kubernetes')
+    manifest_dir = os.path.join(directory_utils.get_sky_dir(),
+                                'utils/kubernetes')
 
     # Apply the RBAC manifest using kubectl since it contains multiple resources
     with rich_utils.client_status('Setting up GPU labeling'):
@@ -183,9 +185,17 @@ def wait_for_jobs_completion(jobs_to_node_names: Dict[str, str],
     batch_v1 = kubernetes.batch_api(context=context)
     w = kubernetes.watch()
     completed_jobs = []
+    # Use resource_version="0" to start from the oldest available version.
+    # In multi-replica API server environments, replicas may be at different
+    # resource versions due to replication lag. Without specifying this, the
+    # watch may get version X from one replica but connect to another replica
+    # that only has up to version Y < X, causing "Too large resource version"
+    # errors. Using "0" ensures all replicas can serve the request from their
+    # oldest available version, avoiding version mismatches.
     for event in w.stream(func=batch_v1.list_namespaced_job,
                           namespace=namespace,
-                          timeout_seconds=timeout):
+                          timeout_seconds=timeout,
+                          resource_version='0'):
         job = event['object']
         job_name = job.metadata.name
         if job_name in jobs_to_node_names:
@@ -212,7 +222,7 @@ def wait_for_jobs_completion(jobs_to_node_names: Dict[str, str],
         _format_string(
             f'Timed out after waiting {timeout} seconds '
             'for job to complete', colorama.Style.DIM))
-    return False  #Timed out
+    return False  # Timed out
 
 
 def main():

sky/utils/kubernetes/k8s_gpu_labeler_job.yaml

@@ -13,10 +13,11 @@ spec:
       serviceAccountName: gpu-labeler-sa
       containers:
       - name: gpu-labeler
-        image: us-central1-docker.pkg.dev/skypilot-375900/skypilotk8s/skypilot-gpu:latest # Using this image also serves as a way to "pre-pull" the image onto nodes
+        image: us-docker.pkg.dev/sky-dev-465/skypilotk8s/skypilot-gpu:latest # Using this image also serves as a way to "pre-pull" the image onto nodes
         command: ["/bin/bash", "-i", "-c"]
         args:
           - |
+            source ~/skypilot-runtime/bin/activate
             python /label_gpus.py
         env:
         - name: MY_NODE_NAME

sky/utils/kubernetes/k8s_gpu_labeler_setup.yaml

@@ -53,51 +53,51 @@ data:
     import os
     import subprocess
     from typing import Optional
-    
+
     from kubernetes import client
     from kubernetes import config
-    
+
     canonical_gpu_names = [
-        'A100-80GB', 'A100', 'A10G', 'H100', 'K80', 'M60', 'T4g', 'T4', 'V100', 
+        'A100-80GB', 'A100', 'A10G', 'H100', 'K80', 'M60', 'T4g', 'T4', 'V100',
         'A10', 'P100', 'P40', 'P4', 'L4'
     ]
-    
-    
+
+
     def get_gpu_name() -> Optional[str]:
         try:
             result = subprocess.run(
                 ['nvidia-smi', '--query-gpu=name', '--format=csv,noheader,nounits'],
                 stdout=subprocess.PIPE)
             gpu_name = result.stdout.decode('utf-8').strip()
-            # In the case of multi-gpu nodes, we assume the node is homogenous and 
+            # In the case of multi-gpu nodes, we assume the node is homogenous and
             # just use the first GPU name.
             gpu_name = gpu_name.split('\n')[0]
             return gpu_name.lower()
         except Exception as e:
             print(f'Error getting GPU name: {e}')
             return None
-    
-    
+
+
     def label_node(gpu_name: str) -> None:
         try:
             config.load_incluster_config()  # Load in-cluster configuration
             v1 = client.CoreV1Api()
-    
+
             # Fetch the current node's name from the environment variable
             node_name = os.environ.get('MY_NODE_NAME')
             if not node_name:
                 raise ValueError('Failed to get node name from environment')
-    
+
             # Label the node with the GPU name
             body = {'metadata': {'labels': {'skypilot.co/accelerator': gpu_name}}}
             v1.patch_node(node_name, body)
-    
+
             print(f'Labeled node {node_name} with GPU {gpu_name}')
-    
+
         except Exception as e:
             print(f'Error labeling node: {e}')
-    
-    
+
+
     def main():
         gpu_name = get_gpu_name()
         if gpu_name is not None:
@@ -119,7 +119,7 @@ data:
                 labelled = True
         else:
             print('No GPU detected. Try running nvidia-smi in the container.')
-    
-    
+
+
     if __name__ == '__main__':
         main()