skypilot-nightly 1.0.0.dev20250427__py3-none-any.whl → 1.0.0.dev20250429__py3-none-any.whl

sky/templates/kubernetes-ray.yml.j2

@@ -258,7 +258,7 @@ available_node_types:
         # service is required.
         labels:
             parent: skypilot
-            # component will be set for the head node pod to be the same as the head node service selector above if a 
+            # component will be set for the head node pod to be the same as the head node service selector above if a
             skypilot-cluster: {{cluster_name_on_cloud}}
             # Identifies the SSH jump pod used by this pod. Used in life cycle management of the ssh jump pod.
             skypilot-ssh-jump: {{k8s_ssh_jump_name}}
@@ -267,18 +267,18 @@ available_node_types:
             {%- for label_key, label_value in labels.items() %}
             {{ label_key }}: {{ label_value|tojson }}
             {%- endfor %}
+            {% if high_availability %}
+            app: {{cluster_name_on_cloud}}
+            {% endif %}
       spec:
         # serviceAccountName: skypilot-service-account
         serviceAccountName: {{k8s_service_account_name}}
         automountServiceAccountToken: {{k8s_automount_sa_token}}
-        restartPolicy: Never
+        restartPolicy: {{ "Always" if high_availability else "Never" }}
 
         # Add node selector if GPU/TPUs are requested:
-        {% if (k8s_acc_label_key is not none and k8s_acc_label_value is not none) or (k8s_spot_label_key is not none) %}
+        {% if (k8s_topology_label_key is not none and k8s_topology_label_value is not none) or (k8s_spot_label_key is not none) %}
         nodeSelector:
-            {% if k8s_acc_label_key is not none and k8s_acc_label_value is not none %}
-            {{k8s_acc_label_key}}: {{k8s_acc_label_value}}
-            {% endif %}
             {% if k8s_topology_label_key is not none and k8s_topology_label_value is not none %}
             {{k8s_topology_label_key}}: {{k8s_topology_label_value}}
             {% endif %}
@@ -286,6 +286,19 @@ available_node_types:
             {{k8s_spot_label_key}}: {{k8s_spot_label_value|tojson}}
             {% endif %}
         {% endif %}
+        {% if (k8s_acc_label_key is not none and k8s_acc_label_values is not none) %}
+        affinity:
+          nodeAffinity:
+            requiredDuringSchedulingIgnoredDuringExecution:
+              nodeSelectorTerms:
+              - matchExpressions:
+                - key: {{k8s_acc_label_key}}
+                  operator: In
+                  values:
+                  {% for label_value in k8s_acc_label_values %}
+                  - {{label_value}}
+                  {% endfor %}
+        {% endif %}
 
         {% if k8s_spot_label_key is not none %}
         tolerations:
@@ -311,6 +324,11 @@ available_node_types:
             path: {{k8s_fusermount_shared_dir}}
             type: DirectoryOrCreate
         {% endif %}
+        {% if high_availability %}
+        - name: {{k8s_high_availability_deployment_volume_mount_name}}
+          persistentVolumeClaim:
+            claimName: {{cluster_name_on_cloud}}-{{k8s_high_availability_deployment_volume_mount_name}}
+        {% endif %}
         containers:
         - name: ray-node
           imagePullPolicy: IfNotPresent
@@ -331,15 +349,15 @@ available_node_types:
           # Do not change this command - it keeps the pod alive until it is
           # explicitly killed.
           command: ["/bin/bash", "-c", "--"]
-          args: 
+          args:
             - |
               # For backwards compatibility, we put a marker file in the pod
-              # to indicate that the pod is running with the changes introduced 
+              # to indicate that the pod is running with the changes introduced
               # in project nimbus: https://github.com/skypilot-org/skypilot/pull/4393
               # TODO: Remove this marker file and it's usage in setup_commands
               # after v0.10.0 release.
               touch /tmp/skypilot_is_nimbus
-              
+
               # Helper function to conditionally use sudo
               # TODO(zhwu): consolidate the two prefix_cmd and sudo replacements
               prefix_cmd() { if [ $(id -u) -ne 0 ]; then echo "sudo"; else echo ""; fi; }
@@ -382,7 +400,7 @@ available_node_types:
                 fi;
                 # SSH and other packages are not necessary, so we disable set -e
                 set +e
-                
+
                 if [ ! -z "$MISSING_PACKAGES" ]; then
                   # Install missing packages individually to avoid failure installation breaks the whole install process,
                   # e.g. fuse3 is not available on some distributions.
@@ -435,7 +453,7 @@ available_node_types:
                     $(prefix_cmd) rm -f /bin/fusermount-wrapper
                     $(prefix_cmd) cp -p {{k8s_fusermount_shared_dir}}/fusermount-wrapper /bin/fusermount-wrapper
                 fi
-                {% endif %}            
+                {% endif %}
 
                 $(prefix_cmd) mkdir -p /var/run/sshd;
                 $(prefix_cmd) sed -i "s/PermitRootLogin prohibit-password/PermitRootLogin yes/" /etc/ssh/sshd_config;
@@ -470,6 +488,7 @@ available_node_types:
                 done
                 {{ conda_installation_commands }}
                 {{ ray_installation_commands }}
+
                 VIRTUAL_ENV=~/skypilot-runtime ~/.local/bin/uv pip install skypilot[kubernetes,remote]
                 touch /tmp/ray_skypilot_installation_complete
                 echo "=== Ray and skypilot installation completed ==="
@@ -565,7 +584,7 @@ available_node_types:
                       # File is already being monitored
                       continue
                     fi
-                    
+
                     # Monitor the new file
                     monitor_file $file &
                     already_monitored="${already_monitored} ${file}"
@@ -573,6 +592,28 @@ available_node_types:
                   sleep 0.1
                 done
               }
+
+              {% if high_availability %}
+              mkdir -p {{k8s_high_availability_deployment_run_script_dir}}
+              if [ -f {{k8s_high_availability_deployment_volume_mount_path}}/k8s_container_ready ]; then
+                # ! Keep this aligned with `CloudVmRayBackend._setup()`
+                # Suppose all `task.setup` are the same for skyserve controller task.
+                # So be careful for compatibility issue once you change it.
+                chmod +x {{k8s_high_availability_deployment_setup_script_path}}
+                /bin/bash --login -c "true && export OMP_NUM_THREADS=1 PYTHONWARNINGS='ignore' && {{k8s_high_availability_deployment_setup_script_path}} > /tmp/controller_recovery_setup_commands.log 2>&1"
+                echo "=== Controller setup commands completed for recovery ==="
+
+                for file in {{k8s_high_availability_deployment_run_script_dir}}/*; do
+                  # ! Keep this aligned with `CloudVmRayBackend._execute()`
+                  chmod +x $file
+                  /bin/bash --login -c "true && export OMP_NUM_THREADS=1 PYTHONWARNINGS='ignore' && $file > /tmp/task_run_$(basename $file).log 2>&1"
+                  echo "=== Controller task run for service (file: $file) completed for recovery ==="
+                done
+              fi
+
+              touch {{k8s_high_availability_deployment_volume_mount_path}}/k8s_container_ready
+              {% endif %}
+
               trap : TERM INT; log_tail || sleep infinity & wait
 
           ports:
@@ -593,6 +634,10 @@ available_node_types:
           # /tmp which cause slowdowns if is not a shared memory volume.
           - mountPath: /dev/shm
             name: dshm
+          {% if high_availability %}
+          - name: {{k8s_high_availability_deployment_volume_mount_name}}
+            mountPath: {{k8s_high_availability_deployment_volume_mount_path}}
+          {% endif %}
           {% if k8s_fuse_device_required %}
           - name: fusermount-shared-dir
             mountPath: {{k8s_fusermount_shared_dir}}
@@ -616,7 +661,92 @@ available_node_types:
               {{k8s_resource_key}}: {{accelerator_count}}
               {% endif %}
             {% endif %}
-            
+      
+      {% if high_availability %}
+      pvc_spec:
+        apiVersion: v1
+        kind: PersistentVolumeClaim
+        metadata:
+          name: {{cluster_name_on_cloud}}-{{k8s_high_availability_deployment_volume_mount_name}}
+          namespace: {{k8s_namespace}}
+        spec:
+          accessModes:
+            - ReadWriteOnce # Our controller pod is singleton
+          {% if k8s_high_availability_storage_class_name is not none %}
+          storageClassName: {{k8s_high_availability_storage_class_name}}
+          {% endif %}
+          resources:
+            requests:
+              storage: {{disk_size}}Gi
+
+      deployment_spec:
+        apiVersion: apps/v1
+        kind: Deployment
+        metadata:
+          name: {{cluster_name_on_cloud}}-deployment
+          namespace: {{k8s_namespace}}
+        spec:
+          replicas: 1
+          selector:
+            matchLabels:
+              app: {{cluster_name_on_cloud}}
+          template:
+            # The only difference between the pod spec and this section is the initContainers
+            metadata:
+              # should be replaced by pod metadata
+            spec:
+              securityContext:
+                fsGroup: 1000
+              # To prevent the home dir provided by the docker image from being overriden by pvc mounting,
+              # we use initContainers to copy it first to /mnt/home, which will later be mounted to home dir.
+              initContainers:
+                - name: init-copy-home
+                  image: {{image_id}}
+                  command: ["/bin/sh", "-c"]
+                  args:
+                    - |
+                      # Define path for the marker file created by the main container upon successful startup.
+                      # This file persists in the PVC across Pod restarts.
+                      MARKER_FILE="/mnt/home/k8s_container_ready"
+                      SOURCE_PATH="{{k8s_high_availability_deployment_volume_mount_path}}"
+                      DEST_PATH="/mnt/home"
+
+                      # We only need to copy the initial home directory contents from the image
+                      # the *first* time a Pod uses a *new* PVC.
+                      # On subsequent Pod starts (e.g., after a crash or update), the PVC
+                      # already contains the necessary data (and potentially user modifications).
+                      # The presence of MARKER_FILE (created by the main container in a previous
+                      # successful run) indicates the PVC is already initialized. Checking for
+                      # it prevents unnecessary and time-consuming rsync operations on every restart.
+                      if [ ! -f "$MARKER_FILE" ]; then
+                        echo "Marker '$MARKER_FILE' not found. PVC likely needs initialization."
+                        echo "Copying initial home directory from image ($SOURCE_PATH/) to PVC ($DEST_PATH)..."
+
+                        # Use rsync with -rl (recursive, links) instead of -a (archive).
+                        # This avoids preserving times (-t) and permissions (-p) implied by -a,
+                        # which caused 'Operation not permitted' errors on the PVC root directory (/mnt/home).
+                        # Owner/group preservation (-o, -g) is also skipped (default for -rl), ensuring
+                        # files are owned by the container's user/group.
+                        rsync -rl "$SOURCE_PATH/" "$DEST_PATH"
+
+                        # Check if rsync failed
+                        if [ $? -ne 0 ]; then
+                           echo "ERROR: rsync failed during home directory initialization." >&2
+                           exit 1 # Exit initContainer with error if copy fails
+                        fi
+                        echo "Home directory initialization copy complete."
+                      else
+                        # If marker exists, skip the copy
+                        echo "Marker '$MARKER_FILE' found. Skipping initial home directory copy."
+                      fi
+                      echo "Current contents of $DEST_PATH:"
+                      ls -la "$DEST_PATH"
+                  volumeMounts:
+                    # Mount the persistent volume claim into the initContainer
+                    - name: {{k8s_high_availability_deployment_volume_mount_name}}
+                      mountPath: /mnt/home # Temporary mount point for initialization
+              # should be replaced by pod spec
+      {% endif %}
 setup_commands:
   # Disable `unattended-upgrades` to prevent apt-get from hanging. It should be called at the beginning before the process started to avoid being blocked. (This is a temporary fix.)
   # Add ~/.ssh/sky-cluster-key to SSH config to allow nodes within a cluster to connect to each other
@@ -636,7 +766,7 @@ setup_commands:
     echo "=== Logs for asynchronous ray and skypilot installation ===";
     if [ -f /tmp/skypilot_is_nimbus ]; then
       echo "=== Logs for asynchronous ray and skypilot installation ===";
-      [ -f /tmp/ray_skypilot_installation_complete ] && cat /tmp/${STEPS[1]}.log || 
+      [ -f /tmp/ray_skypilot_installation_complete ] && cat /tmp/${STEPS[1]}.log ||
       { tail -f -n +1 /tmp/${STEPS[1]}.log & TAIL_PID=$!; echo "Tail PID: $TAIL_PID"; until [ -f /tmp/ray_skypilot_installation_complete ]; do sleep 0.5; done; kill $TAIL_PID || true; };
       [ -f /tmp/${STEPS[1]}.failed ] && { echo "Error: ${STEPS[1]} failed. Exiting."; exit 1; } || true;
     fi
@@ -666,7 +796,7 @@ setup_commands:
     # properly written.
     # TODO(Doyoung): Investigate to see why TPU workload fails to run without
     # execution permission, such as granting 766 to log file. Check if it's a
-    # must and see if there's a workaround to grant minimum permission. 
+    # must and see if there's a workaround to grant minimum permission.
     sudo chmod 777 /tmp/tpu_logs;
     {% endif %}
 

sky/templates/nebius-ray.yml.j2

@@ -10,6 +10,27 @@ provider:
   module: sky.provision.nebius
   region: "{{region}}"
 
+{%- if docker_image is not none %}
+docker:
+  image: {{docker_image}}
+  container_name: {{docker_container_name}}
+  run_options:
+    - --ulimit nofile=1048576:1048576
+    {%- for run_option in docker_run_options %}
+    - {{run_option}}
+    {%- endfor %}
+  {%- if docker_login_config is not none %}
+  docker_login_config:
+    username: |-
+      {{docker_login_config.username}}
+    password: |-
+      {{docker_login_config.password}}
+    server: |-
+      {{docker_login_config.server}}
+  {%- endif %}
+{%- endif %}
+
+
 auth:
   ssh_user: ubuntu
   ssh_private_key: {{ssh_private_key}}
@@ -22,6 +43,48 @@ available_node_types:
       ImageId: {{image_id}}
       DiskSize: {{disk_size}}
       UserData: |
+        {%- if docker_image is not none %}
+        runcmd:
+          - sudo sed -i 's/^#\?AllowTcpForwarding.*/AllowTcpForwarding yes/' /etc/ssh/sshd_config
+          - systemctl restart sshd
+        {%- endif %}
+
+        {# Two available OS images:
+               1. ubuntu22.04-driverless - requires Docker installation
+               2. ubuntu22.04-cuda12 - comes with Docker pre-installed
+            To optimize deployment speed, Docker is only installed when using ubuntu22.04-driverless #}
+        {%- if docker_image is not none and image_id == 'ubuntu22.04-driverless' %}
+        apt:
+          sources:
+            docker.list:
+              source: deb [arch=amd64] https://download.docker.com/linux/ubuntu $RELEASE stable
+              keyid: 9DC858229FC7DD38854AE2D88D81803C0EBFCD88
+
+        packages:
+          - apt-transport-https
+          - ca-certificates
+          - curl
+          - gnupg-agent
+          - software-properties-common
+          - docker-ce
+          - docker-ce-cli
+          - containerd.io
+
+        # Enable ipv4 forwarding, required on CIS hardened machines
+        write_files:
+          - path: /etc/sysctl.d/enabled_ipv4_forwarding.conf
+            content: |
+              net.ipv4.conf.all.forwarding=1
+
+        # create the docker group
+        groups:
+          - docker
+
+        # Add default auto created user to docker group
+        system_info:
+          default_user:
+            groups: [docker]
+        {%- endif %}
         users:
           - name: skypilot:ssh_user
             shell: /bin/bash

sky/utils/command_runner.py

@@ -325,6 +325,7 @@ class CommandRunner:
         direction = 'up' if up else 'down'
         error_msg = (f'Failed to rsync {direction}: {source} -> {target}. '
                      'Ensure that the network is stable, then retry.')
+
         subprocess_utils.handle_returncode(returncode,
                                            command,
                                            error_msg,
@@ -718,6 +719,7 @@ class KubernetesCommandRunner(CommandRunner):
     def __init__(
         self,
         node: Tuple[Tuple[str, Optional[str]], str],
+        deployment: Optional[str] = None,
         **kwargs,
     ):
         """Initialize KubernetesCommandRunner.
@@ -733,11 +735,19 @@ class KubernetesCommandRunner(CommandRunner):
         del kwargs
         super().__init__(node)
         (self.namespace, self.context), self.pod_name = node
+        self.deployment = deployment
 
     @property
     def node_id(self) -> str:
         return f'{self.context}-{self.namespace}-{self.pod_name}'
 
+    @property
+    def kube_identifier(self) -> str:
+        if self.deployment is not None:
+            return f'deployment/{self.deployment}'
+        else:
+            return f'pod/{self.pod_name}'
+
     def port_forward_command(self,
                              port_forward: List[Tuple[int, int]],
                              connect_timeout: int = 1) -> List[str]:
@@ -758,11 +768,12 @@ class KubernetesCommandRunner(CommandRunner):
             kubectl_args += ['--context', self.context]
         local_port, remote_port = port_forward[0]
         local_port_str = f'{local_port}' if local_port is not None else ''
+
         kubectl_cmd = [
             'kubectl',
             *kubectl_args,
             'port-forward',
-            f'pod/{self.pod_name}',
+            self.kube_identifier,
             f'{local_port_str}:{remote_port}',
         ]
         return kubectl_cmd
@@ -785,7 +796,8 @@ class KubernetesCommandRunner(CommandRunner):
             source_bashrc: bool = False,
             skip_num_lines: int = 0,
             **kwargs) -> Union[int, Tuple[int, str, str]]:
-        """Uses 'kubectl exec' to run 'cmd' on a pod by its name and namespace.
+        """Uses 'kubectl exec' to run 'cmd' on a pod or deployment by its
+        name and namespace.
 
         Args:
             cmd: The command to run.
@@ -828,7 +840,9 @@ class KubernetesCommandRunner(CommandRunner):
         # case, need to set KUBECONFIG to /dev/null to avoid using kubeconfig.
         if self.context is None:
             kubectl_args += ['--kubeconfig', '/dev/null']
-        kubectl_args += [self.pod_name]
+
+        kubectl_args += [self.kube_identifier]
+
         if ssh_mode == SshMode.LOGIN:
             assert isinstance(cmd, list), 'cmd must be a list for login mode.'
             base_cmd = ['kubectl', 'exec', '-it', *kubectl_args, '--']

sky/utils/command_runner.pyi

@@ -206,6 +206,8 @@ class KubernetesCommandRunner(CommandRunner):
     def __init__(
         self,
         node: Tuple[Tuple[str, Optional[str]], str],
+        deployment: Optional[str] = ...,
+        **kwargs,
     ) -> None:
         ...
 

sky/utils/controller_utils.py

@@ -193,6 +193,30 @@ class Controllers(enum.Enum):
         return None
 
 
+def high_availability_specified(cluster_name: Optional[str],
+                                skip_warning: bool = True) -> bool:
+    """Check if the controller high availability is specified in user config.
+    """
+    controller = Controllers.from_name(cluster_name)
+    if controller is None:
+        return False
+
+    if skypilot_config.loaded():
+        high_availability = skypilot_config.get_nested(
+            (controller.value.controller_type, 'controller',
+             'high_availability'), False)
+        if high_availability:
+            if controller.value.controller_type != 'serve':
+                if not skip_warning:
+                    print(f'{colorama.Fore.RED}High availability controller is'
+                          'only supported for SkyServe controller. It cannot'
+                          f'be enabled for {controller.value.name}.'
+                          f'Skipping this flag.{colorama.Style.RESET_ALL}')
+            else:
+                return True
+    return False
+
+
 # Install cli dependencies. Not using SkyPilot wheels because the wheel
 # can be cleaned up by another process.
 def _get_cloud_dependencies_installation_commands(

sky/utils/kubernetes/rsync_helper.sh

@@ -1,3 +1,4 @@
+#!/bin/bash
 # We need to determine the pod, namespace and context from the args
 # For backward compatibility, we use + as the separator between namespace and context and add handling when context is not provided
 if [ "$1" = "-l" ]; then
@@ -7,7 +8,7 @@ if [ "$1" = "-l" ]; then
     pod=$1
     shift
     encoded_namespace_context=$1
-    shift
+    shift # Shift past the encoded namespace+context
     echo "pod: $pod" >&2
     # Revert the encoded namespace+context to the original string.
     namespace_context=$(echo "$encoded_namespace_context" | sed 's|%40|@|g' | sed 's|%3A|:|g' | sed 's|%2B|+|g' | sed 's|%2F|/|g')
@@ -16,7 +17,7 @@ else
     # pod@namespace+context ...
     # used by openrsync
     encoded_pod_namespace_context=$1
-    shift
+    shift # Shift past the pod@namespace+context
     pod_namespace_context=$(echo "$encoded_pod_namespace_context" | sed 's|%40|@|g' | sed 's|%3A|:|g' | sed 's|%2B|+|g' | sed 's|%2F|/|g')
     echo "pod_namespace_context: $pod_namespace_context" >&2
     pod=$(echo $pod_namespace_context | cut -d@ -f1)
@@ -24,16 +25,31 @@ else
     namespace_context=$(echo $pod_namespace_context | cut -d@ -f2-)
     echo "namespace_context: $namespace_context" >&2
 fi
+
 namespace=$(echo $namespace_context | cut -d+ -f1)
 echo "namespace: $namespace" >&2
 context=$(echo $namespace_context | grep '+' >/dev/null && echo $namespace_context | cut -d+ -f2- || echo "")
 echo "context: $context" >&2
 context_lower=$(echo "$context" | tr '[:upper:]' '[:lower:]')
 
+# Check if the resource is a pod or a deployment (or other type)
+if [[ "$pod" == *"/"* ]]; then
+    # Format is resource_type/resource_name
+    echo "Resource contains type: $pod" >&2
+    resource_type=$(echo $pod | cut -d/ -f1)
+    resource_name=$(echo $pod | cut -d/ -f2)
+    echo "Resource type: $resource_type, Resource name: $resource_name" >&2
+else
+    # For backward compatibility or simple pod name, assume it's a pod
+    resource_type="pod"
+    resource_name=$pod
+    echo "Assuming resource is a pod: $resource_name" >&2
+fi
+
 if [ -z "$context" ] || [ "$context_lower" = "none" ]; then
     # If context is none, it means we are using incluster auth. In this case,
     # use need to set KUBECONFIG to /dev/null to avoid using kubeconfig file.
-    kubectl exec -i $pod -n $namespace --kubeconfig=/dev/null -- "$@"
+    kubectl exec -i "$resource_type/$resource_name" -n "$namespace" --kubeconfig=/dev/null -- "$@"
 else
-    kubectl exec -i $pod -n $namespace --context=$context -- "$@"
+    kubectl exec -i "$resource_type/$resource_name" -n "$namespace" --context="$context" -- "$@"
 fi

sky/utils/schemas.py

@@ -759,6 +759,9 @@ def get_config_schema():
                 'additionalProperties': False,
                 'properties': {
                     'resources': resources_schema,
+                    'high_availability': {
+                        'type': 'boolean',
+                    },
                     'autostop': autostop_schema,
                 }
             },
@@ -920,6 +923,16 @@ def get_config_schema():
                         for type in kubernetes_enums.KubernetesAutoscalerType
                     ]
                 },
+                'high_availability': {
+                    'type': 'object',
+                    'required': [],
+                    'additionalProperties': False,
+                    'properties': {
+                        'storage_class_name': {
+                            'type': 'string',
+                        }
+                    }
+                },
             }
         },
         'oci': {

{skypilot_nightly-1.0.0.dev20250427.dist-info → skypilot_nightly-1.0.0.dev20250429.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: skypilot-nightly
-Version: 1.0.0.dev20250427
+Version: 1.0.0.dev20250429
 Summary: SkyPilot: An intercloud broker for the clouds
 Author: SkyPilot Team
 License: Apache 2.0
@@ -47,7 +47,6 @@ Requires-Dist: python-multipart
 Requires-Dist: aiofiles
 Requires-Dist: httpx
 Requires-Dist: setproctitle
-Requires-Dist: omegaconf<2.5,>=2.4.0dev3
 Provides-Extra: aws
 Requires-Dist: urllib3<2; extra == "aws"
 Requires-Dist: awscli>=1.27.10; extra == "aws"
@@ -204,6 +203,7 @@ Dynamic: summary
 
 ----
 :fire: *News* :fire:
+- [Apr 2025] Spin up **Qwen3** on your cluster/cloud: [**example**](./llm/qwen/)
 - [Mar 2025] Run and serve **Google Gemma 3** using SkyPilot [**example**](./llm/gemma3/)
 - [Feb 2025] Prepare and serve **Retrieval Augmented Generation (RAG) with DeepSeek-R1**: [**blog post**](https://blog.skypilot.co/deepseek-rag), [**example**](./llm/rag/)
 - [Feb 2025] Run and serve **DeepSeek-R1 671B** using SkyPilot and SGLang with high throughput: [**example**](./llm/deepseek-r1/)