skypilot-nightly 1.0.0.dev20241227__py3-none-any.whl → 1.0.0.dev20250124__py3-none-any.whl

sky/provision/do/utils.py

@@ -0,0 +1,306 @@
+"""DigitalOcean API client wrapper for SkyPilot.
+
+Example usage of `pydo` client library was mostly taken from here:
+https://github.com/digitalocean/pydo/blob/main/examples/poc_droplets_volumes_sshkeys.py
+"""
+
+import copy
+import os
+import typing
+from typing import Any, Dict, List, Optional
+import urllib
+import uuid
+
+from sky import sky_logging
+from sky.adaptors import do
+from sky.provision import common
+from sky.provision import constants as provision_constants
+from sky.provision.do import constants
+from sky.utils import common_utils
+
+if typing.TYPE_CHECKING:
+    from sky import resources
+    from sky import status_lib
+
+logger = sky_logging.init_logger(__name__)
+
+POSSIBLE_CREDENTIALS_PATHS = [
+    os.path.expanduser(
+        '~/Library/Application Support/doctl/config.yaml'),  # OS X
+    os.path.expanduser(
+        os.path.join(os.getenv('XDG_CONFIG_HOME', '~/.config/'),
+                     'doctl/config.yaml')),  # Linux
+]
+INITIAL_BACKOFF_SECONDS = 10
+MAX_BACKOFF_FACTOR = 10
+MAX_ATTEMPTS = 6
+SSH_KEY_NAME_ON_DO = f'sky-key-{common_utils.get_user_hash()}'
+
+CREDENTIALS_PATH = '~/.config/doctl/config.yaml'
+_client = None
+_ssh_key_id = None
+
+
+class DigitalOceanError(Exception):
+    pass
+
+
+def _init_client():
+    global _client, CREDENTIALS_PATH
+    assert _client is None
+    CREDENTIALS_PATH = None
+    credentials_found = 0
+    for path in POSSIBLE_CREDENTIALS_PATHS:
+        if os.path.exists(path):
+            CREDENTIALS_PATH = path
+            credentials_found += 1
+            logger.debug(f'Digital Ocean credential path found at {path}')
+    if not credentials_found > 1:
+        logger.debug('more than 1 credential file found')
+    if CREDENTIALS_PATH is None:
+        raise DigitalOceanError(
+            'no credentials file found from '
+            f'the following paths {POSSIBLE_CREDENTIALS_PATHS}')
+
+    # attempt default context
+    credentials = common_utils.read_yaml(CREDENTIALS_PATH)
+    default_token = credentials.get('access-token', None)
+    if default_token is not None:
+        try:
+            test_client = do.pydo.Client(token=default_token)
+            test_client.droplets.list()
+            logger.debug('trying `default` context')
+            _client = test_client
+            return _client
+        except do.exceptions().HttpResponseError:
+            pass
+
+    auth_contexts = credentials.get('auth-contexts', None)
+    if auth_contexts is not None:
+        for context, api_token in auth_contexts.items():
+            try:
+                test_client = do.pydo.Client(token=api_token)
+                test_client.droplets.list()
+                logger.debug(f'using {context} context')
+                _client = test_client
+                break
+            except do.exceptions().HttpResponseError:
+                continue
+        else:
+            raise DigitalOceanError(
+                'no valid api tokens found try '
+                'setting a new API token with `doctl auth init`')
+    return _client
+
+
+def client():
+    global _client
+    if _client is None:
+        _client = _init_client()
+    return _client
+
+
+def ssh_key_id(public_key: str):
+    global _ssh_key_id
+    if _ssh_key_id is None:
+        page = 1
+        paginated = True
+        while paginated:
+            try:
+                resp = client().ssh_keys.list(per_page=50, page=page)
+                for ssh_key in resp['ssh_keys']:
+                    if ssh_key['public_key'] == public_key:
+                        _ssh_key_id = ssh_key
+                        return _ssh_key_id
+            except do.exceptions().HttpResponseError as err:
+                raise DigitalOceanError(
+                    f'Error: {err.status_code} {err.reason}: '
+                    f'{err.error.message}') from err
+
+            pages = resp['links']
+            if 'pages' in pages and 'next' in pages['pages']:
+                pages = pages['pages']
+                parsed_url = urllib.parse.urlparse(pages['next'])
+                page = int(urllib.parse.parse_qs(parsed_url.query)['page'][0])
+            else:
+                paginated = False
+
+        request = {
+            'public_key': public_key,
+            'name': SSH_KEY_NAME_ON_DO,
+        }
+        _ssh_key_id = client().ssh_keys.create(body=request)['ssh_key']
+    return _ssh_key_id
+
+
+def _create_volume(request: Dict[str, Any]) -> Dict[str, Any]:
+    try:
+        resp = client().volumes.create(body=request)
+        volume = resp['volume']
+    except do.exceptions().HttpResponseError as err:
+        raise DigitalOceanError(
+            f'Error: {err.status_code} {err.reason}: {err.error.message}'
+        ) from err
+    else:
+        return volume
+
+
+def _create_droplet(request: Dict[str, Any]) -> Dict[str, Any]:
+    try:
+        resp = client().droplets.create(body=request)
+        droplet_id = resp['droplet']['id']
+
+        get_resp = client().droplets.get(droplet_id)
+        droplet = get_resp['droplet']
+    except do.exceptions().HttpResponseError as err:
+        raise DigitalOceanError(
+            f'Error: {err.status_code} {err.reason}: {err.error.message}'
+        ) from err
+    return droplet
+
+
+def create_instance(region: str, cluster_name_on_cloud: str, instance_type: str,
+                    config: common.ProvisionConfig) -> Dict[str, Any]:
+    """Creates a instance and mounts the requested block storage
+
+    Args:
+        region (str): instance region
+        instance_name (str): name of instance
+        config (common.ProvisionConfig): provisioner configuration
+
+    Returns:
+        Dict[str, Any]: instance metadata
+    """
+    # sort tags by key to support deterministic unit test stubbing
+    tags = dict(sorted(copy.deepcopy(config.tags).items()))
+    tags = {
+        'Name': cluster_name_on_cloud,
+        provision_constants.TAG_RAY_CLUSTER_NAME: cluster_name_on_cloud,
+        provision_constants.TAG_SKYPILOT_CLUSTER_NAME: cluster_name_on_cloud,
+        **tags
+    }
+    tags = [f'{key}:{value}' for key, value in tags.items()]
+    default_image = constants.GPU_IMAGES.get(
+        config.node_config['InstanceType'],
+        'gpu-h100x1-base',
+    )
+    image_id = config.node_config['ImageId']
+    image_id = image_id if image_id is not None else default_image
+    instance_name = (f'{cluster_name_on_cloud}-'
+                     f'{uuid.uuid4().hex[:4]}-{instance_type}')
+    instance_request = {
+        'name': instance_name,
+        'region': region,
+        'size': config.node_config['InstanceType'],
+        'image': image_id,
+        'ssh_keys': [
+            ssh_key_id(
+                config.authentication_config['ssh_public_key'])['fingerprint']
+        ],
+        'tags': tags,
+    }
+    instance = _create_droplet(instance_request)
+
+    volume_request = {
+        'size_gigabytes': config.node_config['DiskSize'],
+        'name': instance_name,
+        'region': region,
+        'filesystem_type': 'ext4',
+        'tags': tags
+    }
+    volume = _create_volume(volume_request)
+
+    attach_request = {'type': 'attach', 'droplet_id': instance['id']}
+    try:
+        client().volume_actions.post_by_id(volume['id'], attach_request)
+    except do.exceptions().HttpResponseError as err:
+        raise DigitalOceanError(
+            f'Error: {err.status_code} {err.reason}: {err.error.message}'
+        ) from err
+    logger.debug(f'{instance_name} created')
+    return instance
+
+
+def start_instance(instance: Dict[str, Any]):
+    try:
+        client().droplet_actions.post(droplet_id=instance['id'],
+                                      body={'type': 'power_on'})
+    except do.exceptions().HttpResponseError as err:
+        raise DigitalOceanError(
+            f'Error: {err.status_code} {err.reason}: {err.error.message}'
+        ) from err
+
+
+def stop_instance(instance: Dict[str, Any]):
+    try:
+        client().droplet_actions.post(
+            droplet_id=instance['id'],
+            body={'type': 'shutdown'},
+        )
+    except do.exceptions().HttpResponseError as err:
+        raise DigitalOceanError(
+            f'Error: {err.status_code} {err.reason}: {err.error.message}'
+        ) from err
+
+
+def down_instance(instance: Dict[str, Any]):
+    # We use dangerous destroy to atomically delete
+    # block storage and instance for autodown
+    try:
+        client().droplets.destroy_with_associated_resources_dangerous(
+            droplet_id=instance['id'], x_dangerous=True)
+    except do.exceptions().HttpResponseError as err:
+        if 'a destroy is already in progress' in err.error.message:
+            return
+        raise DigitalOceanError(
+            f'Error: {err.status_code} {err.reason}: {err.error.message}'
+        ) from err
+
+
+def rename_instance(instance: Dict[str, Any], new_name: str):
+    try:
+        client().droplet_actions.rename(droplet=instance['id'],
+                                        body={
+                                            'type': 'rename',
+                                            'name': new_name
+                                        })
+    except do.exceptions().HttpResponseError as err:
+        raise DigitalOceanError(
+            f'Error: {err.status_code} {err.reason}: {err.error.message}'
+        ) from err
+
+
+def filter_instances(
+        cluster_name_on_cloud: str,
+        status_filters: Optional[List[str]] = None) -> Dict[str, Any]:
+    """Returns Dict mapping instance name
+    to instance metadata filtered by status
+    """
+
+    filtered_instances: Dict[str, Any] = {}
+    page = 1
+    paginated = True
+    while paginated:
+        try:
+            resp = client().droplets.list(
+                tag_name=f'{provision_constants.TAG_SKYPILOT_CLUSTER_NAME}:'
+                f'{cluster_name_on_cloud}',
+                per_page=50,
+                page=page)
+            for instance in resp['droplets']:
+                if status_filters is None or instance[
+                        'status'] in status_filters:
+                    filtered_instances[instance['name']] = instance
+        except do.exceptions().HttpResponseError as err:
+            raise DigitalOceanError(
+                f'Error: {err.status_code} {err.reason}: {err.error.message}'
+            ) from err
+
+        pages = resp['links']
+        if 'pages' in pages and 'next' in pages['pages']:
+            pages = pages['pages']
+            parsed_url = urllib.parse.urlparse(pages['next'])
+            page = int(urllib.parse.parse_qs(parsed_url.query)['page'][0])
+        else:
+            paginated = False
+    return filtered_instances

sky/provision/docker_utils.py

@@ -38,6 +38,13 @@ class DockerLoginConfig:
     password: str
     server: str
 
+    def format_image(self, image: str) -> str:
+        """Format the image name with the server prefix."""
+        server_prefix = f'{self.server}/'
+        if not image.startswith(server_prefix):
+            return f'{server_prefix}{image}'
+        return image
+
     @classmethod
     def from_env_vars(cls, d: Dict[str, str]) -> 'DockerLoginConfig':
         return cls(
@@ -220,9 +227,7 @@ class DockerInitializer:
                 wait_for_docker_daemon=True)
             # We automatically add the server prefix to the image name if
             # the user did not add it.
-            server_prefix = f'{docker_login_config.server}/'
-            if not specific_image.startswith(server_prefix):
-                specific_image = f'{server_prefix}{specific_image}'
+            specific_image = docker_login_config.format_image(specific_image)
 
         if self.docker_config.get('pull_before_run', True):
             assert specific_image, ('Image must be included in config if ' +
@@ -338,14 +343,20 @@ class DockerInitializer:
         no_exist = 'NoExist'
         # SkyPilot: Add the current user to the docker group first (if needed),
         # before checking if docker is installed to avoid permission issues.
-        cleaned_output = self._run(
-            'id -nG $USER | grep -qw docker || '
-            'sudo usermod -aG docker $USER > /dev/null 2>&1;'
-            f'command -v {self.docker_cmd} || echo {no_exist!r}')
-        if no_exist in cleaned_output or 'docker' not in cleaned_output:
-            logger.error(
-                f'{self.docker_cmd.capitalize()} not installed. Please use an '
-                f'image with {self.docker_cmd.capitalize()} installed.')
+        docker_cmd = ('id -nG $USER | grep -qw docker || '
+                      'sudo usermod -aG docker $USER > /dev/null 2>&1;'
+                      f'command -v {self.docker_cmd} || echo {no_exist!r}')
+        cleaned_output = self._run(docker_cmd)
+        timeout = 60 * 10  # 10 minute timeout
+        start = time.time()
+        while no_exist in cleaned_output or 'docker' not in cleaned_output:
+            if time.time() - start > timeout:
+                logger.error(
+                    f'{self.docker_cmd.capitalize()} not installed. Please use '
+                    f'an image with {self.docker_cmd.capitalize()} installed.')
+                return
+            time.sleep(5)
+            cleaned_output = self._run(docker_cmd)
 
     def _check_container_status(self):
         if self.initialized:

sky/provision/gcp/instance_utils.py

@@ -38,7 +38,7 @@ _FIREWALL_RESOURCE_NOT_FOUND_PATTERN = re.compile(
     r'The resource \'projects/.*/global/firewalls/.*\' was not found')
 
 
-def _retry_on_http_exception(
+def _retry_on_gcp_http_exception(
     regex: Optional[str] = None,
     max_retries: int = GCP_MAX_RETRIES,
     retry_interval_s: int = GCP_RETRY_INTERVAL_SECONDS,
@@ -49,17 +49,18 @@ def _retry_on_http_exception(
 
         @functools.wraps(func)
         def wrapper(*args, **kwargs):
-            exception_type = gcp.http_error_exception()
 
             def try_catch_exc():
                 try:
                     value = func(*args, **kwargs)
                     return value
                 except Exception as e:  # pylint: disable=broad-except
-                    if not isinstance(e, exception_type) or (
-                            regex and not re.search(regex, str(e))):
-                        raise
-                    return e
+                    if (isinstance(e, gcp.http_error_exception()) and
+                        (regex is None or re.search(regex, str(e)))):
+                        logger.error(
+                            f'Retrying for gcp.http_error_exception: {e}')
+                        return e
+                    raise
 
             for _ in range(max_retries):
                 ret = try_catch_exc()
@@ -431,7 +432,7 @@ class GCPComputeInstance(GCPInstance):
         logger.debug(
             f'Waiting GCP operation {operation["name"]} to be ready ...')
 
-        @_retry_on_http_exception(
+        @_retry_on_gcp_http_exception(
             f'Failed to wait for operation {operation["name"]}')
         def call_operation(fn, timeout: int):
             request = fn(
@@ -613,6 +614,11 @@ class GCPComputeInstance(GCPInstance):
         return operation
 
     @classmethod
+    # When there is a cloud function running in parallel to set labels for
+    # newly created instances, it may fail with the following error:
+    #   "Labels fingerprint either invalid or resource labels have changed"
+    # We should retry until the labels are set successfully.
+    @_retry_on_gcp_http_exception('Labels fingerprint either invalid')
     def set_labels(cls, project_id: str, availability_zone: str, node_id: str,
                    labels: dict) -> None:
         node = cls.load_resource().instances().get(
@@ -1211,7 +1217,7 @@ class GCPTPUVMInstance(GCPInstance):
         """Poll for TPU operation until finished."""
         del project_id, region, zone  # unused
 
-        @_retry_on_http_exception(
+        @_retry_on_gcp_http_exception(
             f'Failed to wait for operation {operation["name"]}')
         def call_operation(fn, timeout: int):
             request = fn(name=operation['name'])
@@ -1379,7 +1385,7 @@ class GCPTPUVMInstance(GCPInstance):
                     f'Failed to get VPC name for instance {instance}') from e
 
     @classmethod
-    @_retry_on_http_exception('unable to queue the operation')
+    @_retry_on_gcp_http_exception('unable to queue the operation')
     def set_labels(cls, project_id: str, availability_zone: str, node_id: str,
                    labels: dict) -> None:
         while True:

sky/provision/kubernetes/instance.py

@@ -804,7 +804,8 @@ def _create_pods(region: str, cluster_name_on_cloud: str,
 
     # Create pods in parallel
     pods = subprocess_utils.run_in_parallel(_create_pod_thread,
-                                            range(to_start_count), _NUM_THREADS)
+                                            list(range(to_start_count)),
+                                            _NUM_THREADS)
 
     # Process created pods
     for pod in pods:
@@ -975,7 +976,7 @@ def terminate_instances(
         _terminate_node(namespace, context, pod_name)
 
     # Run pod termination in parallel
-    subprocess_utils.run_in_parallel(_terminate_pod_thread, pods.items(),
+    subprocess_utils.run_in_parallel(_terminate_pod_thread, list(pods.items()),
                                      _NUM_THREADS)
 
 

sky/provision/kubernetes/utils.py

@@ -7,6 +7,7 @@ import os
 import re
 import shutil
 import subprocess
+import time
 import typing
 from typing import Any, Dict, List, Optional, Set, Tuple, Union
 from urllib.parse import urlparse
@@ -105,6 +106,75 @@ ANNOTATIONS_POD_NOT_FOUND_ERROR_MSG = ('Pod {pod_name} not found in namespace '
 
 logger = sky_logging.init_logger(__name__)
 
+# Default retry settings for Kubernetes API calls
+DEFAULT_MAX_RETRIES = 3
+DEFAULT_RETRY_INTERVAL_SECONDS = 1
+
+
+def _retry_on_error(max_retries=DEFAULT_MAX_RETRIES,
+                    retry_interval=DEFAULT_RETRY_INTERVAL_SECONDS,
+                    resource_type: Optional[str] = None):
+    """Decorator to retry Kubernetes API calls on transient failures.
+
+    Args:
+        max_retries: Maximum number of retry attempts
+        retry_interval: Initial seconds to wait between retries
+        resource_type: Type of resource being accessed (e.g. 'node', 'pod').
+            Used to provide more specific error messages.
+    """
+
+    def decorator(func):
+
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            last_exception = None
+            backoff = common_utils.Backoff(initial_backoff=retry_interval,
+                                           max_backoff_factor=3)
+
+            for attempt in range(max_retries):
+                try:
+                    return func(*args, **kwargs)
+                except (kubernetes.max_retry_error(),
+                        kubernetes.api_exception(),
+                        kubernetes.config_exception()) as e:
+                    last_exception = e
+                    # Don't retry on permanent errors like 401 (Unauthorized)
+                    # or 403 (Forbidden)
+                    if (isinstance(e, kubernetes.api_exception()) and
+                            e.status in (401, 403)):
+                        raise
+                    if attempt < max_retries - 1:
+                        sleep_time = backoff.current_backoff()
+                        logger.debug(f'Kubernetes API call {func.__name__} '
+                                     f'failed with {str(e)}. Retrying in '
+                                     f'{sleep_time:.1f}s...')
+                        time.sleep(sleep_time)
+                        continue
+
+            # Format error message based on the type of exception
+            resource_msg = f' when trying to get {resource_type} info' \
+                if resource_type else ''
+            debug_cmd = f' To debug, run: kubectl get {resource_type}s' \
+                if resource_type else ''
+
+            if isinstance(last_exception, kubernetes.max_retry_error()):
+                error_msg = f'Timed out{resource_msg} from Kubernetes cluster.'
+            elif isinstance(last_exception, kubernetes.api_exception()):
+                error_msg = (f'Kubernetes API error{resource_msg}: '
+                             f'{str(last_exception)}')
+            else:
+                error_msg = (f'Kubernetes configuration error{resource_msg}: '
+                             f'{str(last_exception)}')
+
+            raise exceptions.ResourcesUnavailableError(
+                f'{error_msg}'
+                f' Please check if the cluster is healthy and retry.'
+                f'{debug_cmd}') from last_exception
+
+        return wrapper
+
+    return decorator
+
 
 class GPULabelFormatter:
     """Base class to define a GPU label formatter for a Kubernetes cluster
@@ -340,14 +410,15 @@ class GFDLabelFormatter(GPULabelFormatter):
         """
         canonical_gpu_names = [
             'A100-80GB', 'A100', 'A10G', 'H100', 'K80', 'M60', 'T4g', 'T4',
-            'V100', 'A10', 'P4000', 'P100', 'P40', 'P4', 'L4'
+            'V100', 'A10', 'P4000', 'P100', 'P40', 'P4', 'L40', 'L4'
         ]
         for canonical_name in canonical_gpu_names:
             # A100-80G accelerator is A100-SXM-80GB or A100-PCIE-80GB
             if canonical_name == 'A100-80GB' and re.search(
                     r'A100.*-80GB', value):
                 return canonical_name
-            elif canonical_name in value:
+            # Use word boundary matching to prevent substring matches
+            elif re.search(rf'\b{re.escape(canonical_name)}\b', value):
                 return canonical_name
 
         # If we didn't find a canonical name:
@@ -445,6 +516,7 @@ def detect_accelerator_resource(
 
 
 @functools.lru_cache(maxsize=10)
+@_retry_on_error(resource_type='node')
 def get_kubernetes_nodes(context: Optional[str] = None) -> List[Any]:
     """Gets the kubernetes nodes in the context.
 
@@ -453,17 +525,12 @@ def get_kubernetes_nodes(context: Optional[str] = None) -> List[Any]:
     if context is None:
         context = get_current_kube_config_context_name()
 
-    try:
-        nodes = kubernetes.core_api(context).list_node(
-            _request_timeout=kubernetes.API_TIMEOUT).items
-    except kubernetes.max_retry_error():
-        raise exceptions.ResourcesUnavailableError(
-            'Timed out when trying to get node info from Kubernetes cluster. '
-            'Please check if the cluster is healthy and retry. To debug, run: '
-            'kubectl get nodes') from None
+    nodes = kubernetes.core_api(context).list_node(
+        _request_timeout=kubernetes.API_TIMEOUT).items
     return nodes
 
 
+@_retry_on_error(resource_type='pod')
 def get_all_pods_in_kubernetes_cluster(
         context: Optional[str] = None) -> List[Any]:
     """Gets pods in all namespaces in kubernetes cluster indicated by context.
@@ -473,14 +540,8 @@ def get_all_pods_in_kubernetes_cluster(
     if context is None:
         context = get_current_kube_config_context_name()
 
-    try:
-        pods = kubernetes.core_api(context).list_pod_for_all_namespaces(
-            _request_timeout=kubernetes.API_TIMEOUT).items
-    except kubernetes.max_retry_error():
-        raise exceptions.ResourcesUnavailableError(
-            'Timed out when trying to get pod info from Kubernetes cluster. '
-            'Please check if the cluster is healthy and retry. To debug, run: '
-            'kubectl get pods') from None
+    pods = kubernetes.core_api(context).list_pod_for_all_namespaces(
+        _request_timeout=kubernetes.API_TIMEOUT).items
     return pods
 
 
@@ -892,6 +953,52 @@ def check_credentials(context: Optional[str],
         return True, None
 
 
+def check_pod_config(pod_config: dict) \
+    -> Tuple[bool, Optional[str]]:
+    """Check if the pod_config is a valid pod config
+
+    Using deserialize api to check the pod_config is valid or not.
+
+    Returns:
+        bool: True if pod_config is valid.
+        str: Error message about why the pod_config is invalid, None otherwise.
+    """
+    errors = []
+    # This api_client won't be used to send any requests, so there is no need to
+    # load kubeconfig
+    api_client = kubernetes.kubernetes.client.ApiClient()
+
+    # Used for kubernetes api_client deserialize function, the function will use
+    # data attr, the detail ref:
+    # https://github.com/kubernetes-client/python/blob/master/kubernetes/client/api_client.py#L244
+    class InnerResponse():
+
+        def __init__(self, data: dict):
+            self.data = json.dumps(data)
+
+    try:
+        # Validate metadata if present
+        if 'metadata' in pod_config:
+            try:
+                value = InnerResponse(pod_config['metadata'])
+                api_client.deserialize(
+                    value, kubernetes.kubernetes.client.V1ObjectMeta)
+            except ValueError as e:
+                errors.append(f'Invalid metadata: {str(e)}')
+        # Validate spec if present
+        if 'spec' in pod_config:
+            try:
+                value = InnerResponse(pod_config['spec'])
+                api_client.deserialize(value,
+                                       kubernetes.kubernetes.client.V1PodSpec)
+            except ValueError as e:
+                errors.append(f'Invalid spec: {str(e)}')
+        return len(errors) == 0, '.'.join(errors)
+    except Exception as e:  # pylint: disable=broad-except
+        errors.append(f'Validation error: {str(e)}')
+        return False, '.'.join(errors)
+
+
 def is_kubeconfig_exec_auth(
         context: Optional[str] = None) -> Tuple[bool, Optional[str]]:
     """Checks if the kubeconfig file uses exec-based authentication
@@ -1711,8 +1818,6 @@ def merge_dicts(source: Dict[Any, Any], destination: Dict[Any, Any]):
             else:
                 destination[key].extend(value)
         else:
-            if destination is None:
-                destination = {}
             destination[key] = value