skypilot-nightly 1.0.0.dev20241011__py3-none-any.whl → 1.0.0.dev20241013__py3-none-any.whl

sky/provision/azure/config.py

@@ -5,16 +5,18 @@ a cluster to be launched.
 """
 import hashlib
 import json
-import logging
 from pathlib import Path
 import random
 import time
 from typing import Any, Callable
 
+from sky import exceptions
+from sky import sky_logging
 from sky.adaptors import azure
 from sky.provision import common
+from sky.utils import common_utils
 
-logger = logging.getLogger(__name__)
+logger = sky_logging.init_logger(__name__)
 
 UNIQUE_ID_LEN = 4
 _DEPLOYMENT_NAME = 'skypilot-config'
@@ -92,10 +94,19 @@ def bootstrap_instances(
                 retry += 1
                 continue
             raise
+        except azure.exceptions().ClientAuthenticationError as e:
+            message = (
+                'Failed to authenticate with Azure. Please check your Azure '
+                f'credentials. Error: {common_utils.format_exception(e)}'
+            ).replace('\n', ' ')
+            logger.error(message)
+            raise exceptions.NoClusterLaunchedError(message) from e
     else:
-        raise TimeoutError(
+        message = (
             f'Timed out waiting for resource group {resource_group} to be '
             'deleted.')
+        logger.error(message)
+        raise TimeoutError(message)
 
     # load the template file
     current_path = Path(__file__).parent

sky/provision/azure/instance.py

@@ -441,15 +441,21 @@ def run_instances(region: str, cluster_name_on_cloud: str,
     if to_start_count > 0:
         resource_client = azure.get_client('resource', subscription_id)
         logger.debug(f'run_instances: Creating {to_start_count} instances.')
-        created_instances = _create_instances(
-            compute_client=compute_client,
-            resource_client=resource_client,
-            cluster_name_on_cloud=cluster_name_on_cloud,
-            resource_group=resource_group,
-            provider_config=provider_config,
-            node_config=config.node_config,
-            tags=tags,
-            count=to_start_count)
+        try:
+            created_instances = _create_instances(
+                compute_client=compute_client,
+                resource_client=resource_client,
+                cluster_name_on_cloud=cluster_name_on_cloud,
+                resource_group=resource_group,
+                provider_config=provider_config,
+                node_config=config.node_config,
+                tags=tags,
+                count=to_start_count)
+        except Exception as e:
+            err_message = common_utils.format_exception(
+                e, use_bracket=True).replace('\n', ' ')
+            logger.error(f'Failed to create instances: {err_message}')
+            raise
         created_instance_ids = [inst.name for inst in created_instances]
 
     non_running_instance_statuses = list(

sky/provision/kubernetes/instance.py

@@ -632,7 +632,9 @@ def run_instances(region: str, cluster_name_on_cloud: str,
     try:
         return _create_pods(region, cluster_name_on_cloud, config)
     except (kubernetes.api_exception(), config_lib.KubernetesError) as e:
-        logger.warning(f'run_instances: Error occurred when creating pods: {e}')
+        e_msg = common_utils.format_exception(e).replace('\n', ' ')
+        logger.warning('run_instances: Error occurred when creating pods: '
+                       f'{e_msg}')
         raise
 
 

sky/provision/kubernetes/utils.py

@@ -1998,3 +1998,28 @@ def get_context_from_config(provider_config: Dict[str, Any]) -> Optional[str]:
         # we need to use in-cluster auth.
         context = None
     return context
+
+
+def get_skypilot_pods(context: Optional[str] = None) -> List[Any]:
+    """Gets all SkyPilot pods in the Kubernetes cluster.
+
+    Args:
+        context: Kubernetes context to use. If None, uses the current context.
+
+    Returns:
+        A list of Kubernetes pod objects.
+    """
+    if context is None:
+        context = get_current_kube_config_context_name()
+
+    try:
+        pods = kubernetes.core_api(context).list_pod_for_all_namespaces(
+            label_selector='skypilot-cluster',
+            _request_timeout=kubernetes.API_TIMEOUT).items
+    except kubernetes.max_retry_error():
+        raise exceptions.ResourcesUnavailableError(
+            'Timed out trying to get SkyPilot pods from Kubernetes cluster. '
+            'Please check if the cluster is healthy and retry. To debug, run: '
+            'kubectl get pods --selector=skypilot-cluster --all-namespaces'
+        ) from None
+    return pods

sky/provision/provisioner.py

@@ -14,6 +14,7 @@ import colorama
 
 import sky
 from sky import clouds
+from sky import exceptions
 from sky import provision
 from sky import sky_logging
 from sky import status_lib
@@ -42,76 +43,50 @@ _TITLE = '\n\n' + '=' * 20 + ' {} ' + '=' * 20 + '\n'
 def _bulk_provision(
     cloud: clouds.Cloud,
     region: clouds.Region,
-    zones: Optional[List[clouds.Zone]],
     cluster_name: resources_utils.ClusterName,
     bootstrap_config: provision_common.ProvisionConfig,
 ) -> provision_common.ProvisionRecord:
     provider_name = repr(cloud)
     region_name = region.name
 
-    style = colorama.Style
-
-    if not zones:
-        # For Azure, zones is always an empty list.
-        zone_str = 'all zones'
-    else:
-        zone_str = ','.join(z.name for z in zones)
-
-    if isinstance(cloud, clouds.Kubernetes):
-        # Omit the region name for Kubernetes.
-        logger.info(f'{style.BRIGHT}Launching on {cloud}{style.RESET_ALL} '
-                    f'{cluster_name!r}.')
-    else:
-        logger.info(f'{style.BRIGHT}Launching on {cloud} '
-                    f'{region_name}{style.RESET_ALL} ({zone_str})')
-
     start = time.time()
-    with rich_utils.safe_status('[bold cyan]Launching[/]') as status:
+    # TODO(suquark): Should we cache the bootstrapped result?
+    #  Currently it is not necessary as bootstrapping takes
+    #  only ~3s, caching it seems over-engineering and could
+    #  cause other issues like the cache is not synced
+    #  with the cloud configuration.
+    config = provision.bootstrap_instances(provider_name, region_name,
+                                           cluster_name.name_on_cloud,
+                                           bootstrap_config)
+
+    provision_record = provision.run_instances(provider_name,
+                                               region_name,
+                                               cluster_name.name_on_cloud,
+                                               config=config)
+
+    backoff = common_utils.Backoff(initial_backoff=1, max_backoff_factor=3)
+    logger.debug(f'\nWaiting for instances of {cluster_name!r} to be ready...')
+    rich_utils.force_update_status(
+        ux_utils.spinner_message('Launching - Checking instance status',
+                                 str(provision_logging.config.log_path)))
+    # AWS would take a very short time (<<1s) updating the state of the
+    # instance.
+    time.sleep(1)
+    for retry_cnt in range(_MAX_RETRY):
         try:
-            # TODO(suquark): Should we cache the bootstrapped result?
-            #  Currently it is not necessary as bootstrapping takes
-            #  only ~3s, caching it seems over-engineering and could
-            #  cause other issues like the cache is not synced
-            #  with the cloud configuration.
-            config = provision.bootstrap_instances(provider_name, region_name,
-                                                   cluster_name.name_on_cloud,
-                                                   bootstrap_config)
-        except Exception as e:
-            logger.error(f'{colorama.Fore.YELLOW}Failed to configure '
-                         f'{cluster_name!r} on {cloud} {region} ({zone_str}) '
-                         'with the following error:'
-                         f'{colorama.Style.RESET_ALL}\n'
-                         f'{common_utils.format_exception(e)}')
-            raise
-
-        provision_record = provision.run_instances(provider_name,
-                                                   region_name,
-                                                   cluster_name.name_on_cloud,
-                                                   config=config)
-
-        backoff = common_utils.Backoff(initial_backoff=1, max_backoff_factor=3)
-        logger.debug(
-            f'\nWaiting for instances of {cluster_name!r} to be ready...')
-        status.update('[bold cyan]Launching - Checking instance status[/]')
-        # AWS would take a very short time (<<1s) updating the state of the
-        # instance.
-        time.sleep(1)
-        for retry_cnt in range(_MAX_RETRY):
-            try:
-                provision.wait_instances(provider_name,
-                                         region_name,
-                                         cluster_name.name_on_cloud,
-                                         state=status_lib.ClusterStatus.UP)
-                break
-            except (aws.botocore_exceptions().WaiterError, RuntimeError):
-                time.sleep(backoff.current_backoff())
-        else:
-            raise RuntimeError(
-                f'Failed to wait for instances of {cluster_name!r} to be '
-                f'ready on the cloud provider after max retries {_MAX_RETRY}.')
-        logger.debug(
-            f'Instances of {cluster_name!r} are ready after {retry_cnt} '
-            'retries.')
+            provision.wait_instances(provider_name,
+                                     region_name,
+                                     cluster_name.name_on_cloud,
+                                     state=status_lib.ClusterStatus.UP)
+            break
+        except (aws.botocore_exceptions().WaiterError, RuntimeError):
+            time.sleep(backoff.current_backoff())
+    else:
+        raise RuntimeError(
+            f'Failed to wait for instances of {cluster_name!r} to be '
+            f'ready on the cloud provider after max retries {_MAX_RETRY}.')
+    logger.debug(f'Instances of {cluster_name!r} are ready after {retry_cnt} '
+                 'retries.')
 
     logger.debug(
         f'\nProvisioning {cluster_name!r} took {time.time() - start:.2f} '
@@ -162,8 +137,11 @@ def bulk_provision(
             logger.debug(
                 'Provision config:\n'
                 f'{json.dumps(dataclasses.asdict(bootstrap_config), indent=2)}')
-            return _bulk_provision(cloud, region, zones, cluster_name,
+            return _bulk_provision(cloud, region, cluster_name,
                                    bootstrap_config)
+        except exceptions.NoClusterLaunchedError:
+            # Skip the teardown if the cluster was never launched.
+            raise
         except Exception:  # pylint: disable=broad-except
             zone_str = 'all zones'
             if zones:
@@ -440,23 +418,30 @@ def _post_provision_setup(
     # We don't set docker_user here, as we are configuring the VM itself.
     ssh_credentials = backend_utils.ssh_credential_from_yaml(
         cluster_yaml, ssh_user=cluster_info.ssh_user)
+    docker_config = config_from_yaml.get('docker', {})
 
     with rich_utils.safe_status(
-            '[bold cyan]Launching - Waiting for SSH access[/]') as status:
+            ux_utils.spinner_message(
+                'Launching - Waiting for SSH access',
+                provision_logging.config.log_path)) as status:
 
         logger.debug(
             f'\nWaiting for SSH to be available for {cluster_name!r} ...')
         wait_for_ssh(cluster_info, ssh_credentials)
-        logger.debug(f'SSH Conection ready for {cluster_name!r}')
+        logger.debug(f'SSH Connection ready for {cluster_name!r}')
+        vm_str = 'Instance' if cloud_name.lower() != 'kubernetes' else 'Pod'
         plural = '' if len(cluster_info.instances) == 1 else 's'
-        logger.info(f'{colorama.Fore.GREEN}Successfully provisioned '
-                    f'or found existing instance{plural}.'
-                    f'{colorama.Style.RESET_ALL}')
+        verb = 'is' if len(cluster_info.instances) == 1 else 'are'
+        indent_str = (ux_utils.INDENT_SYMBOL
+                      if docker_config else ux_utils.INDENT_LAST_SYMBOL)
+        logger.info(f'{indent_str}{colorama.Style.DIM}{vm_str}{plural} {verb} '
+                    f'up.{colorama.Style.RESET_ALL}')
 
-        docker_config = config_from_yaml.get('docker', {})
         if docker_config:
             status.update(
-                '[bold cyan]Launching - Initializing docker container[/]')
+                ux_utils.spinner_message(
+                    'Launching - Initializing docker container',
+                    provision_logging.config.log_path))
             docker_user = instance_setup.initialize_docker(
                 cluster_name.name_on_cloud,
                 docker_config=docker_config,
@@ -470,6 +455,8 @@ def _post_provision_setup(
             cluster_info.docker_user = docker_user
             ssh_credentials['docker_user'] = docker_user
             logger.debug(f'Docker user: {docker_user}')
+            logger.info(f'{ux_utils.INDENT_LAST_SYMBOL}{colorama.Style.DIM}'
+                        f'Docker container is up.{colorama.Style.RESET_ALL}')
 
         # We mount the metadata with sky wheel for speedup.
         # NOTE: currently we mount all credentials for all nodes, because
@@ -482,8 +469,9 @@ def _post_provision_setup(
         # for later.
         file_mounts = config_from_yaml.get('file_mounts', {})
 
-        runtime_preparation_str = ('[bold cyan]Preparing SkyPilot '
-                                   'runtime ({step}/3 - {step_name})')
+        runtime_preparation_str = (ux_utils.spinner_message(
+            'Preparing SkyPilot runtime ({step}/3 - {step_name})',
+            provision_logging.config.log_path))
         status.update(
             runtime_preparation_str.format(step=1, step_name='initializing'))
         instance_setup.internal_file_mounts(cluster_name.name_on_cloud,
@@ -551,8 +539,9 @@ def _post_provision_setup(
         instance_setup.start_skylet_on_head_node(cluster_name.name_on_cloud,
                                                  cluster_info, ssh_credentials)
 
-    logger.info(f'{colorama.Fore.GREEN}Successfully provisioned cluster: '
-                f'{cluster_name}{colorama.Style.RESET_ALL}')
+    logger.info(
+        ux_utils.finishing_message(f'Cluster launched: {cluster_name}.',
+                                   provision_logging.config.log_path))
     return cluster_info
 
 

sky/serve/core.py

@@ -129,8 +129,10 @@ def up(
         task, use_mutated_config_in_current_request=False)
     task = dag.tasks[0]
 
-    controller_utils.maybe_translate_local_file_mounts_and_sync_up(task,
-                                                                   path='serve')
+    with rich_utils.safe_status(
+            ux_utils.spinner_message('Initializing service')):
+        controller_utils.maybe_translate_local_file_mounts_and_sync_up(
+            task, path='serve')
 
     with tempfile.NamedTemporaryFile(
             prefix=f'service-task-{service_name}-',
@@ -215,7 +217,8 @@ def up(
         # TODO(tian): Cache endpoint locally to speedup. Endpoint won't
         # change after the first time, so there is no consistency issue.
         with rich_utils.safe_status(
-                '[cyan]Waiting for the service to register[/]'):
+                ux_utils.spinner_message(
+                    'Waiting for the service to register')):
             # This function will check the controller job id in the database
             # and return the endpoint if the job id matches. Otherwise it will
             # return None.
@@ -274,34 +277,31 @@ def up(
             f'{style.BRIGHT}{service_name}{style.RESET_ALL}'
             f'\n{fore.CYAN}Endpoint URL: '
             f'{style.BRIGHT}{endpoint}{style.RESET_ALL}'
-            '\nTo see detailed info:\t\t'
-            f'{backend_utils.BOLD}sky serve status {service_name} '
-            f'[--endpoint]{backend_utils.RESET_BOLD}'
-            '\nTo teardown the service:\t'
-            f'{backend_utils.BOLD}sky serve down {service_name}'
-            f'{backend_utils.RESET_BOLD}'
-            '\n'
-            '\nTo see logs of a replica:\t'
-            f'{backend_utils.BOLD}sky serve logs {service_name} [REPLICA_ID]'
-            f'{backend_utils.RESET_BOLD}'
-            '\nTo see logs of load balancer:\t'
-            f'{backend_utils.BOLD}sky serve logs --load-balancer {service_name}'
-            f'{backend_utils.RESET_BOLD}'
-            '\nTo see logs of controller:\t'
-            f'{backend_utils.BOLD}sky serve logs --controller {service_name}'
-            f'{backend_utils.RESET_BOLD}'
-            '\n'
-            '\nTo monitor replica status:\t'
-            f'{backend_utils.BOLD}watch -n10 sky serve status {service_name}'
-            f'{backend_utils.RESET_BOLD}'
-            '\nTo send a test request:\t\t'
-            f'{backend_utils.BOLD}curl {endpoint}'
-            f'{backend_utils.RESET_BOLD}'
-            '\n'
-            f'\n{fore.GREEN}SkyServe is spinning up your service now.'
-            f'{style.RESET_ALL}'
-            f'\n{fore.GREEN}The replicas should be ready within a '
-            f'short time.{style.RESET_ALL}')
+            f'\n📋 Useful Commands'
+            f'\n{ux_utils.INDENT_SYMBOL}To check service status:\t'
+            f'{ux_utils.BOLD}sky serve status {service_name} '
+            f'[--endpoint]{ux_utils.RESET_BOLD}'
+            f'\n{ux_utils.INDENT_SYMBOL}To teardown the service:\t'
+            f'{ux_utils.BOLD}sky serve down {service_name}'
+            f'{ux_utils.RESET_BOLD}'
+            f'\n{ux_utils.INDENT_SYMBOL}To see replica logs:\t'
+            f'{ux_utils.BOLD}sky serve logs {service_name} [REPLICA_ID]'
+            f'{ux_utils.RESET_BOLD}'
+            f'\n{ux_utils.INDENT_SYMBOL}To see load balancer logs:\t'
+            f'{ux_utils.BOLD}sky serve logs --load-balancer {service_name}'
+            f'{ux_utils.RESET_BOLD}'
+            f'\n{ux_utils.INDENT_SYMBOL}To see controller logs:\t'
+            f'{ux_utils.BOLD}sky serve logs --controller {service_name}'
+            f'{ux_utils.RESET_BOLD}'
+            f'\n{ux_utils.INDENT_SYMBOL}To monitor the status:\t'
+            f'{ux_utils.BOLD}watch -n10 sky serve status {service_name}'
+            f'{ux_utils.RESET_BOLD}'
+            f'\n{ux_utils.INDENT_LAST_SYMBOL}To send a test request:\t'
+            f'{ux_utils.BOLD}curl {endpoint}'
+            f'{ux_utils.RESET_BOLD}'
+            '\n\n' +
+            ux_utils.finishing_message('Service is spinning up and replicas '
+                                       'will be ready shortly.'))
         return service_name, endpoint
 
 
@@ -323,11 +323,11 @@ def update(
         controller=controller_utils.Controllers.SKY_SERVE_CONTROLLER,
         stopped_message=
         'Service controller is stopped. There is no service to update. '
-        f'To spin up a new service, use {backend_utils.BOLD}'
-        f'sky serve up{backend_utils.RESET_BOLD}',
+        f'To spin up a new service, use {ux_utils.BOLD}'
+        f'sky serve up{ux_utils.RESET_BOLD}',
         non_existent_message='Service does not exist. '
         'To spin up a new service, '
-        f'use {backend_utils.BOLD}sky serve up{backend_utils.RESET_BOLD}',
+        f'use {ux_utils.BOLD}sky serve up{ux_utils.RESET_BOLD}',
     )
 
     backend = backend_utils.get_backend_from_handle(handle)
@@ -353,8 +353,8 @@ def update(
     if len(service_statuses) == 0:
         with ux_utils.print_exception_no_traceback():
             raise RuntimeError(f'Cannot find service {service_name!r}.'
-                               f'To spin up a service, use {backend_utils.BOLD}'
-                               f'sky serve up{backend_utils.RESET_BOLD}')
+                               f'To spin up a service, use {ux_utils.BOLD}'
+                               f'sky serve up{ux_utils.RESET_BOLD}')
 
     if len(service_statuses) > 1:
         with ux_utils.print_exception_no_traceback():
@@ -374,8 +374,10 @@ def update(
         with ux_utils.print_exception_no_traceback():
             raise RuntimeError(prompt)
 
-    controller_utils.maybe_translate_local_file_mounts_and_sync_up(task,
-                                                                   path='serve')
+    with rich_utils.safe_status(
+            ux_utils.spinner_message('Initializing service')):
+        controller_utils.maybe_translate_local_file_mounts_and_sync_up(
+            task, path='serve')
 
     code = serve_utils.ServeCodeGen.add_version(service_name)
     returncode, version_string_payload, stderr = backend.run_on_head(
@@ -433,8 +435,8 @@ def update(
 
     print(f'{colorama.Fore.GREEN}Service {service_name!r} update scheduled.'
           f'{colorama.Style.RESET_ALL}\n'
-          f'Please use {backend_utils.BOLD}sky serve status {service_name} '
-          f'{backend_utils.RESET_BOLD}to check the latest status.')
+          f'Please use {ux_utils.BOLD}sky serve status {service_name} '
+          f'{ux_utils.RESET_BOLD}to check the latest status.')
 
 
 @usage_lib.entrypoint

sky/sky_logging.py

@@ -10,10 +10,10 @@ import colorama
 from sky.utils import env_options
 from sky.utils import rich_utils
 
-# If the SKYPILOT_MINIMIZE_LOGGING environment variable is set to True,
-# remove logging prefixes and unnecessary information in optimizer
-_FORMAT = (None if env_options.Options.MINIMIZE_LOGGING.get() else
-           '%(levelname).1s %(asctime)s %(filename)s:%(lineno)d] %(message)s')
+# UX: Should we show logging prefixes and some extra information in optimizer?
+_show_logging_prefix = (env_options.Options.SHOW_DEBUG_INFO.get() or
+                        not env_options.Options.MINIMIZE_LOGGING.get())
+_FORMAT = '%(levelname).1s %(asctime)s %(filename)s:%(lineno)d] %(message)s'
 _DATE_FORMAT = '%m-%d %H:%M:%S'
 
 
@@ -45,6 +45,7 @@ _root_logger = logging.getLogger('sky')
 _default_handler = None
 _logging_config = threading.local()
 
+NO_PREFIX_FORMATTER = NewLineFormatter(None, datefmt=_DATE_FORMAT)
 FORMATTER = NewLineFormatter(_FORMAT, datefmt=_DATE_FORMAT)
 DIM_FORMATTER = NewLineFormatter(_FORMAT, datefmt=_DATE_FORMAT, dim=True)
 
@@ -67,7 +68,10 @@ def _setup_logger():
         else:
             _default_handler.setLevel(logging.INFO)
         _root_logger.addHandler(_default_handler)
-    _default_handler.setFormatter(FORMATTER)
+    if _show_logging_prefix:
+        _default_handler.setFormatter(FORMATTER)
+    else:
+        _default_handler.setFormatter(NO_PREFIX_FORMATTER)
     # Setting this will avoid the message
     # being propagated to the parent logger.
     _root_logger.propagate = False

sky/skylet/log_lib.py

@@ -21,6 +21,7 @@ from sky.skylet import constants
 from sky.skylet import job_lib
 from sky.utils import log_utils
 from sky.utils import subprocess_utils
+from sky.utils import ux_utils
 
 _SKY_LOG_WAITING_GAP_SECONDS = 1
 _SKY_LOG_WAITING_MAX_RETRY = 5
@@ -377,7 +378,9 @@ def _follow_job_logs(file,
                     wait_last_logs = False
                     continue
                 status_str = status.value if status is not None else 'None'
-                print(f'INFO: Job finished (status: {status_str}).')
+                print(
+                    ux_utils.finishing_message(
+                        f'Job finished (status: {status_str}).'))
                 return
 
             time.sleep(_SKY_LOG_TAILING_GAP_SECONDS)
@@ -412,8 +415,6 @@ def tail_logs(job_id: Optional[int],
         return
     logger.debug(f'Tailing logs for job, real job_id {job_id}, managed_job_id '
                  f'{managed_job_id}.')
-    logger.info(f'{colorama.Fore.YELLOW}Start streaming logs for {job_str}.'
-                f'{colorama.Style.RESET_ALL}')
     log_path = os.path.join(log_dir, 'run.log')
     log_path = os.path.expanduser(log_path)
 
@@ -437,7 +438,7 @@ def tail_logs(job_id: Optional[int],
         time.sleep(_SKY_LOG_WAITING_GAP_SECONDS)
         status = job_lib.update_job_status([job_id], silent=True)[0]
 
-    start_stream_at = 'INFO: Tip: use Ctrl-C to exit log'
+    start_stream_at = 'Waiting for task resources on '
     if follow and status in [
             job_lib.JobStatus.SETTING_UP,
             job_lib.JobStatus.PENDING,

sky/skylet/providers/lambda_cloud/node_provider.py

@@ -25,7 +25,7 @@ _TAG_PATH_PREFIX = '~/.sky/generated/lambda_cloud/metadata'
 _REMOTE_SSH_KEY_NAME = '~/.lambda_cloud/ssh_key_name'
 _REMOTE_RAY_SSH_KEY = '~/ray_bootstrap_key.pem'
 _REMOTE_RAY_YAML = '~/ray_bootstrap_config.yaml'
-_GET_INTERNAL_IP_CMD = 'ip -4 -br addr show | grep UP | grep -Eo "(10\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)|172\.(1[6-9]|2[0-9]|3[0-1]))\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)"'
+_GET_INTERNAL_IP_CMD = 's=$(ip -4 -br addr show | grep UP); echo "$s"; echo "$s" | grep -Eo "(10\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)|172\.(1[6-9]|2[0-9]|3[0-1])|104\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?))\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)"'
 
 logger = logging.getLogger(__name__)