osbot-utils 1.16.0__py3-none-any.whl → 1.20.0__py3-none-any.whl

osbot_utils/helpers/ssh/SSH__Execute.py

@@ -0,0 +1,158 @@
+from osbot_utils.base_classes.Type_Safe import Type_Safe
+from osbot_utils.context_managers.capture_duration          import capture_duration
+from osbot_utils.decorators.lists.group_by                  import group_by
+from osbot_utils.decorators.lists.index_by                  import index_by
+from osbot_utils.utils.Dev                                  import pprint
+from osbot_utils.utils.Env                                  import get_env
+from osbot_utils.utils.Misc                                 import str_to_int, str_to_bool
+from osbot_utils.utils.Process import start_process, run_process
+from osbot_utils.utils.Status                               import status_error
+
+ENV_VAR__SSH__HOST              = 'SSH__HOST'
+ENV_VAR__SSH__PORT              = 'SSH__PORT'
+ENV_VAR__SSH__KEY_FILE          = 'SSH__KEY_FILE'
+ENV_VAR__SSH__USER              = 'SSH__USER'
+ENV_VAR__SSH__STRICT_HOST_CHECK = 'SSH__STRICT_HOST_CHECK'
+
+
+class SSH__Execute(Type_Safe):
+    ssh_host          : str
+    ssh_port          : int  = 22
+    ssh_key_file      : str
+    ssh_key_user      : str
+    strict_host_check : bool = False
+
+    # execution & other commands # todo refactor into separate class
+    def exec(self, command):
+        return self.execute_command__return_stdout(command)
+
+    def exec__print(self, command):
+        result = self.execute_command__return_stdout(command)
+        self.print_header_for_command(command)
+        print(result)
+        return result
+
+    def execute_command(self, command):
+        if self.ssh_setup_ok()  and command:
+            ssh_args = self.execute_command_args(command)
+            with capture_duration() as duration:
+                result          = start_process("ssh", ssh_args)                                 # execute command using subprocess.run(...)
+            result['duration']  = duration.data()
+            return result
+        return status_error(error='in execute_command not all required vars were setup')
+
+    def execute_command__print(self, command):
+        self.print_header_for_command(command)
+        result = self.execute_command(command)
+        pprint(result)
+        return result
+
+    def execute_ssh_args(self):
+        ssh_args = []
+        if self.ssh_port:
+            ssh_args += ['-p', str(self.ssh_port)]
+        if self.strict_host_check is False:
+            ssh_args += ['-o', 'StrictHostKeyChecking=no']
+        if self.ssh_key_file:
+            ssh_args += ['-i', self.ssh_key_file]
+        return ssh_args
+
+    def execute_command_args(self, command=None):
+        ssh_args = self.execute_ssh_args()
+        if self.ssh_host:
+            ssh_args += [self.execute_command_target_host()]
+        if command:
+           ssh_args += [command]
+        return ssh_args
+
+    def execute_command_target_host(self):
+        if self.ssh_key_user:
+            return f'{self.ssh_key_user}@{self.ssh_host}'
+        else:
+            return f'{self.ssh_host}'
+
+    def execute_command__return_stdout(self, command):
+        return self.execute_command(command).get('stdout', '').strip()
+
+    def execute_command__return_stderr(self, command):
+        return self.execute_command(command).get('stderr', '').strip()
+
+    @index_by
+    @group_by
+    def execute_command__return_dict(self, command):
+        stdout = self.execute_command(command).get('stdout').strip()
+        return self.parse_stdout_to_dict(stdout)
+
+    @index_by
+    @group_by
+    def execute_command__return_list(self, command):
+        stdout = self.execute_command(command).get('stdout').strip()
+        return self.parse_stdout_to_list(stdout)
+
+    # setup commands             # todo refactor into separate class
+    def setup(self):
+        self.setup_using_env_vars()
+        return self
+
+    def setup_using_env_vars(self):         # move this to a CONFIG class (see code in SSH__Health_Check)
+        ssh_host              = get_env(ENV_VAR__SSH__HOST               )
+        ssh_port              = get_env(ENV_VAR__SSH__PORT              )
+        ssh_key_file          = get_env(ENV_VAR__SSH__KEY_FILE          )
+        ssh_key_user          = get_env(ENV_VAR__SSH__USER              )
+        ssh_strict_host_check = get_env(ENV_VAR__SSH__STRICT_HOST_CHECK )
+        if ssh_host:
+            self.ssh_host = ssh_host
+        if ssh_port:
+            self.ssh_port = str_to_int(ssh_port)
+        if ssh_key_file:
+            self.ssh_key_file = ssh_key_file
+        if ssh_key_user:
+            self.ssh_key_user = ssh_key_user
+        if ssh_strict_host_check is not None:
+            self.strict_host_check = str_to_bool(ssh_strict_host_check)
+
+    def parse_stdout_to_dict(self, stdout):
+        lines = stdout.splitlines()
+        headers = lines[0].split()
+        result = []
+
+        for line in lines[1:]:                                          # Split each line into parts based on whitespace
+            parts = line.split()                                        # Combine the parts with headers to create a dictionary
+            entry = {headers[i]: parts[i] for i in range(len(headers))}
+            result.append(entry)
+
+        return result
+
+    def parse_stdout_to_list(self, stdout):  # todo: add support for more ways to split the data
+        lines = stdout.splitlines()
+        return lines
+
+
+    def ssh_setup_ok(self):
+        # todo: add check to see if ssh executable exists (this check can be cached)
+        if self.ssh_host and self.ssh_key_file and self.ssh_key_user:
+            return True
+        return False
+
+    def ssh_not__setup_ok(self):
+        return self.ssh_setup_ok() is False
+
+    # print helpers
+    # def print_ls(self, path=''):
+    #     pprint(self.ls(path))
+    #     return self
+
+    def print_exec(self, command=''):
+        return self.exec__print(command)
+
+    def print_header_for_command(self, command):
+        print('\n')
+        print('*' * (30 + len(command)))
+        print(f'******   stdout for: {command}   ******')
+        print('*' * (30 + len(command)))
+        print()
+
+    def remove_server_ssh_host_fingerprint(self):           # todo: refactor to utils class
+        cmd_ssh_keyscan = "ssh-keygen"
+        cmd_remove_host = ['-R', f'[{self.ssh_host}]:{self.ssh_port}']
+        return run_process(cmd_ssh_keyscan, cmd_remove_host)

osbot_utils/helpers/ssh/SSH__Health_Check.py

@@ -0,0 +1,49 @@
+from osbot_utils.helpers.ssh.SSH__Execute import ENV_VAR__SSH__HOST, ENV_VAR__SSH__KEY_FILE, ENV_VAR__SSH__USER, \
+    ENV_VAR__SSH__PORT, ENV_VAR__SSH__STRICT_HOST_CHECK, SSH__Execute
+from osbot_utils.utils.Env import get_env
+from osbot_utils.utils.Misc import list_set
+from osbot_utils.utils.Status import status_ok, status_error
+
+ENV_VARS__FOR_SSH = {'ssh_host'         : ENV_VAR__SSH__HOST              ,
+                     'ssh_key_file'     : ENV_VAR__SSH__KEY_FILE          ,
+                     'ssh_key_user'     : ENV_VAR__SSH__USER              ,
+                     'ssh_port'         : ENV_VAR__SSH__PORT              ,
+                     'strict_host_check': ENV_VAR__SSH__STRICT_HOST_CHECK }
+
+class SSH__Health_Check(SSH__Execute):
+
+    def check_connection(self):
+        text_message = 'test connection'  #random_text('echo')
+        response = self.execute_command(f'echo {text_message}')
+        if response.get('status') == 'ok':
+            stderr = response.get('stderr').strip()
+            if stderr == '':
+                stdout = response.get('stdout').strip()
+                if stdout == text_message:
+                    return status_ok(message='connection ok')
+                else:
+                    return status_error(message=f'expected stdout did not march: {text_message} != {stdout}')
+            else:
+                return status_error(message=f'stderr was not empty', error=stderr, data=response)
+        else:
+            return status_error(message=f'request failed', data=response)
+
+    def env_vars_names(self):
+        return list_set(ENV_VARS__FOR_SSH)
+
+    def env_vars_values(self):
+        values = {}
+        for key, value in ENV_VARS__FOR_SSH.items():
+            env_value = get_env(value)
+            values[key] = env_value
+        return values
+
+    def env_vars_set_ok(self):
+        env_values = self.env_vars_values()
+        if  (env_values.get('ssh_host'    ) and
+             env_values.get('ssh_key_file') and
+             env_values.get('ssh_key_user')    ):
+                return True
+        return False
+
+

osbot_utils/helpers/ssh/SSH__Linux.py

@@ -0,0 +1,106 @@
+from osbot_utils.base_classes.Kwargs_To_Self    import Kwargs_To_Self
+from osbot_utils.decorators.lists.index_by      import index_by
+from osbot_utils.helpers.ssh.SSH__Execute       import SSH__Execute
+
+
+class SSH__Linux(Kwargs_To_Self):
+    ssh_execute : SSH__Execute
+
+    def apt_update(self):
+        return self.ssh_execute.execute_command__return_stdout('apt-get update')
+
+    def apt_install(self, package_name):
+        return self.ssh_execute.execute_command(f'apt-get install -y {package_name}')
+
+    def cat(self, path=''):
+        command = f'cat {path}'
+        return self.ssh_execute.execute_command__return_stdout(command)
+
+    @index_by
+    def disk_space(self):
+        command = "df -h"
+        stdout = self.ssh_execute.execute_command__return_stdout(command)
+        stdout_disk_space = stdout.replace('Mounted on', 'Mounted_on')  # todo, find a better way to do this
+        disk_space = self.ssh_execute.parse_stdout_to_dict(stdout_disk_space)
+        return disk_space
+
+    def dir_exists(self, folder_name):
+        message__folder_exists     = "Folder exists"
+        message__folder_not_exists = "Folder does not exist"
+        test_command               = f'test -d {folder_name} && echo "{message__folder_exists}"  || echo "{message__folder_not_exists}"'
+        result                     = self.ssh_execute.execute_command__return_stdout(test_command)
+        if result == message__folder_exists:
+            return True
+        if result == message__folder_not_exists:
+            return False
+
+    def echo(self, message):
+        return self.ssh_execute.execute_command__return_stdout(f"echo '{message}'")
+
+    def find(self, path=''):
+        command = f'find {path}'
+        return self.ssh_execute.execute_command__return_list(command)
+
+    def ls(self, path=''):
+        command = f'ls {path}'
+        ls_raw  = self.ssh_execute.execute_command__return_stdout(command)
+        return ls_raw.splitlines()
+
+    def memory_usage(self):
+        command = "free -h"
+        memory_usage_raw = self.ssh_execute.execute_command__return_stdout(command)     # todo: add fix for data parsing issue
+        return memory_usage_raw.splitlines()
+
+
+    def mkdir(self, folder):
+        command = f'mkdir -p {folder}'
+        return self.ssh_execute.execute_command(command)
+
+    def mv(self, source, destination):
+        command = f'mv {source} {destination}'
+        return self.ssh_execute.execute_command(command)
+
+    def pwd(self):
+        return self.ssh_execute.execute_command__return_stdout('pwd')
+
+    def rm(self, path=''):
+        command = f'rm {path}'
+        return self.ssh_execute.execute_command__return_stderr(command)
+
+    def rmdir(self, folder):
+        command = f'rmdir {folder}'
+        return self.ssh_execute.execute_command(command)
+
+    def running_processes(self,**kwargs):
+        command = "ps aux"
+        return self.ssh_execute.execute_command__return_dict(command, **kwargs)
+
+    def system_uptime(self):
+        command = "uptime"
+        uptime_raw = self.ssh_execute.execute_command__return_stdout(command)
+        return uptime_raw.strip()
+
+    def uname(self):
+        return self.ssh_execute.execute_command__return_stdout('uname')
+
+    def which(self, target):
+        command = f'which {target}'                                     # todo: security-vuln: add protection against code injection
+        return self.ssh_execute.execute_command__return_stdout(command)
+
+    def whoami(self):
+        command = f'whoami'
+        return self.ssh_execute.execute_command__return_stdout(command)
+
+    # todo: add methods below (and respective tests)
+
+    # def ifconfig(self):
+    #     command = "export PATH=$PATH:/sbin && ifconfig"               # todo add example with PATH modification
+    #     return self.execute_command__return_stdout(command)
+
+    # def ifconfig(self):                                               # todo add command to execute in separate bash (see when it is needed)
+    #     command = "bash -l -c 'ifconfig'"
+    #     return self.execute_command__return_stdout(command)
+    # if port_forward:      # todo: add support for port forward   (this will need async execution)
+    #     local_port  = port_forward.get('local_port' )
+    #     remote_ip   = port_forward.get('remote_ip'  )
+    #     remote_port = port_forward.get('remote_port')

osbot_utils/helpers/ssh/SSH__Python.py

@@ -0,0 +1,48 @@
+from osbot_utils.base_classes.Type_Safe import Type_Safe
+from osbot_utils.helpers.ssh.SSH__Execute import SSH__Execute
+from osbot_utils.helpers.ssh.SSH__Linux import SSH__Linux
+from osbot_utils.utils.Dev import pprint
+from osbot_utils.utils.Functions import function_source_code
+from osbot_utils.utils.Lists import list_index_by
+
+PYTHON3__LINUX__INSTALLER = 'python3'
+
+class SSH__Python(Type_Safe):
+    ssh_execute: SSH__Execute
+    ssh_linux  : SSH__Linux
+
+    def execute_python__code(self, python_code, python_executable='python3'):
+        python_command  = f"{python_executable} -c \"{python_code}\""
+        return self.ssh_execute.execute_command(python_command)
+
+    def execute_python__code__return_stdout(self, *args, **kwargs):
+        return self.execute_python__code(*args, **kwargs).get('stdout').strip()
+
+    def execute_python__function(self, function, python_executable='python3'):
+        function_name   = function.__name__
+        function_code   = function_source_code(function)
+        exec_code       = f"{function_code}\nresult= {function_name}(); print(result)"
+        return self.execute_python__code(exec_code)
+
+    def execute_python__function__return_stderr(self, *args, **kwargs):
+        return self.execute_python__function(*args, **kwargs).get('stderr').strip()
+
+    def execute_python__function__return_stdout(self, *args, **kwargs):
+        return self.execute_python__function(*args, **kwargs).get('stdout').strip()
+
+    def install_python3(self):
+        return self.ssh_linux.apt_install(PYTHON3__LINUX__INSTALLER)
+
+    def pip_list(self):
+        pip_list = self.ssh_execute.execute_command__return_dict('pip list')
+        return list_index_by(pip_list, 'Package')
+
+    def pip_install(self, package_name):
+        return self.ssh_execute.execute_command__return_stdout(f'pip install {package_name}')
+
+    def pip_version(self):
+        return self.ssh_execute.execute_command__return_stdout('pip --version')
+
+    def python_version(self):
+        return self.ssh_execute.execute_command__return_stdout('python3 --version')
+

osbot_utils/helpers/ssh/TestCase__SSH.py

@@ -0,0 +1,50 @@
+from unittest import TestCase
+
+import osbot_utils
+from osbot_utils.helpers.ssh.SSH                    import SSH
+from osbot_utils.helpers.ssh.SSH__Cache__Requests   import SSH__Cache__Requests
+from osbot_utils.helpers.ssh.SSH__Execute import SSH__Execute
+from osbot_utils.utils.Env                          import load_dotenv
+from osbot_utils.utils.Files                        import path_combine
+
+ENV_FILE__WITH_ENV_VARS           = "../.ssh.env"
+
+class TestCase__SSH(TestCase):
+    ssh  : SSH
+    cache: SSH__Cache__Requests
+
+    @classmethod
+    def setUpClass(cls):
+        cls.load_dotenv()
+        cls.ssh = SSH().setup()
+        if not cls.ssh.ssh_execute().ssh_host:
+            import pytest                           # we can only import this locally since this dependency doesn't exist in the main osbot_utils codebase
+            pytest.skip("SSH host not set")
+
+        cls.cache = SSH__Cache__Requests()
+        cls.cache.patch_apply()
+
+    @classmethod
+    def tearDownClass(cls):
+        cls.cache.patch_restore()
+        assert SSH__Execute.execute_command.__qualname__ == 'SSH__Execute.execute_command'
+
+    @staticmethod
+    def load_dotenv():
+        env_file_path = path_combine(osbot_utils.path, ENV_FILE__WITH_ENV_VARS)
+        load_dotenv(dotenv_path=env_file_path)
+
+    def cache_disable(self):
+        self.cache.disable()
+
+    def cache_update(self):
+        self.cache.update()
+
+    def ssh_execute(self):
+        return self.ssh.ssh_execute()
+
+    def ssh_linux(self):
+        return self.ssh.ssh_linux()
+
+    def ssh_python(self):
+        return self.ssh.ssh_python()

osbot_utils/helpers/trace/Trace_Call__Print_Lines.py

@@ -1,8 +1,8 @@
 from osbot_utils.utils.Lists import list_sorted
 from osbot_utils.base_classes.Kwargs_To_Self            import Kwargs_To_Self
-from osbot_utils.utils.Misc                             import ansi_text_visible_length
 from osbot_utils.helpers.trace.Trace_Call__Config       import Trace_Call__Config
 from osbot_utils.helpers.trace.Trace_Call__Print_Traces import text_grey, text_bold_green, text_olive, text_light_grey
+from osbot_utils.utils.Str import ansi_text_visible_length
 
 
 class Trace_Call__Print_Lines(Kwargs_To_Self):

osbot_utils/testing/Logging.py

@@ -8,8 +8,9 @@ from osbot_utils.decorators.methods.cache_on_self import cache_on_self
 
 #DEFAULT_LOG_FORMAT  = '%(asctime)s.%(msecs)03d %(levelname)s - %(message)s'
 DEFAULT_LOG_FORMAT  = '%(levelname)s - %(message)s'
-DEFAULT_LOG_LEVEL   = logging.DEBUG
+DEFAULT_LOG_LEVEL   = logging.INFO
 DEFAULT_DATE_FORMAT = '%M:%S'
+
 class Logging:
 
     def __init__(self, target=None, log_level: int = None, log_format=None, log_to_console=False, date_format=None):
@@ -27,7 +28,7 @@ class Logging:
         stream_handler = logging.StreamHandler(stream=stream)
         self.logger().addHandler(stream_handler)
         self.set_logger_level()
-        self.set_format(stream_handler)
+        self.set_format_on_stream_handler(stream_handler)
 
         return stream_handler
 
@@ -38,7 +39,7 @@ class Logging:
                 self.target = self.target.__name__
         return logging.getLogger(self.target)
 
-    def enable_log_to_console(self, log_level=logging.INFO):
+    def enable_log_to_console(self, log_level=None):
         self.log_to_sys_stdout()
         self.set_logger_level(log_level)
         return self
@@ -60,7 +61,7 @@ class Logging:
         return self.add_stream_handler(log_stream)
 
 
-    def set_format(self, stream_handler):
+    def set_format_on_stream_handler(self, stream_handler):
         formatter = logging.Formatter(fmt=self.log_format, datefmt=self.date_format)
         stream_handler.setFormatter(formatter)
         return formatter
@@ -71,7 +72,16 @@ class Logging:
         return self
 
     def set_logger_level(self, level=None):
-        self.logger().setLevel(level or self.log_level)
+        if level:
+            self.log_level = level
+        self.logger().setLevel(self.log_level)
+
+    def set_log_format(self, log_format=None, date_format=None):
+        if log_format:
+            self.log_format  = log_format
+        if date_format:
+            self.date_format = date_format
+        return self
 
 
     def info    (self,message, *args, **kwargs): self.logger().info     (message, *args, **kwargs)

osbot_utils/testing/Pytest.py

@@ -0,0 +1,18 @@
+from osbot_utils.utils.Env import get_env, load_dotenv, in_python_debugger
+
+needs_load_dotenv = True
+
+def skip_pytest(message=r"Skipping pytest for some reason ¯\_o_/¯"):
+    import pytest                                                # we can only import this locally since this dependency doesn't exist in the main osbot_utils codebase
+    pytest.skip(message)
+
+def skip_pytest__if_env_var_is_not_set(env_var_name):
+    if needs_load_dotenv:
+        load_dotenv()
+
+    if not get_env(env_var_name):
+        skip_pytest(f"Skipping tests because the {env_var_name} env var doesn't have a value")
+
+def skip__if_in_python_debugger():
+    if in_python_debugger():
+        skip_pytest("Skipping tests because we are in a debugger")

osbot_utils/utils/Env.py

@@ -1,11 +1,10 @@
-# In Misc.py
-import os
-from sys import platform
 
-from osbot_utils.utils.Dev import pprint
-from osbot_utils.utils.Files import all_parent_folders
-from osbot_utils.utils.Misc import list_set
-from osbot_utils.utils.Str import strip_quotes
+import os
+import sys
+from sys                        import platform
+from osbot_utils.utils.Files    import all_parent_folders, file_exists
+from osbot_utils.utils.Misc     import list_set
+from osbot_utils.utils.Str      import strip_quotes
 
 def env__home_root():
     return os.getenv('HOME') == '/root'
@@ -39,12 +38,14 @@ def env_vars(reload_vars=False):
     return data
 
 def env_load_from_file(path, override=False):
-    if os.path.exists(path):
+    if file_exists(path):
         with open(path) as f:
             for line in f:
                 line = line.strip()
                 if not line or line.startswith('#'):                # Strip whitespace and ignore comments
                     continue
+                if line.startswith('export '):                      # if the line starts with export, we can ignore it and continue
+                    line = line[7:]
                 key, value = line.split(sep='=', maxsplit=1)        # Split the line into key and value
                 value = strip_quotes(value.strip())                 # Handle case when the value is in quotes
                 if override or key.strip() not in os.environ:       # Set the environment variable
@@ -62,6 +63,20 @@ def env_unload_from_file(path):
                 if key in os.environ:  # Remove the environment variable if it exists
                     del os.environ[key]
 
+def in_github_action():
+    return os.getenv('GITHUB_ACTIONS') == 'true'
+
+def in_python_debugger():
+    if sys.gettrace() is not None:              # Check for a trace function
+        return True
+
+    pycharm_hosted           = os.getenv('PYCHARM_HOSTED') == '1'                     # Check for PyCharm specific environment variables and other potential indicators
+    pydevd_load_values_async = os.getenv('PYDEVD_LOAD_VALUES_ASYNC') is not None
+    if pycharm_hosted and pydevd_load_values_async:
+        return True
+
+    return False
+
 def load_dotenv(dotenv_path=None, override=False):
     if dotenv_path:                                                 # If a specific dotenv path is provided, load from it
         env_load_from_file(dotenv_path, override)
@@ -74,6 +89,10 @@ def load_dotenv(dotenv_path=None, override=False):
                 break                                               # Stop after loading the first .env file                                                     # Stop after loading the first .env file
 
 
+def not_in_github_action():
+    return in_github_action() is False
+
+
 def unload_dotenv(dotenv_path=None):
     if dotenv_path:                                                 # If a specific dotenv path is provided, unload from it
         env_unload_from_file(dotenv_path)
@@ -86,4 +105,5 @@ def unload_dotenv(dotenv_path=None):
                 break                                               # Stop after unloading the first .env file
 
 
-env_load = load_dotenv
+env_load = load_dotenv
+get_env  = os.getenv

osbot_utils/utils/Json.py

@@ -1,16 +1,9 @@
 import json
-import gzip
-import logging
 import os
 
 from osbot_utils.utils.Misc import str_lines, str_md5, str_sha256
-from osbot_utils.utils.Status import log_exception
-
-logger_json = logging.getLogger()   # todo: start using this API for capturing error messages from methods bellow
-
 from osbot_utils.utils.Files import file_create_gz, file_create, load_file_gz, file_contents, file_lines, file_lines_gz
 
-
 def json_dumps(python_object, indent=4, pretty=True, sort_keys=False, default=str, raise_exception=False):
     if python_object:
         try:
@@ -19,7 +12,7 @@ def json_dumps(python_object, indent=4, pretty=True, sort_keys=False, default=st
             return json.dumps(python_object, default=default)
         except Exception as error:
             error_message = f'Error in load_json: {error}'
-            log_exception(message=error_message, error=error)
+            #log_exception(message=error_message, error=error)              # todo: find a better way to do this , since this never worked well
             if raise_exception:
                 raise error
 
@@ -89,7 +82,7 @@ class Json:
             try:
                 return json.loads(json_data)
             except Exception as error:
-                log_exception(message='Error in load_json', error=error)
+                #log_exception(message='Error in load_json', error=error)
                 if raise_exception:
                     raise error