osbot-utils 1.7.7__py3-none-any.whl

osbot_utils/testing/Temp_Folder.py

@@ -0,0 +1,114 @@
+import random
+
+from osbot_utils.utils.Misc import random_string
+
+from osbot_utils.utils.Files import path_combine, temp_folder_current, safe_file_name, folder_exists, folder_create, \
+    folder_delete_recursively, files_list, file_create, folder_files, folders_recursive, files_find, files_recursive, \
+    temp_file_in_folder, create_folder, filter_parent_folder
+from osbot_utils.utils.Zip import zip_folder
+
+
+class Temp_Folder:
+
+    def __init__(self, folder_name=None, parent_folder=None, temp_prefix='', delete_on_exit=True, temp_files_to_add=0):
+        if type(parent_folder) is Temp_Folder:
+            parent_folder = parent_folder.path()
+        self.folder_name         = folder_name or f"temp_folder_{random_string(prefix=temp_prefix)}"
+        self.parent_folder       = parent_folder or temp_folder_current()
+        self.full_path           = path_combine(self.parent_folder, safe_file_name(self.folder_name))
+        self.delete_on_exit      = delete_on_exit
+        self.temp_files_to_add   = temp_files_to_add
+
+
+    def __enter__(self):
+        folder_create(self.full_path)
+        self.add_temp_files_and_folders(max_total_files=self.temp_files_to_add)
+        return self
+
+    def __exit__(self, type, value, traceback):
+        if self.delete_on_exit:
+            folder_delete_recursively(self.full_path)
+
+    def __repr__(self):
+        return f"<Temp_Folder: {self.full_path}>"
+
+    def __str__(self):
+        return self.full_path
+
+    def add_temp_files(self, count=0):
+        if count is None: count = 1
+        for i in range(count):
+            self.add_file()
+
+    def add_temp_files_and_folders(self, target_folder=None, max_depth_param=5, max_files_per_folder=4, max_total_files=20, current_depth=0):
+        if max_total_files <=0: return max_total_files
+        if target_folder is None: target_folder = self.full_path
+
+        # Base case: if max_depth_param is 0 or we've reached max_total_files, we stop
+        if max_depth_param == 0 or max_total_files <= 0:
+            return max_total_files
+
+        # Randomly decide the number of subfolders and files for this folder
+        num_subfolders = random.randint(1, max_depth_param)
+        num_files      = random.randint(1, min(max_files_per_folder, max_total_files))
+        #print(f'Creating {num_subfolders} subfolders and {num_files} files in {target_folder}')
+        # Create the random files
+        for _ in range(num_files):
+            temp_file_in_folder(target_folder,prefix= f'temp_file__{max_total_files}')
+            max_total_files -= 1
+            if max_total_files <= 0:
+                return max_total_files
+
+        current_depth +=1
+        # Recursively create subfolders and their contents
+        for _ in range(0, num_subfolders):
+            subfolder_name = f"_[{current_depth}]_temp_folder_{str(random.randint(0, 512))}_"
+            subfolder_path = path_combine(target_folder, subfolder_name)
+            create_folder(subfolder_path)
+
+            # Recursive call with decremented depth and updated max_total_files
+            max_total_files = self.add_temp_files_and_folders(subfolder_path, max_depth_param - 1, max_files_per_folder, max_total_files, current_depth=current_depth)
+
+            if max_total_files <= 0:
+                break
+
+        return max_total_files
+
+    def add_file(self, file_name=None, contents=None):
+        if file_name is None: file_name = f"temp_file_{random_string()}.txt"
+        if contents  is None: contents  = random_string()
+        file_path = path_combine(self.full_path, safe_file_name(file_name))
+        return file_create(file_path, contents)
+
+    def add_folder(self, name=None):
+        if name is None: name = f"temp_folder_{random_string()}"
+        new_folder = path_combine(self.path(), safe_file_name(name))
+        return folder_create(new_folder)
+
+
+    def exists(self):
+        return folder_exists(self.full_path)
+
+    def path(self):
+        return self.full_path
+
+    def files(self, show_parent_folder=False, include_folders=False):
+        all_files = files_recursive(self.path(), include_folders=include_folders)
+        if show_parent_folder:
+            return all_files
+        return filter_parent_folder(all_files, self.path())
+
+    def files_and_folders(self, show_parent_folder=False):
+        all_files_and_folders = files_recursive(self.path(), include_folders=True)
+        if show_parent_folder:
+            return all_files_and_folders
+        return filter_parent_folder(all_files_and_folders, self.path())
+
+    def folders(self, show_parent_folder=False):
+        all_folders = folders_recursive(self.path())
+        if show_parent_folder:
+            return all_folders
+        return filter_parent_folder(all_folders, self.path())
+
+    def zip(self):
+        return zip_folder(self.path())

osbot_utils/testing/Temp_Sys_Path.py

@@ -0,0 +1,13 @@
+import sys
+
+
+class Temp_Sys_Path:
+    def __init__(self, path):
+        self.path = path
+
+    def __enter__(self):
+        sys.path.append(self.path)
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        sys.path.remove(self.path)

osbot_utils/testing/Temp_Web_Server.py

@@ -0,0 +1,83 @@
+from contextlib import contextmanager
+from functools import partial
+from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer
+from threading import Thread
+from urllib.parse import urljoin
+
+from osbot_utils.utils.Files import file_create, path_combine, temp_filename, file_create_all_parent_folders
+
+from osbot_utils.utils.Misc import random_port, random_string
+
+from osbot_utils.utils.Http import port_is_open, GET
+
+
+class Temp_Web_Server:
+    server        : ThreadingHTTPServer
+    server_thread : Thread
+
+    def __init__(self, host: str = None, port: int = None, root_folder: str = None, server_name = None, http_handler = None, wait_for_stop=False):
+        self.host          = host         or "127.0.0.1"
+        self.port          = port         or random_port()
+        self.root_folder   = root_folder  or "."
+        self.server_name   = server_name  or "Temp_Web_Server"
+        self.http_handler  = http_handler or SimpleHTTPRequestHandler
+        self.wait_for_stop = wait_for_stop
+
+    def __enter__(self):
+        self.start()
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.stop()
+
+    def add_file(self, relative_file_path=None, file_contents=None):
+        if relative_file_path is None:
+            relative_file_path = temp_filename()
+        if file_contents is None:
+            file_contents = random_string()
+        full_path = path_combine(self.root_folder, relative_file_path)      # todo: fix the path transversal vulnerability that exists in this function #security
+        file_create_all_parent_folders(full_path)
+        file_create(path=full_path, contents=file_contents)
+        return full_path
+
+    def GET(self, path=''):
+        url = self.url(path)
+        try:
+            return GET(url)
+        except Exception as error:
+            print(error)                    # todo: add support for using logging
+            return None
+
+    def GET_contains(self, content, path=''):
+        page_html = self.GET(path=path)
+        if type(content) is list:
+            for item in content:
+                if item not in page_html:
+                    return False
+            return True
+        return content in page_html
+
+    def server_port_open(self):
+        return port_is_open(host=self.host, port=self.port)
+
+    def stop(self):
+        self.server.server_close()
+        if self.wait_for_stop:
+            self.server.shutdown()
+            self.server_thread.join()
+        else:
+            self.server._BaseServer__shutdown_request = True  # simulate what happens inside self.server.shutdown()
+
+    def start(self):
+        if self.http_handler is  SimpleHTTPRequestHandler:
+            handler_config = partial(self.http_handler, directory=self.root_folder)
+        else:
+            handler_config = partial(self.http_handler)
+        self.server        = ThreadingHTTPServer((self.host, self.port), handler_config)
+        self.server_thread = Thread(target=self.server.serve_forever, name=self.server_name)
+        self.server_thread.start()
+
+    def url(self,path=''):
+        base_url = f"http://{self.host}:{self.port}"
+        url      = urljoin(base_url, path)
+        return url

osbot_utils/testing/Temp_Zip.py

@@ -0,0 +1,45 @@
+from osbot_utils.testing.Temp_Folder import Temp_Folder
+from osbot_utils.utils.Files import Files, is_folder, file_exists, file_name
+from osbot_utils.utils.Zip import zip_folder, zip_file_list
+
+
+class Temp_Zip():
+    def __init__(self, target=None, target_zip_file=None, delete_zip_file=True):
+        if type(target) is Temp_Folder:
+            target = target.path()
+        self.target          = target
+        self.delete_zip_file = delete_zip_file
+        self.target_zip_file = target_zip_file
+        self.zip_file        = None
+        self.zip_bytes       = None
+        self.target_zipped   = False
+
+    def __enter__(self):
+        if is_folder(self.target):
+            self.zip_file = zip_folder(self.target)
+            self.target_zipped = True
+        return self
+
+    def __exit__(self, type, value, traceback):
+        if Files.exists(self.zip_file) and self.delete_zip_file:
+            Files.delete(self.zip_file)
+
+    def __repr__(self):
+        return f"<Temp_Zip: {self.path()}>"
+
+    def file_name(self):
+        return file_name(self.zip_file)
+
+    def path(self):
+        return self.zip_file
+
+    def files(self):
+        return zip_file_list(self.zip_file)
+
+    def print_path(self):
+        print()
+        print(self.path())
+        return self
+
+    def zip_file_exists(self):
+        return file_exists(self.zip_file)

osbot_utils/testing/Temp_Zip_In_Memory.py

@@ -0,0 +1,90 @@
+import io
+
+from osbot_utils.testing.Temp_File import Temp_File
+from osbot_utils.testing.Temp_Folder import Temp_Folder
+from osbot_utils.utils.Files import is_file, is_folder, files_recursive, filter_parent_folder, temp_file
+from osbot_utils.utils.Zip import zip_files_to_bytes, zip_bytes_file_list, zip_bytes_add_file, zip_bytes_get_file
+
+
+class Temp_Zip_In_Memory:
+
+    def __init__(self, targets=None, targets_as_bytes=None):
+        self.targets            = targets          or []
+        self.targets_as_content = targets_as_bytes or []
+        self.root_folder = None
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, type, value, traceback):
+        pass
+
+    def add_file(self, file, root_folder=None):
+        if type(file) is Temp_File:
+            file = file.path()
+        if is_file(file):
+            self.add_target(file, root_folder)
+        return self
+
+    def add_file_from_content(self, file_path, file_contents):
+        self.targets_as_content.append({'file_path': file_path, 'file_contents': file_contents})
+        return self
+
+    def add_folder(self, folder, root_folder=None):
+        if type(folder) is Temp_Folder:
+            folder = folder.path()
+        if is_folder(folder):
+            self.add_target(folder, root_folder)
+        return self
+
+    def add_target(self, target, root_folder=None):
+        if target:
+            self.targets.append({'target': target, 'root_folder': root_folder})
+        return self
+
+    def create_zip_file(self, target_zip_file=None):
+        if target_zip_file is None:
+            target_zip_file = temp_file(extension='.zip')
+        with open(target_zip_file, 'wb') as f:
+            f.write(self.zip_bytes())
+        return target_zip_file
+
+    def set_root_folder(self, root_folder):
+        if type(root_folder) is Temp_Folder:
+            self.root_folder = root_folder.path()
+        else:
+            self.root_folder = root_folder
+        return self
+
+    def target_files(self):
+        return [entry.get('file') for entry in self.target_files_with_root_folder()]
+
+    def target_files_with_root_folder(self):
+        all_files = []
+        for entry in self.targets:
+            target      = entry.get('target')
+            root_folder = entry.get('root_folder') or self.root_folder
+            if is_file(target):
+                all_files.append({'file': target, 'root_folder': root_folder})
+            elif is_folder(target):
+                for file in files_recursive(target):
+                    all_files.append({'file': file, 'root_folder': root_folder})
+        return all_files
+
+    def zip_bytes(self):
+        zip_bytes = self.zip_buffer().getvalue()
+        for items in self.targets_as_content:
+            file_path     = items.get('file_path')
+            file_contents = items.get('file_contents')
+            zip_bytes = zip_bytes_add_file(zip_bytes, file_path, file_contents)
+        return zip_bytes
+
+    def zip_bytes_file_content(self, file_path):
+        return zip_bytes_get_file(self.zip_bytes(), file_path)
+
+    def zip_bytes_files(self):
+        return zip_bytes_file_list(self.zip_bytes())
+
+    def zip_buffer(self):
+        targets = self.target_files_with_root_folder()
+        return zip_files_to_bytes(targets, root_folder=self.root_folder)

osbot_utils/testing/Unit_Test.py

@@ -0,0 +1,34 @@
+import base64
+from   unittest              import TestCase
+from   osbot_utils.utils.Dev import pprint
+
+
+class Unit_Test(TestCase):
+    """Unit test helpers
+
+        - self.result will be written to the output
+        - self.png_data will be saved to '/tmp/unit-test.png'"""
+
+    def setUp(self):
+        self.png_file = '/tmp/unit-test.png'
+        self.result   = None
+        self.png_data = None
+
+    def tearDown(self):
+        if self.result is not None:
+            pprint(self.result)
+        if self.png_data is not None:
+            if type(self.png_data) is bytes:
+                with open(self.png_file, "wb") as file:
+                    file.write(self.png_data)
+                pprint(f'Png data with size {len(self.png_data)} saved to {self.png_file}')
+            elif type(self.png_data) is str:
+                try:
+                    with open(self.png_file, "wb") as file:
+                        file.write(base64.decodebytes(self.png_data.encode()))
+                    pprint(f'Png data with size {len(self.png_data)} saved to {self.png_file}')
+                except Exception as error:
+                    pprint(f'png save error: {error}')
+                    pprint(self.png_data)
+            else:
+                pprint(f'Error Png data was not a string: {self.png_data}')

osbot_utils/testing/Unzip_File.py

@@ -0,0 +1,30 @@
+from osbot_utils.utils.Files import Files, folder_exists, folder_delete_all, temp_folder, files_recursive
+from osbot_utils.utils.Zip import unzip_file
+
+
+class Unzip_File:
+    def __init__(self, zip_file=None, target_folder=None, delete_target_folder=True):
+        self.target_folder          = target_folder
+        self.zip_file               = zip_file
+        self.delete_target_folder   = delete_target_folder
+        self.files_unzipped         = False
+        self.target_folder_deleted  = False
+
+    def __enter__(self):
+        if Files.exists(self.zip_file):
+            if self.target_folder is None:
+                self.target_folder = temp_folder("unzipped_")
+            unzip_file(self.zip_file, self.target_folder)
+            self.files_unzipped = True
+        return self
+
+    def __exit__(self, type, value, traceback):
+        if folder_exists(self.target_folder) and self.delete_target_folder:
+            self.target_folder_deleted = folder_delete_all(self.target_folder)
+            #print("\n\ndeleting", self.target_folder)
+
+    def path(self):
+        return self.target_folder
+
+    def files(self):
+        return files_recursive(self.target_folder)

osbot_utils/utils/Assert.py

@@ -0,0 +1,52 @@
+import re
+import datetime
+
+from osbot_utils.utils.Objects import class_name
+
+
+class Assert:
+    def __init__(self ,target):
+        self.target = target
+
+    def is_class(self, name):
+        assert class_name(self.target) in name
+
+    def contains(self, text):
+        assert text in self.target
+
+    def field_is_equal(self, field_name, expected_value=None):
+        field_value = self.target.get(field_name)
+        assert  field_value == expected_value , "{0} != {1}".format(field_value, expected_value)
+        return self
+
+    def is_bigger_than(self, value):
+        if type(self.target) is list:
+            list_len = len(self.target)
+            assert  list_len > value   , "array with len {0} was not bigger than {1}".format(list_len, value)
+        else:
+            assert self.target > value , "value {0} was not bigger than {1}".format(self.target, value)
+        return self
+
+    def is_smaller_than(self, value):
+        if type(self.target) is list:
+            list_len = len(self.target)
+            assert  list_len < value   , "array with len {0} was not smaller than {1}".format(list_len, value)
+        else:
+            assert self.target < value , "value {0} was not smaller than {1}".format(self.target, value)
+        return self
+
+    def is_equal(self, to):
+        assert self.target == to
+
+    def is_today(self):
+        assert type(self.target) == datetime.datetime
+        assert str(self.target.date())  == str(datetime.datetime.utcnow().date())
+
+    def match_regex(self, regex):
+        assert re.compile(regex).match(self.target) is not None
+
+    def size_is(self, to):
+        assert len(self.target) == to
+
+    def regex_not_match(self,regex):
+        assert re.compile(regex).match(self.target) is None

osbot_utils/utils/Call_Stack.py

@@ -0,0 +1,187 @@
+import linecache
+import sys
+import traceback
+from osbot_utils.helpers.CPrint              import CPrint
+from osbot_utils.helpers.Print_Table         import Print_Table, CHAR_TABLE_HORIZONTAL, CHAR_TABLE_TOP_LEFT, CHAR_TABLE_VERTICAL, CHAR_TABLE_BOTTOM_LEFT
+from osbot_utils.base_classes.Kwargs_To_Self import Kwargs_To_Self
+from osbot_utils.utils.Objects               import obj_data, class_full_name
+
+MODULES_TO_STOP_CAPTURE   = ['unittest.case']
+PRINT_STACK_COLOR_THEMES  = { 'default'    : ('none'          , 'none'  , 'none' ),
+                              'minty'      : ('green'         , 'grey'  , 'cyan' ) ,
+                              'meadow'     : ('blue'          , 'none'  ,'green' ) ,
+                              'aquamarine' : ('bright_cyan'   , 'cyan'  ,'blue'  ),
+                              'autumn'     : ('bright_yellow' , 'yellow','red'   )}
+
+class Frame_Data(Kwargs_To_Self):
+    depth         : int
+    caller_line   : str
+    method_line   : str
+    method_name   : str
+    line_number   : int
+    local_self    : str
+    module        : str
+
+    def __repr__(self):
+        return f"{self.data()})"
+
+    def data(self):
+        return self.__locals__()
+
+class Call_Stack(Kwargs_To_Self):
+    capture_locals : bool = False
+    cprint         : CPrint
+    cprint_theme   : str = 'meadow'
+    frames         : list
+    max_depth      : int  = 10
+
+    __print_order__: list = ['module', 'method_name', 'caller_line', 'method_line',  'local_self', 'line_number', 'depth']
+
+    def __repr__(self):
+        return "Call_Stack"
+
+    def calls(self):
+        calls = []
+        for frame in self.frames:
+            method_name = frame.method_name
+            module      = frame.module
+            call        = f"{module}.{method_name}"
+            calls.append(call)
+        return calls
+
+    def calls__source_code(self):
+        calls = []
+        for frame in self.frames:
+            calls.append(frame.caller_line)
+        return calls
+
+
+    def capture(self,skip_caller=True):
+        current_frame = sys._getframe().f_back
+        if skip_caller:
+            current_frame = current_frame.f_back
+        return self.capture_frame(current_frame)
+
+    def capture_frame(self, frame):
+        depth = 0
+        while frame:
+            if self.stop_capture(frame, depth):
+                break
+            new_frame = self.new_frame(frame,depth)
+            self.frames.append(new_frame)
+            depth += 1
+            frame = frame.f_back
+        return self
+
+    def stats(self):
+        return { 'depth' : len(self.frames)  }
+
+    def stop_capture(self, frame, depth):
+        if frame  is None:
+            return True
+        if depth > self.max_depth:
+            return True
+        module = frame.f_globals.get("__name__", "")
+
+        if module in MODULES_TO_STOP_CAPTURE:
+            return True
+        return False
+
+    def new_frame(self, frame, depth):
+        file_path          = frame.f_code.co_filename
+        method_name        = frame.f_code.co_name
+        caller_line_number = frame.f_lineno
+        method_line_number = frame.f_code.co_firstlineno
+
+        caller_line    = linecache.getline(file_path, caller_line_number).strip()
+        method_line    = linecache.getline(file_path, method_line_number).strip()   # see if we need to add the code to resolve the function from the decorators
+        module         = frame.f_globals.get("__name__", "")
+        local_self     = class_full_name(frame.f_locals.get('self'))
+        frame_data     = Frame_Data( depth          = depth              ,
+                                     caller_line    = caller_line        ,
+                                     method_line    = method_line        ,
+                                     method_name    = method_name        ,
+                                     module         = module             ,
+                                     local_self     = local_self         ,
+                                     line_number    = caller_line_number )
+        if self.capture_locals:
+            frame_data.locals = frame.f_locals,
+        return frame_data
+
+    def print(self):
+        for line in self.stack_lines__calls():
+            print(line)
+
+    def print__source_code(self):
+        for line in self.stack_lines__source_code():
+            print(line)
+
+    def stack_lines(self, items):
+        self.cprint.lines      = []
+        self.cprint.auto_print = False
+
+        if len(items) == 0:
+            return []
+
+        if len(items) == 1:
+            self.print_with_color('none', f"{CHAR_TABLE_HORIZONTAL} {items[0]}")
+        else:
+            color_top, color_middle, color_bottom = self.stack_colors()
+            self.print_with_color(color_top, text=f"{CHAR_TABLE_TOP_LEFT} {items[0]}")
+            for call in items[1:-1]:
+                self.print_with_color(color_middle, text=f"{CHAR_TABLE_VERTICAL} {call}")
+            self.print_with_color(color_bottom, text=f"{CHAR_TABLE_BOTTOM_LEFT} {items[-1]}")
+        return self.cprint.lines
+
+    def stack_lines__calls(self):
+        calls = self.calls()
+        return self.stack_lines(calls)
+
+    def stack_lines__source_code(self):
+        calls_source_code = self.calls__source_code()
+        return self.stack_lines(calls_source_code)
+
+    def stack_colors(self):
+        color_themes = PRINT_STACK_COLOR_THEMES
+        stack_colors = color_themes.get(self.cprint_theme)
+        if not stack_colors:
+            stack_colors = color_themes.get('default')
+        return stack_colors
+
+    def print_with_color(self, color_name, text):
+        self.cprint.__getattribute__(color_name)(text)
+
+    def print_table(self,headers_to_hide=None):
+        all_data = []
+        for frame in self.frames:
+            all_data.append(frame.data())
+
+        with Print_Table() as _:
+            _.add_data(all_data)
+            _.hide_headers(headers_to_hide)
+            _.print(order=self.__print_order__)
+
+
+# static methods
+# todo: refactor these static methods to be in a separate class and Call_Stack into the helpers folder
+
+def call_stack_current_frame(return_caller=True):
+    if return_caller:
+        return sys._getframe().f_back
+    return sys._getframe()
+
+def call_stack_format_stack(depth=None):
+    return traceback.format_stack(limit=depth)
+
+def call_stack_frames(depth=None):
+    return traceback.extract_stack(limit=depth)
+
+def call_stack_frames_data(depth=None):
+    frames_data = []
+    for frame in call_stack_frames(depth=depth):
+        frames_data.append(obj_data(frame))
+    return frames_data
+
+def frames_in_threads():
+    return sys._current_frames()
+

osbot_utils/utils/Csv.py

@@ -0,0 +1,32 @@
+import csv
+from io                                     import StringIO
+from osbot_utils.utils.Http                 import GET
+from osbot_utils.utils.Files                import file_open
+from osbot_utils.decorators.lists.group_by  import group_by
+from osbot_utils.decorators.lists.index_by  import index_by
+
+
+@index_by
+@group_by
+def load_csv_from_iterable(iterable, delimiter=','):
+    csv_reader = csv.DictReader(iterable, delimiter=delimiter)
+    return [row for row in csv_reader]
+
+@index_by
+@group_by
+def load_csv_from_file(file_path, delimiter=','):
+    iterable = file_open(file_path)
+    return load_csv_from_iterable(iterable, delimiter=delimiter)
+
+@index_by
+@group_by
+def load_csv_from_str(csv_data, delimiter=','):
+    iterable = StringIO(csv_data)
+    return load_csv_from_iterable(iterable, delimiter=delimiter)
+
+@index_by
+@group_by
+def load_csv_from_url(url, headers, delimiter=','):
+    csv_data = GET(url=url, headers=headers)
+    return load_csv_from_str(csv_data=csv_data, delimiter=delimiter)
+