osbot-utils 1.26.0__tar.gz → 1.28.0__tar.gz

{osbot_utils-1.26.0 → osbot_utils-1.28.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: osbot_utils
-Version: 1.26.0
+Version: 1.28.0
 Summary: OWASP Security Bot - Utils
 Home-page: https://github.com/owasp-sbot/OSBot-Utils
 License: MIT
@@ -22,7 +22,7 @@ Description-Content-Type: text/markdown
 
 Powerful Python util methods and classes that simplify common apis and tasks.
 
-![Current Release](https://img.shields.io/badge/release-v1.26.0-blue)
+![Current Release](https://img.shields.io/badge/release-v1.28.0-blue)
 [![codecov](https://codecov.io/gh/owasp-sbot/OSBot-Utils/graph/badge.svg?token=GNVW0COX1N)](https://codecov.io/gh/owasp-sbot/OSBot-Utils)
 
 

{osbot_utils-1.26.0 → osbot_utils-1.28.0}/README.md

@@ -2,7 +2,7 @@
 
 Powerful Python util methods and classes that simplify common apis and tasks.
 
-![Current Release](https://img.shields.io/badge/release-v1.26.0-blue)
+![Current Release](https://img.shields.io/badge/release-v1.28.0-blue)
 [![codecov](https://codecov.io/gh/owasp-sbot/OSBot-Utils/graph/badge.svg?token=GNVW0COX1N)](https://codecov.io/gh/owasp-sbot/OSBot-Utils)
 
 

osbot_utils-1.28.0/osbot_utils/helpers/Zip_Bytes.py

@@ -0,0 +1,79 @@
+from osbot_utils.base_classes.Type_Safe import Type_Safe
+from osbot_utils.utils.Dev import pprint
+from osbot_utils.utils.Files import files_list, file_create_from_bytes, temp_file, parent_folder, parent_folder_create
+from osbot_utils.utils.Zip import zip_bytes_empty, zip_bytes__files, zip_bytes__add_file, zip_bytes__add_files, \
+    zip_bytes__replace_files, zip_bytes__replace_file, zip_bytes__file_list, zip_bytes__file, \
+    zip_bytes__add_file__from_disk, zip_bytes__add_files__from_disk, zip_files, zip_file__files
+
+
+class Zip_Bytes(Type_Safe):
+    zip_bytes : bytes
+
+    def __enter__(self):
+        self.zip_bytes = zip_bytes_empty()
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        pass
+
+    def add_file(self, file_path, file_contents):
+        self.zip_bytes = zip_bytes__add_file(self.zip_bytes, file_path, file_contents)
+        return self
+
+    def add_file__from_disk(self, base_path, file_to_add):
+        self.zip_bytes = zip_bytes__add_file__from_disk(self.zip_bytes, base_path, file_to_add)
+        return self
+
+    def add_files(self, files_to_add):
+        self.zip_bytes = zip_bytes__add_files(self.zip_bytes, files_to_add)
+        return self
+
+    def add_files__from_disk(self, base_path, files_to_add):
+        self.zip_bytes = zip_bytes__add_files__from_disk(self.zip_bytes, base_path, files_to_add)
+        return self
+
+    def add_folder__from_disk(self, base_path, folder_to_add, pattern="*"):
+        files_to_add = files_list(folder_to_add, pattern=pattern)
+        return self.add_files__from_disk(base_path, files_to_add)
+
+    def add_from_zip_file(self, path_zip_file):
+        files_to_add = zip_file__files(path_zip_file)
+        self.add_files(files_to_add)
+        return self
+
+    def empty(self):
+        return self.size() == 0
+
+    def file(self, file_path):
+        return zip_bytes__file(self.zip_bytes, file_path)
+
+    def files(self):
+        return zip_bytes__files(self.zip_bytes)
+
+    def files_list(self):
+        return zip_bytes__file_list(self.zip_bytes)
+
+    def print_files_list(self):
+        pprint(self.files_list())
+        return self
+    def replace_files(self, files_to_replace):
+        self.zip_bytes = zip_bytes__replace_files(self.zip_bytes, files_to_replace)
+        return self
+
+    def replace_file(self, file_path, file_contents):
+        self.zip_bytes = zip_bytes__replace_file(self.zip_bytes, file_path, file_contents)
+        return self
+
+    def save(self, path=None):
+        if path is None:
+            path = temp_file(extension='.zip')
+        zip_file = file_create_from_bytes(bytes=self.zip_bytes, path=path)
+        return zip_file
+
+    def save_to(self, path):
+        parent_folder_create(path)           # make sure the parent folder exists
+        return self.save(path)
+
+
+    def size(self):
+        return len(self.files_list())

{osbot_utils-1.26.0 → osbot_utils-1.28.0}/osbot_utils/testing/Temp_Folder.py

@@ -4,7 +4,7 @@ from osbot_utils.utils.Misc import random_string
 
 from osbot_utils.utils.Files import path_combine, temp_folder_current, safe_file_name, folder_exists, folder_create, \
     folder_delete_recursively, files_list, file_create, folder_files, folders_recursive, files_find, files_recursive, \
-    temp_file_in_folder, create_folder, filter_parent_folder
+    temp_file_in_folder, create_folder, filter_parent_folder, file_contents
 from osbot_utils.utils.Zip import zip_folder
 
 
@@ -92,6 +92,10 @@ class Temp_Folder:
     def path(self):
         return self.full_path
 
+    def file_contents(self, target_file):
+        file_path = path_combine(self.full_path, target_file)
+        return file_contents(file_path)
+
     def files(self, show_parent_folder=False, include_folders=False):
         all_files = files_recursive(self.path(), include_folders=include_folders)
         if show_parent_folder:

{osbot_utils-1.26.0 → osbot_utils-1.28.0}/osbot_utils/testing/Temp_Zip.py

@@ -1,6 +1,7 @@
 from osbot_utils.testing.Temp_Folder import Temp_Folder
-from osbot_utils.utils.Files import Files, is_folder, file_exists, file_name
-from osbot_utils.utils.Zip import zip_folder, zip_file_list
+from osbot_utils.utils.Files import Files, is_folder, file_exists, file_name, file_move_to_folder, file_move, \
+    file_move_to
+from osbot_utils.utils.Zip import zip_folder, zip_file__list
 
 
 class Temp_Zip():
@@ -34,7 +35,7 @@ class Temp_Zip():
         return self.zip_file
 
     def files(self):
-        return zip_file_list(self.zip_file)
+        return zip_file__list(self.zip_file)
 
     def print_path(self):
         print()
@@ -42,4 +43,7 @@ class Temp_Zip():
         return self
 
     def zip_file_exists(self):
-        return file_exists(self.zip_file)
+        return file_exists(self.zip_file)
+
+    def move_to(self, target_file):
+        file_move_to(self.zip_file, target_file)

{osbot_utils-1.26.0 → osbot_utils-1.28.0}/osbot_utils/testing/Temp_Zip_In_Memory.py

@@ -3,7 +3,7 @@ import io
 from osbot_utils.testing.Temp_File import Temp_File
 from osbot_utils.testing.Temp_Folder import Temp_Folder
 from osbot_utils.utils.Files import is_file, is_folder, files_recursive, filter_parent_folder, temp_file
-from osbot_utils.utils.Zip import zip_files_to_bytes, zip_bytes_file_list, zip_bytes_add_file, zip_bytes_get_file
+from osbot_utils.utils.Zip import zip_files_to_bytes, zip_bytes__file_list, zip_bytes__add_file, zip_bytes__get_file
 
 
 class Temp_Zip_In_Memory:
@@ -76,14 +76,14 @@ class Temp_Zip_In_Memory:
         for items in self.targets_as_content:
             file_path     = items.get('file_path')
             file_contents = items.get('file_contents')
-            zip_bytes = zip_bytes_add_file(zip_bytes, file_path, file_contents)
+            zip_bytes = zip_bytes__add_file(zip_bytes, file_path, file_contents)
         return zip_bytes
 
     def zip_bytes_file_content(self, file_path):
-        return zip_bytes_get_file(self.zip_bytes(), file_path)
+        return zip_bytes__get_file(self.zip_bytes(), file_path)
 
     def zip_bytes_files(self):
-        return zip_bytes_file_list(self.zip_bytes())
+        return zip_bytes__file_list(self.zip_bytes())
 
     def zip_buffer(self):
         targets = self.target_files_with_root_folder()

{osbot_utils-1.26.0 → osbot_utils-1.28.0}/osbot_utils/utils/Files.py

@@ -8,6 +8,8 @@ import shutil
 import tempfile
 from   os.path              import abspath, join
 from pathlib import Path, PosixPath
+from typing import Union
+
 from osbot_utils.utils.Misc import bytes_to_base64, base64_to_bytes, random_string
 
 
@@ -79,10 +81,12 @@ class Files:
         return glob.glob(path_pattern, recursive=recursive)
 
     @staticmethod
-    def files(path, pattern= '*.*'):                        # todo: check behaviour and improve ability to detect file (vs folders)
+    def files(path, pattern= '*.*', only_files=True):
         result = []
         for file in Path(path).rglob(pattern):
-            result.append(str(file))                        # todo: see if there is a better way to do this conversion to string
+            if only_files and is_not_file(file):
+                continue
+            result.append(str(file))                                  # todo: see if there is a better way to do this conversion to string
         return sorted(result)
 
     @staticmethod
@@ -194,7 +198,7 @@ class Files:
         return path
 
     @staticmethod
-    def folder_create_in_parent(path, name):
+    def folder_create_in_parent(path, name):                # todo: revise the naming of this method, since it really doesn't have to do with 'parent' (it will depend on value of name)
         folder_path = path_combine(path, name)
         return folder_create(folder_path)
 
@@ -306,13 +310,24 @@ class Files:
         return Files.open(path, mode='rb')
 
     @staticmethod
-    def path_combine(path1, path2):
-        if type(path1) in [str, Path] and type(path2) in [str, Path]:
-            parent_path = str(path1)
-            sub_path    = str(path2)
-            if sub_path.startswith('/'):
-                sub_path = sub_path[1:]
-            return abspath(join(parent_path,sub_path))
+    # def path_combine(path1, path2):
+    #     if type(path1) in [str, Path] and type(path2) in [str, Path]:
+    #         parent_path = str(path1)
+    #         sub_path    = str(path2)
+    #         if sub_path.startswith('/'):
+    #             sub_path = sub_path[1:]
+    #         return abspath(join(parent_path,sub_path))
+
+    def path_combine(path1: Union[str, os.PathLike], path2: Union[str, os.PathLike]) -> str:
+        if path1 is None or path2 is None:
+            raise ValueError("Both paths must be provided")
+
+        parent_path = str(path1)
+        sub_path = str(path2)
+
+        sub_path = sub_path.lstrip('/')     # Remove leading slashes from sub_path
+
+        return abspath(join(parent_path, sub_path))
 
     @staticmethod
     def parent_folder(path, use_full_path=False):
@@ -451,8 +466,11 @@ def all_parent_folders(path=None, include_path=False):
             if path and path not in parent_directories:  # to handle the root path case
                 parent_directories.append(path)
             break
-
     return parent_directories
+
+def is_not_file(target):
+    return is_file(target) is False
+
 def file_move(source_file, target_file):
     if file_exists(source_file):
         file_copy(source_file, target_file)
@@ -461,10 +479,27 @@ def file_move(source_file, target_file):
                 return True
     return False
 
+def file_move_to_folder(source_file, target_folder):
+    if file_exists(source_file):
+        if folder_exists(target_folder):
+            target_file = path_combine(target_folder, file_name(source_file))
+            if file_move(source_file, target_file):
+                return target_file
+
+
 def folders_names_in_folder(target):
     folders = folders_in_folder(target)
     return folders_names(folders)
 
+def parent_folder_create(target):
+    return folder_create(parent_folder(target))
+
+def parent_folder_exists(target):
+    return folder_exists(parent_folder(target))
+
+def parent_folder_not_exists(target):
+    return parent_folder_exists(target) is False
+
 def stream_to_bytes(stream):
     return stream.read()
 
@@ -506,6 +541,7 @@ file_full_path                 = absolute_path
 file_lines                     = Files.lines
 file_lines_gz                  = Files.lines_gz
 file_md5                       = Files.contents_md5
+file_move_to                   = file_move
 file_name                      = Files.file_name
 file_name_without_extension    = Files.file_name_without_extension
 file_not_exists                = Files.not_exists

osbot_utils-1.28.0/osbot_utils/utils/Zip.py

@@ -0,0 +1,235 @@
+import gzip
+import io
+import os
+import shutil
+import tarfile
+import zipfile
+from os.path import abspath
+
+from osbot_utils.utils.Files import temp_folder, folder_files, temp_file, is_file, file_copy, file_move, file_exists, \
+    file_contents_as_bytes
+
+#########################
+# actions on gz_tar_bytes
+
+def gz_tar_bytes__file_list(gz_bytes):
+    gz_buffer_from_bytes = io.BytesIO(gz_bytes)
+    with gzip.GzipFile(fileobj=gz_buffer_from_bytes, mode='rb') as gz:
+        decompressed_data = gz.read()
+        tar_buffer_from_bytes = io.BytesIO(decompressed_data)                       # Assuming the decompressed data is a tag file, process it
+        with tarfile.open(fileobj=tar_buffer_from_bytes, mode='r:') as tar:
+            return sorted(tar.getnames())
+
+def gz_tar_bytes__get_file(gz_bytes, tar_file_path):
+    gz_buffer_from_bytes = io.BytesIO(gz_bytes)
+    with gzip.GzipFile(fileobj=gz_buffer_from_bytes, mode='rb') as gz:
+        decompressed_data = gz.read()
+        tar_buffer_from_bytes = io.BytesIO(decompressed_data)
+        with tarfile.open(fileobj=tar_buffer_from_bytes, mode='r:') as tar:
+            extracted_file = tar.extractfile(tar_file_path)
+            if extracted_file:
+                return extracted_file.read()
+            else:
+                raise FileNotFoundError(f"The file {tar_file_path} was not found in the tar archive.")
+
+#########################
+# actions on gz_zip_bytes
+
+def gz_zip_bytes__file_list(gz_bytes):
+    gz_buffer_from_bytes = io.BytesIO(gz_bytes)
+    with gzip.GzipFile(fileobj=gz_buffer_from_bytes, mode='rb') as gz:
+        decompressed_data = gz.read()
+        zip_buffer_from_bytes = io.BytesIO(decompressed_data)                   # Assuming the decompressed data is a zip file, process it
+        with zipfile.ZipFile(zip_buffer_from_bytes, 'r') as zf:
+            return sorted(zf.namelist())
+
+#########################
+# actions on zipped bytes
+
+def zip_bytes__add_file(zip_bytes, zip_file_path, file_contents):                                       # todo: see if this is actually a valid use case (or if we should be using replace in all scenarios)
+    return zip_bytes__add_files(zip_bytes, {zip_file_path: file_contents})
+
+def zip_bytes__add_file__from_disk(zip_bytes, base_path, file_to_add):
+    return zip_bytes__add_files__from_disk(zip_bytes, base_path, files_to_add=[file_to_add])
+
+def zip_bytes__add_files__from_disk(zip_bytes, base_path, files_to_add, replace_files=True):
+    zip_files_to_add = {}
+    if base_path[:-1] != '/':
+        base_path += "/"
+    for file_to_add in files_to_add:
+        if file_exists(file_to_add):
+            file_contents = file_contents_as_bytes(file_to_add)
+            zip_file_path = file_to_add.replace(base_path, '')
+            zip_files_to_add[zip_file_path] = file_contents
+
+    if replace_files:
+        return zip_bytes__replace_files(zip_bytes, zip_files_to_add)
+    else:
+        return zip_bytes__add_files(zip_bytes, zip_files_to_add)                        # todo: see if this is actually a valid use case (or if we should be using replace in all scenarios)
+
+def zip_bytes__add_files(zip_bytes, files_to_add):                                      # todo: see if this is actually a valid use case (or if we should be using replace in all scenarios)
+    zip_buffer = io.BytesIO(zip_bytes)                                                  # Create a BytesIO buffer from the input zip bytes
+
+    with zipfile.ZipFile(zip_buffer, 'a', zipfile.ZIP_DEFLATED) as zf:
+        for file_path, file_contents in files_to_add.items():
+            if isinstance(file_contents, str):
+                file_contents = file_contents.encode('utf-8')
+            elif not isinstance(file_contents, bytes):
+                continue
+            zf.writestr(file_path, file_contents)
+
+    return zip_buffer.getvalue()
+
+def zip_bytes__file(zip_bytes, zip_file_path):
+    zip_buffer = io.BytesIO(zip_bytes)
+    with zipfile.ZipFile(zip_buffer, 'r') as zf:
+        if zip_file_path in zf.namelist():
+            return zf.read(zip_file_path)
+
+def zip_bytes__files(zip_bytes):
+    zip_buffer = io.BytesIO(zip_bytes)                      # Create a BytesIO buffer from the input zip bytes
+    files_dict = {}                                         # Create a dictionary to store file contents
+
+    with zipfile.ZipFile(zip_buffer, 'r') as zf:      # Open the zip file in read mode
+        for file_name in zf.namelist():                     # Iterate over each file in the zip archive
+            files_dict[file_name] = zf.read(file_name)      # Read the content of the file
+
+    return files_dict                                       # Return the dictionary with file contents
+
+def zip_bytes__file_list(zip_bytes):
+    zip_buffer_from_bytes = io.BytesIO(zip_bytes)
+    with zipfile.ZipFile(zip_buffer_from_bytes, 'r') as zf:
+        return sorted(zf.namelist())
+
+def zip_bytes__remove_file(zip_bytes, file_to_remove):
+    return zip_bytes__remove_files(zip_bytes, [file_to_remove])
+
+def zip_bytes__remove_files(zip_bytes, files_to_remove):
+    files_to_remove = set(files_to_remove)                                      # Convert files_to_remove to a set for faster lookup
+    zip_buffer      = io.BytesIO(zip_bytes)                                     # Create a BytesIO buffer from the input zip bytes
+    new_zip_buffer  = io.BytesIO()                                              # Create a new BytesIO buffer for the updated zip
+
+    with zipfile.ZipFile(zip_buffer, 'r') as original_zip:
+        with zipfile.ZipFile(new_zip_buffer, 'w') as new_zip:
+            for item in original_zip.infolist():                                # Iterate over each item in the original zip file
+                if item.filename not in files_to_remove:                        # Read the original content and write it to the new zip file
+                    new_zip.writestr(item, original_zip.read(item.filename))
+    return new_zip_buffer.getvalue()                                            # Get the updated zip content as bytes
+
+def zip_bytes__replace_file(zip_bytes, zip_file_path, file_contents):
+    files_to_replace = {zip_file_path: file_contents }
+    return zip_bytes__replace_files(zip_bytes, files_to_replace)
+
+def zip_bytes__replace_files(zip_bytes, files_to_replace):
+    zip_bytes__without_files       = zip_bytes__remove_files(zip_bytes               , set(files_to_replace))
+    zip_bytes__with_replaced_files = zip_bytes__add_files   (zip_bytes__without_files, files_to_replace     )
+    return zip_bytes__with_replaced_files
+
+
+def zip_bytes__unzip(zip_bytes, target_folder=None):
+    target_folder = target_folder or temp_folder()              # Use the provided target folder or create a temporary one
+    zip_buffer = io.BytesIO(zip_bytes)                          # Create a BytesIO buffer from the zip bytes
+    with zipfile.ZipFile(zip_buffer, 'r') as zf:          # Open the zip file from the buffer
+        zf.extractall(target_folder)                            # Extract all files to the target folder
+    return target_folder                                        # Return the path of the target folder
+
+
+########################
+# actions on zipped file
+
+def zip_file__list(path_zip_file):
+    if is_file(path_zip_file):
+        with zipfile.ZipFile(path_zip_file) as zip_file:
+            return sorted(zip_file.namelist())
+    return []
+
+def zip_file__files(path_zip_file):
+    if is_file(path_zip_file):
+        zip_bytes = file_contents_as_bytes(path_zip_file)
+        return zip_bytes__files(zip_bytes)
+    return []
+
+def zip_file__unzip(path_zip_file, target_folder=None, format='zip'):
+    target_folder = target_folder or temp_folder()
+    shutil.unpack_archive(path_zip_file, extract_dir=target_folder, format=format)
+    return target_folder
+
+# zip creation actions
+def zip_bytes_empty():
+
+    zip_buffer = io.BytesIO()                                               # Create a BytesIO buffer to hold the zip file
+    with zipfile.ZipFile(zip_buffer, mode='w') as _:                        # Use the zipfile.ZipFile class to create an empty zip file
+        pass                                                                # No files to add, so we just create the zip structure
+    return zip_buffer.getvalue()                                            # Get the zip file content as bytes
+
+def zip_bytes_to_file(zip_bytes, target_file=None):
+    if target_file is None:
+        target_file = temp_file(extension='.zip')
+    with open(target_file, 'wb') as f:
+        f.write(zip_bytes)
+    return target_file
+
+def zip_files_to_bytes(target_files, root_folder=None):
+    zip_buffer = io.BytesIO()                                                   # Create a BytesIO buffer to hold the zipped file
+    with zipfile.ZipFile(zip_buffer, 'w', zipfile.ZIP_DEFLATED) as zf:          # Create a ZipFile object with the buffer as the target
+        for entry in target_files:
+            if type(entry) is str:                                              # if entry is a string, assume it's a file path
+                file_path = entry
+                file_root_folder = root_folder
+            else:
+                file_path = entry.get('file')
+                file_root_folder = entry.get('root_folder') or root_folder
+            if file_root_folder:
+                arcname = file_path.replace(file_root_folder,'')                      # Define the arcname, which is the name inside the zip file
+            else:
+                arcname = file_path                                                 # if root_path is not provided, use the full file path
+            zf.write(file_path, arcname)                                            # Add the file to the zip file
+    zip_buffer.seek(0)
+    return zip_buffer
+
+def zip_folder(root_dir, format='zip'):
+    return shutil.make_archive(base_name=root_dir, format=format, root_dir=root_dir)
+
+def zip_folder_to_file (root_dir, target_file):
+    zip_file = zip_folder(root_dir)
+    return file_move(zip_file, target_file)
+
+def zip_folder_to_bytes(root_dir):      # todo add unit test
+    zip_buffer = io.BytesIO()                                                   # Create a BytesIO buffer to hold the zipped file
+    with zipfile.ZipFile(zip_buffer, 'w', zipfile.ZIP_DEFLATED) as zf:          # Create a ZipFile object with the buffer as the target
+        for foldername, subfolders, filenames in os.walk(root_dir):             # Walk the root_dir and add all files and folders to the zip file
+            for filename in filenames:
+                absolute_path = os.path.join(foldername, filename)              # Create the complete filepath
+                arcname = os.path.relpath(absolute_path, root_dir)              # Define the arcname, which is the name inside the zip file
+                zf.write(absolute_path, arcname)                                # Add the file to the zip file
+    zip_buffer.seek(0)                                                          # Reset buffer position
+    return zip_buffer
+
+def zip_files(base_folder, file_pattern="*.*", target_file=None):
+    base_folder = abspath(base_folder)
+    file_list   = folder_files(base_folder, file_pattern)
+
+    if len(file_list):                                                  # if there were files found
+        target_file = target_file or temp_file(extension='zip')
+        with zipfile.ZipFile(target_file,'w') as zip:
+            for file_name in file_list:
+                zip_file_path = file_name.replace(base_folder,'')
+                zip.write(file_name, zip_file_path)
+
+        return target_file
+
+###########################
+# extra function's mappings
+
+file_unzip                   = zip_file__unzip
+folder_zip                   = zip_folder
+
+unzip_file                   = zip_file__unzip
+
+zip_bytes__extract_to_folder = zip_bytes__unzip
+zip_bytes__file_contents     = zip_bytes__file
+zip_bytes__get_file          = zip_bytes__file
+zip_bytes__unzip_to_folder   = zip_bytes__unzip
+
+zip_list_files               = zip_file__list
+zip_file__file_list          = zip_file__list

osbot_utils-1.28.0/osbot_utils/version

@@ -0,0 +1 @@
+v1.28.0

{osbot_utils-1.26.0 → osbot_utils-1.28.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name        = "osbot_utils"
-version     = "v1.26.0"
+version     = "v1.28.0"
 description = "OWASP Security Bot - Utils"
 authors     = ["Dinis Cruz <dinis.cruz@owasp.org>"]
 license     = "MIT"