media-toolkit 0.1.1.dev1__tar.gz → 0.1.2__tar.gz

{media_toolkit-0.1.1.dev1 → media_toolkit-0.1.2}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: media-toolkit
-Version: 0.1.1.dev1
+Version: 0.1.2
 Summary: Web-ready standardized file processing and serialization. Read, load and convert to standard file types with a common interface.
 Author: SocAIty
 License: GPLv3

media_toolkit-0.1.2/media_toolkit/core/file_content_buffer.py

@@ -0,0 +1,114 @@
+import io
+import os
+import tempfile
+
+
+class FileContentBuffer:
+    """Handles file content storage either in memory or temporary file."""
+
+    def __init__(
+            self,     
+            use_temp_file: bool = False,
+            temp_dir: str = None
+    ):
+        """Initialize buffer with given configuration.
+        :param use_temp_file: If True, store content in temporary file instead of keeping it in memory.
+        :param temp_dir: Directory to store temporary files in. If None, system default is used.
+        """
+        self._use_temp_file = use_temp_file
+        self._temp_dir = temp_dir
+  
+        self._temp_file = None
+        self._memory_buffer = None
+        self._initialize_buffer()
+
+    @property
+    def name(self) -> str:
+        """Get path to temporary file if used.
+        Returns:
+            str: Path to temporary file. None if not used.
+        """
+        if self._temp_file:
+            return self._temp_file.name
+        return None
+
+    def _initialize_buffer(self):
+        """Initialize the appropriate buffer based on configuration."""
+        if self._use_temp_file:
+            self._temp_file = tempfile.NamedTemporaryFile(
+                delete=False,
+                dir=self._temp_dir
+            )
+            self._memory_buffer = None
+        else:
+            self._memory_buffer = io.BytesIO()
+            self._temp_file = None
+
+    def write(self, data: bytes):
+        """Write data to the buffer.
+
+        Args:
+            data (bytes): Data to write.
+
+        Raises:
+            ValueError: If data exceeds configured size limit.
+        """
+        if self._use_temp_file:
+            self._temp_file.write(data)
+        else:
+            self._memory_buffer.write(data)
+
+    def read(self) -> bytes:
+        """Read all content from the buffer.
+
+        Returns:
+            bytes: Buffer content.
+        """
+        if self._use_temp_file:
+            self._temp_file.seek(0)
+            return self._temp_file.read()
+        else:
+            self._memory_buffer.seek(0)
+            return self._memory_buffer.read()
+
+    def seek(self, offset: int):
+        """Seek to given position in buffer.
+
+        Args:
+            offset (int): Position to seek to.
+        """
+        if self._use_temp_file:
+            self._temp_file.seek(offset)
+        else:
+            self._memory_buffer.seek(offset)
+
+    def truncate(self, size: int):
+        """Truncate buffer to given size.
+
+        Args:
+            size (int): Size to truncate to.
+        """
+        if self._use_temp_file:
+            self._temp_file.truncate(size)
+        else:
+            self._memory_buffer.truncate(size)
+
+    def getbuffer(self) -> memoryview:
+        """Get a memoryview of the buffer content.
+
+        Returns:
+            memoryview: View of buffer content.
+        """
+        if self._use_temp_file:
+            return memoryview(self.read())
+        else:
+            return self._memory_buffer.getbuffer()
+
+    def __del__(self):
+        """Cleanup temporary files on deletion."""
+        if self._temp_file:
+            try:
+                self._temp_file.close()
+                os.remove(self._temp_file.name)
+            except:
+                pass

{media_toolkit-0.1.1.dev1 → media_toolkit-0.1.2}/media_toolkit/core/image_file.py

@@ -1,3 +1,5 @@
+import os.path
+
 from media_toolkit.utils.dependency_requirements import requires_numpy, requires_cv2, requires
 from media_toolkit.core.media_file import MediaFile
 
@@ -44,15 +46,34 @@ class ImageFile(MediaFile):
 
     @requires_cv2()
     def save(self, path: str):
+        # set to working directory if path is None
+        if path is None:
+            path = os.path.curdir
+        # create folder if not exists
+        elif os.path.dirname(path) != "" and not os.path.exists(os.path.dirname(path)):
+            os.makedirs(os.path.dirname(path))
+
+        # check if path contains a file name add default if not given
+        if os.path.isdir(path):
+            if self.file_name is None:
+                self.file_name = "media_toolkit_output"
+                print(f"No file name given. Using {self.file_name}")
+            path = os.path.join(path, self.file_name)
         cv2.imwrite(path, self.to_np_array())
 
     def _file_info(self):
         super()._file_info()
         np_array = self.to_np_array()
+        if self.file_size() > 0:
+            try:
+                img_type, self._channels = self.detect_image_type_and_channels(np_array)
+            except Exception as e:
+                print(f"Could not detect image type and channels. Error: {e}")
+                img_type = None
+                self._channels = None
 
-        img_type, self._channels = self.detect_image_type_and_channels(np_array)
-        if img_type is not None:
-            self.content_type = f"image/{img_type}"
+            if img_type is not None:
+                self.content_type = f"image/{img_type}"
         
 
     @staticmethod
@@ -61,14 +82,16 @@ class ImageFile(MediaFile):
         if isinstance(image, list):
             image = np.array(image)
 
-        # Check the number of _channels
+        if not hasattr(image, 'shape'):
+            raise ValueError("Unsupported image type")
+
         if len(image.shape) == 2:
             channels = 1  # Grayscale
         elif len(image.shape) == 3:
             channels = image.shape[2]
         else:
-            #raise ValueError("Unsupported image shape: {}".format(image.shape))
-            return None, None
+            raise ValueError("Unsupported image shape: {}".format(image.shape))
+            # return None, None
 
         # Detect image type by checking for specific markers
         image_type = None

{media_toolkit-0.1.1.dev1 → media_toolkit-0.1.2}/media_toolkit/core/media_file.py

@@ -2,10 +2,11 @@ import base64
 import io
 import mimetypes
 
-from typing import Union, BinaryIO
+from typing import Union, BinaryIO, Tuple, Optional
 import os
 from urllib.parse import urlparse
 
+from media_toolkit.core.file_content_buffer import FileContentBuffer
 from media_toolkit.utils.dependency_requirements import requires_numpy
 
 import re
@@ -22,15 +23,25 @@ class MediaFile:
     Works natively with bytesio, base64 and binary data.
     """
 
-    def __init__(self, file_name: str = "file", content_type: str = "application/octet-stream"):
+    def __init__(
+            self,
+            file_name: str = "file",
+            content_type: str = "application/octet-stream",
+            use_temp_file: bool = False,
+            temp_dir: str = None
+    ):
         """
         :param file_name: The name of the file. Note it is overwritten if you use from_file/from_starlette.
         :param content_type: The content type of the file. Note it is overwritten if you use from_file/from_starlette.
+        :param use_temp_file: If True, the file is saved to a temporary file. This is useful for large files.
+        :param max_file_size: The maximum file size in bytes. If the file is larger lib will throw an error.
+        :param temp_dir: The directory where the temporary file is saved. If None, the system temp dir is used.
         """
         self.content_type = content_type
         self.file_name = file_name  # the name of the file also when specified in bytesio
         self.path = None  # the path of the file if it was provided. Is also indicator if file was loaded from file.
-        self._content_buffer = io.BytesIO()
+
+        self._content_buffer = FileContentBuffer(use_temp_file=use_temp_file, temp_dir=temp_dir)
 
     def from_any(self, data):
         """
@@ -71,7 +82,7 @@ class MediaFile:
         Set the content of the file from a BytesIO or a file handle.
         :params buffer: The buffer to read from.
         :params copy: If true, the buffer is completely read to bytes and the bytes copied to this file.
-            If false file works with the provided buffer. Danger -- The buffer is kept open.
+            If false file works with the provided buffer. Danger -- The buffer is kept open (not thread safe).
         """
         if not type(buffer) in [io.BytesIO, io.BufferedReader]:
             raise ValueError(f"Buffer must be of type BytesIO or BufferedReader. Got {type(buffer)}")
@@ -141,10 +152,9 @@ class MediaFile:
         """
         Load a file which was encoded as a base64 string.
         """
-
-        decoded = self._decode_base_64_if_is(base64_str)
+        decoded, media_type = self._decode_base_64_if_is(base64_str)
         if decoded is not None:
-            return self.from_bytes(base64.b64decode(base64_str))
+            return self.from_bytes(decoded)
         else:
             err_str = base64_str if len(base64_str) <= 50 else base64_str[:50] + "..."
             raise ValueError(f"Decoding from base64 like string {err_str} was not possible. Check your data.")
@@ -169,14 +179,15 @@ class MediaFile:
         self.from_base64(file_result_json["content"])
         return self
 
-    def from_url(self, url: str):
+    def from_url(self, url: str, headers: dict = None):
         """
         Download a file from an url.
         """
         # code inspired by: https://github.com/runpod/runpod-python/blob/main/runpod/serverless/utils/rp_download.py
         import requests
-        HEADERS = {"User-Agent": "runpod-python/0.0.0 (https://runpod.io; support@runpod.io)"}
-        with requests.get(url, headers=HEADERS, stream=True, timeout=5) as response:
+
+        headers = headers or {"User-Agent": "runpod-python/0.0.0 (https://runpod.io; support@runpod.io)"}
+        with requests.get(url, headers=headers, stream=True, timeout=5) as response:
             response.raise_for_status()
 
             # get orig file name or create new
@@ -210,9 +221,6 @@ class MediaFile:
                     file.write(chunk)
             file.name = original_file_name
             self.file_name = original_file_name
-
-            # self.url = url
-
             return self.from_bytesio_or_handle(file, copy=False)
 
     @requires_numpy()
@@ -280,10 +288,6 @@ class MediaFile:
                 print(f"No file name given. Using {self.file_name}")
             path = os.path.join(path, self.file_name)
 
-        # check if has extension
-        # if os.path.splitext(path)[1] == "":
-        #    path += ".mp4"
-
         with open(path, 'wb') as file:
             file.write(self.read())
 
@@ -313,7 +317,7 @@ class MediaFile:
 
     def file_size(self, unit="bytes") -> int:
         """
-        :param unit:
+        :param unit: bytes, kb, mb or gb
         """
         size_in_ = self._content_buffer.getbuffer().nbytes
         if unit == "bytes":
@@ -326,6 +330,26 @@ class MediaFile:
             size_in_ = size_in_ / 1000000000
         return size_in_
 
+    @property
+    def extension(self) -> Union[str, None]:
+        """
+        Will try to guess the file type based on the detected mimetype.
+        If no mimetype is detected it will try to guess the file extension based on the file name.
+        :return: the guessed file extension without '.'.
+        """
+        if self.file_name is None and self.content_type == "application/octet-stream":
+            return None
+
+        if self.content_type and self.content_type != "application/octet-stream":
+            guessed_ext = mimetypes.guess_extension(self.content_type)
+            if guessed_ext:
+                return guessed_ext.replace(".", "").lower()
+
+        if self.file_name is not None:
+            return None
+
+        return self.file_name.rsplit(".", 1)[-1]
+
     def __bytes__(self):
         return self.to_bytes()
 
@@ -344,13 +368,42 @@ class MediaFile:
         }
 
     @staticmethod
-    def _decode_base_64_if_is(data: Union[bytes, str]):
+    def _parse_base64_uri(data: str) -> Tuple[str, Optional[str]]:
         """
-        Checks if a string is base64. If it is, it returns the base64 string as bytes; else returns None.
+        Parse base64 string, handling data URI format and extracting content.
+        Args:
+            data (str): Base64 encoded string, potentially with data URI prefix
+        Returns:
+            Tuple of (base64 content, optional media_type)
         """
+        # Regex to match data URI format: data:[<media type>][;base64],<data>
+        data_uri_pattern = r'^data:(?P<mediatype>[\w/\-\.]+)?(?:;base64)?,(?P<base64>.*)'
+
+        # Check if the string matches data URI format
+        match = re.match(data_uri_pattern, data)
+        if match:
+            # Extract media type and base64 content
+            media_type = match.group('mediatype')
+            base64_content = match.group('base64')
+            return base64_content, media_type
+
+        # If no data URI prefix, return the original string
+        return data, None
+
+    @staticmethod
+    def _decode_base_64_if_is(data: Union[bytes, str]) -> [Union[str, None], Union[str, None]]:
+        """
+        Checks if a string is base64 (or base64uri).
+        :param data: The data to decode.
+        :return: If is base64 (decoded base64 data as bytes, optional media_type) else None, None
+        """
+        media_type = None
         if isinstance(data, str):
+            # check if is uri format and parse it
+            data, media_type = MediaFile._parse_base64_uri(data)
             data = data.encode()
 
+        # Decode and Re-encode the data to check if it is valid base64
         try:
             # Decode the data
             decoded = base64.b64decode(data, validate=True)
@@ -358,11 +411,11 @@ class MediaFile:
             back_encoded = base64.b64encode(decoded)
             # Compare with the original encoded data
             if back_encoded == data:
-                return decoded
+                return decoded, media_type
         except Exception:
             pass
 
-        return None
+        return None, None
 
     @staticmethod
     def _is_valid_file_path(path: str):
@@ -374,4 +427,7 @@ class MediaFile:
 
     @staticmethod
     def _is_url(url: str):
+        if not isinstance(url, str):
+            return False
+
         return urlparse(url).scheme in ['http', 'https']

{media_toolkit-0.1.1.dev1 → media_toolkit-0.1.2}/media_toolkit/core/video/video_file.py

@@ -43,6 +43,7 @@ class VideoFile(MediaFile):
         self.shape = None
         self.duration = None
         self.audio_sample_rate = None
+        self._temp_file_path = None  # if to_temp_file is called, the path is stored here. Needed clean deletion
 
     def from_files(self, image_files: Union[List[str], list], frame_rate: int = 30, audio_file=None):
         """
@@ -142,6 +143,11 @@ class VideoFile(MediaFile):
         if suffix == 'octet-stream':
             raise ValueError("The content type of the video file is not valid. Read a video file first.")
 
+        # If already using temp file storage, return path
+        if self._content_buffer._use_temp_file:
+            return self._content_buffer.name
+
+        # create new temp file
         with tempfile.NamedTemporaryFile(delete=False, suffix=f".{suffix}") as temp_video_file:
             temp_video_file.write(self.read())
             temp_video_file_path = temp_video_file.name
@@ -215,20 +221,20 @@ class VideoFile(MediaFile):
         # get video info
         info = mediainfo(path)
 
-        def info_to_number(key: str, default_val=None):
+        def info_to_number(key: str, default_val=None, cast=float):
             if key in info:
                 val = info[key]
                 if val == 'N/A':
                     return default_val
                 # split if / in val and take first
                 val = val.split("/")[0]
-                return float(val)
+                return cast(val)
             return default_val
 
-        self.frame_count = info_to_number('nb_frames')
+        self.frame_count = info_to_number('nb_frames', cast=int)
         self.duration = info_to_number('duration')
-        self.width = info_to_number('width')
-        self.height = info_to_number('height')
+        self.width = info_to_number('width', cast=int)
+        self.height = info_to_number('height', cast=int)
         self.shape = (self.width, self.height)
         self.audio_sample_rate = info_to_number('sample_rate', 44100)
 
@@ -366,4 +372,12 @@ class VideoFile(MediaFile):
         return self.to_video_stream()
 
     def __len__(self):
-        return self.frame_count
+        return int(self.frame_count)
+
+    def __del__(self):
+        if self._temp_file_path is not None:
+            try:
+                os.remove(self._temp_file_path)
+            except Exception as e:
+                print("Could not delete temporary file. Error: ", e)
+

{media_toolkit-0.1.1.dev1 → media_toolkit-0.1.2}/media_toolkit/utils/file_conversion.py

@@ -37,12 +37,15 @@ def media_from_file(file_path: str) -> Union[MediaFile, ImageFile, AudioFile, Vi
     return MediaFile().from_file(file_path)
 
 
-def media_from_any(file, media_file_type=None):
+def media_from_any(file, media_file_type=None, use_temp_file: bool = False, temp_dir: str = None) -> MediaFile:
     """
     Converts a file to a send able format.
     :param file: The file to convert.
     :param media_file_type: The target type to convert to. If not specified will be converted to MediaFile.
         Use ImageFile, AudioFile, VideoFile to convert to those types.
+    :param use_temp_file: If True, a temporary file will be used to store the data within the media-file.
+        If not stored in RAM.
+    :param temp_dir: The directory to store the temporary file in. If not specified, the default temp directory will be used.
     :return: The send able file.
     """
     # it is already converted
@@ -53,7 +56,7 @@ def media_from_any(file, media_file_type=None):
     target_class = MediaFile
     if media_file_type is not None and issubclass(media_file_type, MediaFile):
         target_class = media_file_type
-    media_file_instance = target_class()
+    media_file_instance = target_class(use_temp_file=use_temp_file, temp_dir=temp_dir)
 
     # load data
     media_file_instance = media_file_instance.from_any(file)

{media_toolkit-0.1.1.dev1 → media_toolkit-0.1.2}/media_toolkit.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: media-toolkit
-Version: 0.1.1.dev1
+Version: 0.1.2
 Summary: Web-ready standardized file processing and serialization. Read, load and convert to standard file types with a common interface.
 Author: SocAIty
 License: GPLv3

{media_toolkit-0.1.1.dev1 → media_toolkit-0.1.2}/media_toolkit.egg-info/SOURCES.txt

@@ -9,6 +9,7 @@ media_toolkit.egg-info/requires.txt
 media_toolkit.egg-info/top_level.txt
 media_toolkit/core/__init__.py
 media_toolkit/core/audio_file.py
+media_toolkit/core/file_content_buffer.py
 media_toolkit/core/image_file.py
 media_toolkit/core/media_file.py
 media_toolkit/core/video/__init__.py

{media_toolkit-0.1.1.dev1 → media_toolkit-0.1.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "media-toolkit"
-version = "0.1.1.dev1"
+version = "0.1.2"
 description = "Web-ready standardized file processing and serialization. Read, load and convert to standard file types with a common interface."
 requires-python = ">=3.8"
 authors = [