media-toolkit 0.0.8__tar.gz → 0.1.1.dev1__tar.gz

{media_toolkit-0.0.8 → media_toolkit-0.1.1.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: media-toolkit
-Version: 0.0.8
+Version: 0.1.1.dev1
 Summary: Web-ready standardized file processing and serialization. Read, load and convert to standard file types with a common interface.
 Author: SocAIty
 License: GPLv3
@@ -46,10 +46,11 @@ Load and convert from and to common data types:
 - bytes,
 - base64
 - json
+- urls
 - etc.
 
 Transmit files between services with a common interface
-- Native FastSDK and FastTaskAPI integration
+- Native [FastSDK](https://github.com/SocAIty/fastSDK) and [FastTaskAPI](https://github.com/SocAIty/FastTaskAPI) integration
 - Supports httpx, requests
 
 Work with native python libs like BytesIO.
@@ -63,9 +64,9 @@ You can install the package with PIP, or clone the repository.
 ```bash
 # install from pypi
 pip install media-toolkit
-# install without dependencies: this is useful if you only need the basic functionality
+# install without dependencies: this is useful if you only need the basic functionality (working with files)
 pip install media-toolkit --no-deps
-# if you only want to use certain file types
+# if you want to use certain file types, and convenience functions
 pip install media-toolkit[VideoFile]  # or [AudioFile, VideoFile, ...]
 # install from github for newest release
 pip install git+git://github.com/SocAIty/media-toolkit
@@ -84,7 +85,7 @@ The library automatically detects the data type and loads it correctly.
 ```python
 from media_toolkit import MediaFile, ImageFile, AudioFile, VideoFile
 
-# could be a path, base64, bytesio, file_handle, numpy array ...
+# could be a path, url, base64, bytesio, file_handle, numpy array ...
 arbitrary_data = "...."
 # Instantiate an image file
 new_file = ImageFile().from_any(arbitrary_data)
@@ -189,6 +190,7 @@ my_files = {
 response = httpx.Client().post(url, files=my_files)
 ```
 
+
 # How it works
 
 If media-file is instantiated with ```from_*``` it converts it to an intermediate representation.
@@ -199,5 +201,5 @@ Currently the intermediate representation is supported in memory with (BytesIO).
 
 # ToDo:
 
-[x] additionally support tempfile backend instead of working bytesio memory mode only.
-[x] decreasing redundancies for _file_info() method
+- [x] additionally support tempfile backend instead of working bytesio memory mode only.
+- [x] decreasing redundancies for _file_info() method

{media_toolkit-0.0.8 → media_toolkit-0.1.1.dev1}/README.md

@@ -17,10 +17,11 @@ Load and convert from and to common data types:
 - bytes,
 - base64
 - json
+- urls
 - etc.
 
 Transmit files between services with a common interface
-- Native FastSDK and FastTaskAPI integration
+- Native [FastSDK](https://github.com/SocAIty/fastSDK) and [FastTaskAPI](https://github.com/SocAIty/FastTaskAPI) integration
 - Supports httpx, requests
 
 Work with native python libs like BytesIO.
@@ -34,9 +35,9 @@ You can install the package with PIP, or clone the repository.
 ```bash
 # install from pypi
 pip install media-toolkit
-# install without dependencies: this is useful if you only need the basic functionality
+# install without dependencies: this is useful if you only need the basic functionality (working with files)
 pip install media-toolkit --no-deps
-# if you only want to use certain file types
+# if you want to use certain file types, and convenience functions
 pip install media-toolkit[VideoFile]  # or [AudioFile, VideoFile, ...]
 # install from github for newest release
 pip install git+git://github.com/SocAIty/media-toolkit
@@ -55,7 +56,7 @@ The library automatically detects the data type and loads it correctly.
 ```python
 from media_toolkit import MediaFile, ImageFile, AudioFile, VideoFile
 
-# could be a path, base64, bytesio, file_handle, numpy array ...
+# could be a path, url, base64, bytesio, file_handle, numpy array ...
 arbitrary_data = "...."
 # Instantiate an image file
 new_file = ImageFile().from_any(arbitrary_data)
@@ -160,6 +161,7 @@ my_files = {
 response = httpx.Client().post(url, files=my_files)
 ```
 
+
 # How it works
 
 If media-file is instantiated with ```from_*``` it converts it to an intermediate representation.
@@ -170,5 +172,5 @@ Currently the intermediate representation is supported in memory with (BytesIO).
 
 # ToDo:
 
-[x] additionally support tempfile backend instead of working bytesio memory mode only.
-[x] decreasing redundancies for _file_info() method
+- [x] additionally support tempfile backend instead of working bytesio memory mode only.
+- [x] decreasing redundancies for _file_info() method

media_toolkit-0.1.1.dev1/media_toolkit/__init__.py

@@ -0,0 +1,2 @@
+from media_toolkit.core import MediaFile, ImageFile, VideoFile, AudioFile
+from media_toolkit.utils.file_conversion import media_from_file, media_from_any

{media_toolkit-0.0.8 → media_toolkit-0.1.1.dev1}/media_toolkit/core/image_file.py

@@ -49,6 +49,7 @@ class ImageFile(MediaFile):
     def _file_info(self):
         super()._file_info()
         np_array = self.to_np_array()
+
         img_type, self._channels = self.detect_image_type_and_channels(np_array)
         if img_type is not None:
             self.content_type = f"image/{img_type}"

{media_toolkit-0.0.8 → media_toolkit-0.1.1.dev1}/media_toolkit/core/media_file.py

@@ -1,10 +1,14 @@
 import base64
 import io
 import mimetypes
+
 from typing import Union, BinaryIO
 import os
+from urllib.parse import urlparse
+
 from media_toolkit.utils.dependency_requirements import requires_numpy
 
+import re
 
 try:
     import numpy as np
@@ -17,6 +21,7 @@ class MediaFile:
     Has file conversions that make it easy to work standardized with files across the web and in the sdk.
     Works natively with bytesio, base64 and binary data.
     """
+
     def __init__(self, file_name: str = "file", content_type: str = "application/octet-stream"):
         """
         :param file_name: The name of the file. Note it is overwritten if you use from_file/from_starlette.
@@ -24,7 +29,7 @@ class MediaFile:
         """
         self.content_type = content_type
         self.file_name = file_name  # the name of the file also when specified in bytesio
-        self.path = None    # the path of the file if it was provided. Is also indicator if file was loaded from file.
+        self.path = None  # the path of the file if it was provided. Is also indicator if file was loaded from file.
         self._content_buffer = io.BytesIO()
 
     def from_any(self, data):
@@ -41,8 +46,13 @@ class MediaFile:
         elif isinstance(data, str):
             if self._is_valid_file_path(data):
                 self.from_file(data)
+            elif self._is_url(data):
+                self.from_url(data)
             else:
-                self.from_base64(data)
+                try:
+                    self.from_base64(data)
+                except Exception as e:
+                    print(f"Either wrong file path or not base64. Check your inputs: {data}. Error: {e}")
         elif isinstance(data, bytes):
             self.from_bytes(data)
         elif type(data).__name__ == 'ndarray':
@@ -53,9 +63,9 @@ class MediaFile:
         return self
 
     def from_bytesio_or_handle(
-        self,
-        buffer: Union[io.BytesIO, BinaryIO, io.BufferedReader],
-        copy: bool = True
+            self,
+            buffer: Union[io.BytesIO, BinaryIO, io.BufferedReader],
+            copy: bool = True
     ):
         """
         Set the content of the file from a BytesIO or a file handle.
@@ -85,11 +95,10 @@ class MediaFile:
     def from_bytesio(self, buffer: Union[io.BytesIO, BinaryIO], copy: bool = True):
         return self.from_bytesio_or_handle(buffer=buffer, copy=copy)
 
-    #@staticmethod
-    #@overload
-    #def from_file(path_or_handle: Union[str, io.BytesIO, io.BufferedReader]):
+    # @staticmethod
+    # @overload
+    # def from_file(path_or_handle: Union[str, io.BytesIO, io.BufferedReader]):
     #    return MediaFile().from_file(path_or_handle)
-
     def from_file(self, path_or_handle: Union[str, io.BytesIO, io.BufferedReader]):
         """
         Load a file from a file path, file handle or base64 and convert it to BytesIO.
@@ -160,6 +169,52 @@ class MediaFile:
         self.from_base64(file_result_json["content"])
         return self
 
+    def from_url(self, url: str):
+        """
+        Download a file from an url.
+        """
+        # code inspired by: https://github.com/runpod/runpod-python/blob/main/runpod/serverless/utils/rp_download.py
+        import requests
+        HEADERS = {"User-Agent": "runpod-python/0.0.0 (https://runpod.io; support@runpod.io)"}
+        with requests.get(url, headers=HEADERS, stream=True, timeout=5) as response:
+            response.raise_for_status()
+
+            # get orig file name or create new
+            original_file_name = []
+            if "Content-Disposition" in response.headers.keys():
+                original_file_name = re.findall(
+                    "filename=(.+)",
+                    response.headers["Content-Disposition"]
+                )
+
+            if len(original_file_name) > 0:
+                original_file_name = original_file_name[0]
+            else:
+                download_path = urlparse(url).path
+                original_file_name = os.path.basename(download_path)
+
+            # DOWNLOAD FILE IN Chunks
+            file_size = int(response.headers.get('Content-Length', 0))
+            # calculate chunk_size
+            if file_size <= 1024 * 1024:  # 1 MB
+                chunk_size = 1024  # 1 KB
+            elif file_size <= 1024 * 1024 * 1024:  # 1 GB
+                chunk_size = 1024 * 1024  # 1 MB
+            else:
+                chunk_size = 1024 * 1024 * 10  # 10 MB
+
+            # write the content in chunks to the file
+            file = io.BytesIO()
+            for chunk in response.iter_content(chunk_size=chunk_size):
+                if chunk:  # filter out keep-alive chunks
+                    file.write(chunk)
+            file.name = original_file_name
+            self.file_name = original_file_name
+
+            # self.url = url
+
+            return self.from_bytesio_or_handle(file, copy=False)
+
     @requires_numpy()
     def to_np_array(self, shape=None, dtype=np.uint8):
         """
@@ -194,7 +249,7 @@ class MediaFile:
         return self._content_buffer
 
     def to_base64(self):
-        return base64.b64encode(self.to_bytes()).decode()
+        return base64.b64encode(self.to_bytes()).decode('ascii')
 
     def to_httpx_send_able_tuple(self):
         return self.file_name, self.read(), self.content_type
@@ -226,7 +281,7 @@ class MediaFile:
             path = os.path.join(path, self.file_name)
 
         # check if has extension
-        #if os.path.splitext(path)[1] == "":
+        # if os.path.splitext(path)[1] == "":
         #    path += ".mp4"
 
         with open(path, 'wb') as file:
@@ -246,6 +301,7 @@ class MediaFile:
         # from np_array -> tempfile
         # from starlette_upload_file -> from_buffered_reader(spooled_temporary) -> info from the spooled_temporary
         # from base64 -> from-bytes -> tempfile
+        # from url -> from bytesio
         if self.path is not None:
             self.file_name = os.path.basename(self.path)
             self.content_type = mimetypes.guess_type(self.file_name)[0] or "application/octet-stream"
@@ -255,6 +311,21 @@ class MediaFile:
         if self.content_type is None:
             self.content_type = "application/octet-stream"
 
+    def file_size(self, unit="bytes") -> int:
+        """
+        :param unit:
+        """
+        size_in_ = self._content_buffer.getbuffer().nbytes
+        if unit == "bytes":
+            return size_in_
+        elif unit == "kb":
+            size_in_ = size_in_ / 1000
+        elif unit == "mb":
+            size_in_ = size_in_ / 1000000
+        elif unit == "gb":
+            size_in_ = size_in_ / 1000000000
+        return size_in_
+
     def __bytes__(self):
         return self.to_bytes()
 
@@ -301,3 +372,6 @@ class MediaFile:
         except:
             return False
 
+    @staticmethod
+    def _is_url(url: str):
+        return urlparse(url).scheme in ['http', 'https']

{media_toolkit-0.0.8 → media_toolkit-0.1.1.dev1}/media_toolkit/utils/file_conversion.py

@@ -37,11 +37,11 @@ def media_from_file(file_path: str) -> Union[MediaFile, ImageFile, AudioFile, Vi
     return MediaFile().from_file(file_path)
 
 
-def convert_to_upload_file_type(file, target_media_file=None):
+def media_from_any(file, media_file_type=None):
     """
     Converts a file to a send able format.
     :param file: The file to convert.
-    :param target_media_file: The target type to convert to. If not specified will be converted to MediaFile.
+    :param media_file_type: The target type to convert to. If not specified will be converted to MediaFile.
         Use ImageFile, AudioFile, VideoFile to convert to those types.
     :return: The send able file.
     """
@@ -51,8 +51,8 @@ def convert_to_upload_file_type(file, target_media_file=None):
 
     # determine target class
     target_class = MediaFile
-    if target_media_file is not None and issubclass(target_media_file, MediaFile):
-        target_class = target_media_file
+    if media_file_type is not None and issubclass(media_file_type, MediaFile):
+        target_class = media_file_type
     media_file_instance = target_class()
 
     # load data
@@ -60,7 +60,7 @@ def convert_to_upload_file_type(file, target_media_file=None):
     return media_file_instance
 
 
-def from_file_result(file_result: dict):
+def media_from_file_result(file_result: dict):
     """
     Converts a file result to a MediaFile.
     :param file_result: The file result to convert.

{media_toolkit-0.0.8 → media_toolkit-0.1.1.dev1}/media_toolkit.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: media-toolkit
-Version: 0.0.8
+Version: 0.1.1.dev1
 Summary: Web-ready standardized file processing and serialization. Read, load and convert to standard file types with a common interface.
 Author: SocAIty
 License: GPLv3
@@ -46,10 +46,11 @@ Load and convert from and to common data types:
 - bytes,
 - base64
 - json
+- urls
 - etc.
 
 Transmit files between services with a common interface
-- Native FastSDK and FastTaskAPI integration
+- Native [FastSDK](https://github.com/SocAIty/fastSDK) and [FastTaskAPI](https://github.com/SocAIty/FastTaskAPI) integration
 - Supports httpx, requests
 
 Work with native python libs like BytesIO.
@@ -63,9 +64,9 @@ You can install the package with PIP, or clone the repository.
 ```bash
 # install from pypi
 pip install media-toolkit
-# install without dependencies: this is useful if you only need the basic functionality
+# install without dependencies: this is useful if you only need the basic functionality (working with files)
 pip install media-toolkit --no-deps
-# if you only want to use certain file types
+# if you want to use certain file types, and convenience functions
 pip install media-toolkit[VideoFile]  # or [AudioFile, VideoFile, ...]
 # install from github for newest release
 pip install git+git://github.com/SocAIty/media-toolkit
@@ -84,7 +85,7 @@ The library automatically detects the data type and loads it correctly.
 ```python
 from media_toolkit import MediaFile, ImageFile, AudioFile, VideoFile
 
-# could be a path, base64, bytesio, file_handle, numpy array ...
+# could be a path, url, base64, bytesio, file_handle, numpy array ...
 arbitrary_data = "...."
 # Instantiate an image file
 new_file = ImageFile().from_any(arbitrary_data)
@@ -189,6 +190,7 @@ my_files = {
 response = httpx.Client().post(url, files=my_files)
 ```
 
+
 # How it works
 
 If media-file is instantiated with ```from_*``` it converts it to an intermediate representation.
@@ -199,5 +201,5 @@ Currently the intermediate representation is supported in memory with (BytesIO).
 
 # ToDo:
 
-[x] additionally support tempfile backend instead of working bytesio memory mode only.
-[x] decreasing redundancies for _file_info() method
+- [x] additionally support tempfile backend instead of working bytesio memory mode only.
+- [x] decreasing redundancies for _file_info() method

{media_toolkit-0.0.8 → media_toolkit-0.1.1.dev1}/media_toolkit.egg-info/SOURCES.txt

@@ -18,4 +18,5 @@ media_toolkit/utils/__init__.py
 media_toolkit/utils/dependency_requirements.py
 media_toolkit/utils/file_conversion.py
 media_toolkit/utils/generator_wrapper.py
+test/test_image_file.py
 test/test_video_file.py

{media_toolkit-0.0.8 → media_toolkit-0.1.1.dev1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "media-toolkit"
-version = "0.0.8"
+version = "0.1.1.dev1"
 description = "Web-ready standardized file processing and serialization. Read, load and convert to standard file types with a common interface."
 requires-python = ">=3.8"
 authors = [

media_toolkit-0.1.1.dev1/test/test_image_file.py

@@ -0,0 +1,11 @@
+
+from media_toolkit import ImageFile
+
+outdir = "outdir/"
+def test_img_from_url():
+    url = "https://github.com/SocAIty/face2face/blob/main/test/test_imgs/test_face_1.jpg?raw=true"
+    fromurl = ImageFile().from_any(url)
+    fromurl.save(f"{outdir}test_face_1_1.jpg")
+
+
+test_img_from_url()

media_toolkit-0.1.1.dev1/test/test_video_file.py

@@ -0,0 +1,39 @@
+"""
+Tests some core functionalities of the VideoFile class.
+"""
+from tqdm import tqdm
+
+from media_toolkit import VideoFile, ImageFile
+import cv2
+
+outdir = "outdir/"
+
+def test_video_file():
+    test_video = "test_files/test_video.mp4"
+    vf = VideoFile().from_file(test_video)
+    # extract audio_file
+    vf.extract_audio(f"{outdir}/extracted_audio.mp3")
+    audio_bytes = vf.extract_audio()
+
+def test_video_from_files():
+    files = [f"{outdir}/test_out_video_stream_{i}.png" for i in range(10)]
+    vf = VideoFile().from_files(files)
+    vf.add_audio(f"{outdir}/extracted_audio.mp3")
+    vf.save(f"{outdir}/test_from_files_add_audio.mp4")
+    # from dir; and combine audio and video
+    fromdir = VideoFile().from_dir(outdir, audio=f"{outdir}/extracted_audio.mp3", frame_rate=30)
+    fromdir.save(f"{outdir}/test_from_dir.mp4")
+
+def test_video_stream():
+    audio_array = []
+    image_paths = []
+    for i, (img, audio_part) in tqdm(enumerate(vf.to_video_stream(include_audio=True))):
+        p = f"{outdir}/test_out_video_stream_{i}.png"
+        image_paths.append(p)
+        cv2.imwrite(p, img)
+        audio_array.append(audio_part)
+
+    # test video clients with audio_file
+    fromstream = VideoFile().from_video_stream(fromdir.to_video_stream(include_audio=True))
+    fromstream.save(f"{outdir}/test_from_stream.mp4")
+

media_toolkit-0.0.8/media_toolkit/__init__.py

@@ -1,2 +0,0 @@
-from media_toolkit.core import MediaFile, ImageFile, VideoFile, AudioFile
-from utils import media_from_file

media_toolkit-0.0.8/media_toolkit/utils/__init__.py

@@ -1 +0,0 @@
-from .file_conversion import media_from_file

media_toolkit-0.0.8/test/test_video_file.py

@@ -1,39 +0,0 @@
-"""
-Tests some core functionalities of the VideoFile class.
-"""
-from tqdm import tqdm
-
-from media_toolkit import VideoFile
-import cv2
-
-outdir = "outdir/"
-
-# test video files
-#test_video = "test_files/test_video.mp4"
-#vf = VideoFile().from_file(test_video)
-#### extract audio_file
-#vf.extract_audio(f"{outdir}/extracted_audio.mp3")
-##audio_bytes = vf.extract_audio()
-#
-### test video clients
-#audio_array = []
-#image_paths = []
-#for i, (img, audio_part) in tqdm(enumerate(vf.to_video_stream(include_audio=True))):
-#    p = f"{outdir}/test_out_video_stream_{i}.png"
-#    image_paths.append(p)
-#    cv2.imwrite(p, img)
-#    audio_array.append(audio_part)
-
-# new video File
-#vf = VideoFile().from_files(image_paths)
-#vf.add_audio(audio_array)
-#vf.save(f"{outdir}/test_from_files_add_audio.mp4")
-# from dir; and combine audio and video
-# files = [f"{outdir}/test_out_video_stream_{i}.png" for i in range(10)]
-
-fromdir = VideoFile().from_dir(outdir, audio=f"{outdir}/extracted_audio.mp3", frame_rate=30)
-fromdir.save(f"{outdir}/test_from_dir.mp4")
-
-# test video clients with audio_file
-fromstream = VideoFile().from_video_stream(fromdir.to_video_stream(include_audio=True))
-fromstream.save(f"{outdir}/test_from_stream.mp4")