hjxdl 0.3.39__py3-none-any.whl → 0.3.40__py3-none-any.whl

hdl/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.3.39'
-__version_tuple__ = version_tuple = (0, 3, 39)
+__version__ = version = '0.3.40'
+__version_tuple__ = version_tuple = (0, 3, 40)

hdl/utils/llm/llm_wrapper.py

@@ -142,6 +142,8 @@ class OpenAIWrapper(object):
         assis_info: str = None,
         images: list = None,
         image_keys: tuple = ("image_url", "url"),
+        videos: list = None,
+        video_keys: tuple = ("video_url", "url"),
         model: str=None,
         tools: list = None,
         tool_choice: str = "auto",
@@ -207,10 +209,30 @@ class OpenAIWrapper(object):
         elif len(image_keys) == 1:
             image_keys = (image_keys[0],) * 3
 
+        if isinstance(video_keys, str):
+            video_keys = (video_keys,) * 3
+        elif len(video_keys) == 2:
+            video_keys = (video_keys[0],) + tuple(video_keys)
+        elif len(video_keys) == 1:
+            video_keys = (video_keys[0],) * 3
+
         content = [{
             "type": "text",
             "text": prompt
         }]
+
+        if videos:
+            if isinstance(videos, str):
+                images = [videos]
+            for video in videos:
+                content.append({
+                    "type": video_keys[0],
+                    video_keys[1]: {
+                        video_keys[2]: img
+                    }
+                })
+
+
         if images:
             if isinstance(images, str):
                 images = [images]
@@ -221,8 +243,7 @@ class OpenAIWrapper(object):
                         image_keys[2]: img
                     }
                 })
-        else:
-            # If no images are provided, content is simply the prompt text
+        if (not images) and (not videos):
             content = prompt
 
         # Add the user's input as a message

hdl/utils/llm/vis.py

@@ -24,6 +24,13 @@ from hdl.jupyfuncs.show.pbar import tqdm
 from redis.commands.search.query import Query
 
 
+from decord import VideoReader, cpu
+
+import base64
+from io import BytesIO
+from PIL import Image
+import numpy as np
+import requests
 # from ..database_tools.connect import conn_redis
 
 
@@ -731,4 +738,65 @@ class ImgHandler:
         return results
 
 
+def to_video_base64(video_path, max_frames=80, is_fps_sampling=True):
+    """
+    Converts a video file or a video from a given URL to a list of base64 encoded image frames.
+
+    Args:
+        video_path (str): The path to the video file or the URL of the video.
+        max_frames (int, optional): The maximum number of frames to return. Defaults to 80.
+        is_fps_sampling (bool, optional): A flag to indicate whether to sample frames based on average FPS. Defaults to True.
 
+    Returns:
+        list: A list of base64 encoded strings representing the sampled frames of the video.
+
+    Raises:
+        Exception: If the video cannot be loaded or if the URL is unreachable.
+
+    Note:
+        This function uses the VideoReader class for reading video frames and requires the necessary libraries for handling images and base64 encoding.
+    """
+    if video_path.startswith("http") or video_path.startswith("https"):
+        response = requests.get(video_path)
+        if response.status_code == 200:
+            video_path = BytesIO(response.content)
+        else:
+            print('failed to load the video')
+
+        vr = VideoReader(video_path, ctx=cpu(0), num_threads=1)
+        total_frame_num = len(vr)
+        if is_fps_sampling:
+            # FPS Sampling
+            avg_fps = round(vr.get_avg_fps())
+            frame_idx = [i for i in range(
+                0, total_frame_num, avg_fps)]
+            if len(frame_idx) > max_frames:
+                uniform_sampled_frames = np.linspace(
+                    0, total_frame_num - 1, max_frames, dtype=int
+                )
+                frame_idx = uniform_sampled_frames.tolist()
+            print(frame_idx)
+        else:
+            # uniform sampling
+            if total_frame_num > max_frames:
+                uniform_sampled_frames = np.linspace(
+                    0, total_frame_num - 1, max_frames, dtype=int
+                )
+                frame_idx = uniform_sampled_frames.tolist()
+            else:
+                frame_idx = [i for i in range(0, total_frame_num)]
+            print(frame_idx)
+
+        frames = vr.get_batch(frame_idx).asnumpy()
+        print("actual frames", len(frames))
+
+        base64_frames = []
+        for frame in frames:
+            img = Image.fromarray(frame)
+            output_buffer = BytesIO()
+            img.save(output_buffer, format="PNG")
+
+            byte_data = output_buffer.getvalue()
+            base64_str = base64.b64encode(byte_data).decode("utf-8")
+            base64_frames.append(base64_str)
+        return base64_frames

{hjxdl-0.3.39.dist-info → hjxdl-0.3.40.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hjxdl
-Version: 0.3.39
+Version: 0.3.40
 Summary: A collection of functions for Jupyter notebooks
 Home-page: https://github.com/huluxiaohuowa/hdl
 Author: Jianxing Hu

{hjxdl-0.3.39.dist-info → hjxdl-0.3.40.dist-info}/RECORD

@@ -1,5 +1,5 @@
 hdl/__init__.py,sha256=GffnD0jLJdhkd-vo989v40N90sQbofkayRBwxc6TVhQ,72
-hdl/_version.py,sha256=X3NJYp9_3dCmctRYb1tZdvHuuPikZELQ99611KxJNh4,513
+hdl/_version.py,sha256=FM2ay9B4jNpEspCcoKvkx6ALhoNo3bt_Okj6w9c819c,513
 hdl/args/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hdl/args/loss_args.py,sha256=s7YzSdd7IjD24rZvvOrxLLFqMZQb9YylxKeyelSdrTk,70
 hdl/controllers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -136,16 +136,16 @@ hdl/utils/llm/chatgr.py,sha256=5F5PJHe8vz3iCfi4TT54DCLRi1UeJshECdVtgvvvao0,3696
 hdl/utils/llm/embs.py,sha256=Tf0FOYrOFZp7qQpEPiSCXzlgyHH0X9HVTUtsup74a9E,7174
 hdl/utils/llm/extract.py,sha256=2sK_WJzmYIc8iuWaM9DA6Nw3_6q1O4lJ5pKpcZo-bBA,6512
 hdl/utils/llm/llama_chat.py,sha256=watcHGOaz-bv3x-yDucYlGk5f8FiqfFhwWogrl334fk,4387
-hdl/utils/llm/llm_wrapper.py,sha256=ZjmogEM0TrvlM83LG6l5t8CFnMThBWUmUOvA5WY9ODo,15309
+hdl/utils/llm/llm_wrapper.py,sha256=kectOfy2lbT0UcSYXbbmeQDlYMtdf6gpV10fI7ilgug,15929
 hdl/utils/llm/ollama.py,sha256=uEdLsNAc6b56r37hNiE3nrd6oZ2lmQ0gYbVvOc9YVIM,1389
-hdl/utils/llm/vis.py,sha256=SSP6tOwKLq0hWcpM3twI9TitqzBmKjlcGrnXEWYlCzM,26055
+hdl/utils/llm/vis.py,sha256=CLZApXlC636mYsxP7yg1UWRn0Q16pXpOX8ShJbfv5us,28739
 hdl/utils/llm/visrag.py,sha256=0i-VrxqgiV-J7R3VPshu9oc7-rKjFJOldYik3HDXj6M,10176
 hdl/utils/schedulers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hdl/utils/schedulers/norm_lr.py,sha256=bDwCmdEK-WkgxQMFBiMuchv8Mm7C0-GZJ6usm-PQk14,4461
 hdl/utils/weather/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hdl/utils/weather/weather.py,sha256=k11o6wM15kF8b9NMlEfrg68ak-SfSYLN3nOOflFUv-I,4381
-hjxdl-0.3.39.dist-info/licenses/LICENSE,sha256=lkMiSbeZHBQLB9LJEkS9-L3Z-LBC4yGnKrzHSG8RkPM,2599
-hjxdl-0.3.39.dist-info/METADATA,sha256=lGbo0pGz-o-TMwXK0Fa3NPke0werbwIaNasB3TuDqcY,1358
-hjxdl-0.3.39.dist-info/WHEEL,sha256=DK49LOLCYiurdXXOXwGJm6U4DkHkg4lcxjhqwRa0CP4,91
-hjxdl-0.3.39.dist-info/top_level.txt,sha256=-kxwTM5JPhylp06z3zAVO3w6_h7wtBfBo2zgM6YZoTk,4
-hjxdl-0.3.39.dist-info/RECORD,,
+hjxdl-0.3.40.dist-info/licenses/LICENSE,sha256=lkMiSbeZHBQLB9LJEkS9-L3Z-LBC4yGnKrzHSG8RkPM,2599
+hjxdl-0.3.40.dist-info/METADATA,sha256=otgYEo9D3UeXTRx9Qqgyi7e5f4t6ZonGvfTdDbMoS4I,1358
+hjxdl-0.3.40.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+hjxdl-0.3.40.dist-info/top_level.txt,sha256=-kxwTM5JPhylp06z3zAVO3w6_h7wtBfBo2zgM6YZoTk,4
+hjxdl-0.3.40.dist-info/RECORD,,

{hjxdl-0.3.39.dist-info → hjxdl-0.3.40.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (78.0.2)
+Generator: setuptools (78.1.0)
 Root-Is-Purelib: true
 Tag: py3-none-any