vision-agent 0.0.37__py3-none-any.whl → 0.0.39__py3-none-any.whl

vision_agent/tools/tools.py

@@ -11,6 +11,7 @@ from PIL import Image
 from PIL.Image import Image as ImageType
 
 from vision_agent.image_utils import convert_to_b64, get_image_size
+from vision_agent.tools.video import extract_frames_from_video
 
 _LOGGER = logging.getLogger(__name__)
 
@@ -505,6 +506,47 @@ class Divide(Tool):
         return round(input[0] / input[1], 2)
 
 
+class ExtractFrames(Tool):
+    r"""Extract frames from a video."""
+
+    name = "extract_frames_"
+    description = "'extract_frames_' extract image frames from the input video, return a list of tuple (frame, timestamp), where the timestamp is the relative time in seconds of the frame occurred in the video, the frame is a local image file path that stores the frame."
+    usage = {
+        "required_parameters": [{"name": "video_uri", "type": "str"}],
+        "examples": [
+            {
+                "scenario": "Can you extract the frames from this video? Video: www.foobar.com/video?name=test.mp4",
+                "parameters": {"video_uri": "www.foobar.com/video?name=test.mp4"},
+            },
+            {
+                "scenario": "Can you extract the images from this video file? Video path: tests/data/test.mp4",
+                "parameters": {"video_uri": "tests/data/test.mp4"},
+            },
+        ],
+    }
+
+    def __call__(self, video_uri: str) -> list[tuple[str, float]]:
+        """Extract frames from a video.
+
+
+        Parameters:
+            video_uri: the path to the video file or a url points to the video data
+
+        Returns:
+            a list of tuples containing the extracted frame and the timestamp in seconds. E.g. [(path_to_frame1, 0.0), (path_to_frame2, 0.5), ...]. The timestamp is the time in seconds from the start of the video. E.g. 12.125 means 12.125 seconds from the start of the video. The frames are sorted by the timestamp in ascending order.
+        """
+        frames = extract_frames_from_video(video_uri)
+        result = []
+        _LOGGER.info(
+            f"Extracted {len(frames)} frames from video {video_uri}. Temporarily saving them as images to disk for downstream tasks."
+        )
+        for frame, ts in frames:
+            with tempfile.NamedTemporaryFile(suffix=".jpg", delete=False) as tmp:
+                Image.fromarray(frame).save(tmp)
+            result.append((tmp.name, ts))
+        return result
+
+
 TOOLS = {
     i: {"name": c.name, "description": c.description, "usage": c.usage, "class": c}
     for i, c in enumerate(
@@ -520,6 +562,7 @@ TOOLS = {
             Subtract,
             Multiply,
             Divide,
+            ExtractFrames,
         ]
     )
     if (hasattr(c, "name") and hasattr(c, "description") and hasattr(c, "usage"))

vision_agent/tools/video.py

@@ -0,0 +1,176 @@
+import logging
+import math
+import os
+from concurrent.futures import ProcessPoolExecutor, as_completed
+from typing import List, Tuple, cast
+
+import cv2
+import numpy as np
+from moviepy.video.io.VideoFileClip import VideoFileClip
+from tqdm import tqdm
+
+_LOGGER = logging.getLogger(__name__)
+# The maximum length of the clip to extract frames from, in seconds
+_CLIP_LENGTH = 30.0
+
+
+def extract_frames_from_video(
+    video_uri: str, fps: int = 2, motion_detection_threshold: float = 0.06
+) -> List[Tuple[np.ndarray, float]]:
+    """Extract frames from a video
+
+    Parameters:
+        video_uri: the path to the video file or a video file url
+        fps: the frame rate per second to extract the frames
+        motion_detection_threshold: The threshold to detect motion between changes/frames.
+            A value between 0-1, which represents the percentage change required for the frames to be considered in motion.
+            For example, a lower value means more frames will be extracted.
+
+    Returns:
+        a list of tuples containing the extracted frame and the timestamp in seconds. E.g. [(frame1, 0.0), (frame2, 0.5), ...]. The timestamp is the time in seconds from the start of the video. E.g. 12.125 means 12.125 seconds from the start of the video. The frames are sorted by the timestamp in ascending order.
+    """
+    with VideoFileClip(video_uri) as video:
+        video_duration: float = video.duration
+        num_workers = os.cpu_count()
+        clip_length: float = min(video_duration, _CLIP_LENGTH)
+        start_times = list(range(0, math.ceil(video_duration), math.ceil(clip_length)))
+        assert start_times, f"No frames to extract from the input video: {video_uri}"
+        segment_args = [
+            {
+                "video_uri": video_uri,
+                "start": start,
+                "end": (
+                    start + clip_length if i < len(start_times) - 1 else video_duration
+                ),
+                "fps": fps,
+                "motion_detection_threshold": motion_detection_threshold,
+            }
+            for i, start in enumerate(start_times)
+        ]
+        if (
+            cast(float, segment_args[-1]["end"])
+            - cast(float, segment_args[-1]["start"])
+            < 1
+        ):
+            # If the last segment is less than 1s, merge it with the previous segment
+            # This is to avoid the failure of the last segment extraction
+            assert (
+                len(segment_args) > 1
+            ), "Development bug - Expect at least 2 segments."
+            segment_args[-2]["end"] = video_duration
+            segment_args.pop(-1)
+        _LOGGER.info(
+            f"""Created {len(segment_args)} segments from the input video  {video_uri} of length {video.duration}s, with clip size: {clip_length}s and {num_workers} workers.
+            Segments: {segment_args}
+            """
+        )
+        frames = []
+        with tqdm(total=len(segment_args)) as pbar:
+            with ProcessPoolExecutor(max_workers=num_workers) as executor:
+                futures = [
+                    executor.submit(_extract_frames_by_clip, **kwargs)  # type: ignore
+                    for kwargs in segment_args
+                ]
+                for future in as_completed(futures):
+                    result = future.result()
+                    frames.extend(result)
+                    pbar.update(1)
+        frames.sort(key=lambda x: x[1])
+        _LOGGER.info(f"Extracted {len(frames)} frames from video {video_uri}")
+        return frames
+
+
+def _extract_frames_by_clip(
+    video_uri: str,
+    start: int = 0,
+    end: float = -1,
+    fps: int = 2,
+    motion_detection_threshold: float = 0.06,
+) -> List[Tuple[np.ndarray, float]]:
+    """Extract frames from a video clip with start and end time in seconds.
+
+    Parameters:
+        video_uri: the path to the video file or a video file url
+        start: the start time (in seconds) of the clip to extract
+        end: the end time (in seconds, up to millisecond level precision) of the clip to extract, if -1, extract the whole video
+        fps: the frame rate to extract the frames
+        motion_detection_threshold: the threshold to detect the motion between frames
+    """
+    with VideoFileClip(video_uri) as video:
+        source_fps = video.fps
+        if end <= 0:
+            end = video.duration
+        _LOGGER.info(
+            f"Extracting frames from video {video_uri} ({video.duration}s) with start={start}s and end={end}s"
+        )
+        clip = video.subclip(start, end)
+        processable_frames = int(clip.duration * fps)
+        _LOGGER.info(
+            f"Extracting frames from video clip of length {clip.duration}s with FPS={fps} and start_time={start}s. Total number of frames in clip: {processable_frames}"
+        )
+        frames = []
+        total_count, skipped_count = 0, 0
+        prev_processed_frame = None
+        pbar = tqdm(
+            total=processable_frames, desc=f"Extracting frames from clip {start}-{end}"
+        )
+        for i, frame in enumerate(clip.iter_frames(fps=fps, dtype="uint8")):
+            curr_processed_frame = _preprocess_frame(frame)
+            total_count += 1
+            pbar.update(1)
+            # Skip the frame if it is similar to the previous one
+            if prev_processed_frame is not None and _similar_frame(
+                prev_processed_frame,
+                curr_processed_frame,
+                threshold=motion_detection_threshold,
+            ):
+                skipped_count += 1
+                continue
+            prev_processed_frame = curr_processed_frame
+            ts = round(clip.reader.pos / source_fps, 3)
+            frames.append((frame, ts))
+
+        _LOGGER.info(
+            f"""Finished!
+                Frames extracted: {len(frames)}
+                Extracted frame timestamp: {[f[1] for f in frames]}
+                Total processed frames: {total_count}
+                Skipped frames:  {skipped_count}
+                Scan FPS: {fps}
+                Clip start time: {start}s, {clip.pos}
+                Clip end time: {end}s
+                Clip duration: {clip.duration}s
+                Clip total frames: {clip.duration * source_fps}
+                Video duration: {video.duration}s
+                Video FPS: {video.fps}
+                Video total frames: {video.reader.nframes}"""
+        )
+        return frames
+
+
+def _preprocess_frame(frame: np.ndarray) -> np.ndarray:
+    # Convert to grayscale
+    frame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
+    frame = cv2.GaussianBlur(src=frame, ksize=(5, 5), sigmaX=0)
+    return frame
+
+
+def _similar_frame(
+    prev_frame: np.ndarray, curr_frame: np.ndarray, threshold: float
+) -> bool:
+    """Detect two frames are similar or not
+
+    Parameters:
+        threshold: similarity threshold, a value between 0-1, the percentage change that is considered a different frame.
+    """
+    # calculate difference and update previous frame TODO: don't assume the processed image is cached
+    diff_frame = cv2.absdiff(src1=prev_frame, src2=curr_frame)
+    # Only take different areas that are different enough (>20 / 255)
+    thresh_frame = cv2.threshold(
+        src=diff_frame, thresh=20, maxval=255, type=cv2.THRESH_BINARY
+    )[1]
+    change_percentage = cv2.countNonZero(thresh_frame) / (
+        curr_frame.shape[0] * curr_frame.shape[1]
+    )
+    _LOGGER.debug(f"Image diff: {change_percentage}")
+    return change_percentage < threshold

{vision_agent-0.0.37.dist-info → vision_agent-0.0.39.dist-info}/METADATA

@@ -1,16 +1,19 @@
 Metadata-Version: 2.1
 Name: vision-agent
-Version: 0.0.37
+Version: 0.0.39
 Summary: Toolset for Vision Agent
 Author: Landing AI
 Author-email: dev@landing.ai
-Requires-Python: >=3.10,<3.12
+Requires-Python: >=3.9,<3.12
 Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Requires-Dist: faiss-cpu (>=1.0.0,<2.0.0)
+Requires-Dist: moviepy (>=1.0.0,<2.0.0)
 Requires-Dist: numpy (>=1.21.0,<2.0.0)
 Requires-Dist: openai (>=1.0.0,<2.0.0)
+Requires-Dist: opencv-python-headless (>=4.0.0,<5.0.0)
 Requires-Dist: pandas (>=2.0.0,<3.0.0)
 Requires-Dist: pillow (>=10.0.0,<11.0.0)
 Requires-Dist: requests (>=2.0.0,<3.0.0)
@@ -42,6 +45,11 @@ aims to provide an in-seconds experience by allowing users to describe their pro
 text and utilizing agent frameworks to solve the task for them. Check out our discord
 for updates and roadmaps!
 
+## Documentation
+
+- [Vision Agent Library Docs](https://landing-ai.github.io/vision-agent/)
+
+
 ## Getting Started
 ### Installation
 To get started, you can install the library using pip:
@@ -124,6 +132,7 @@ you. For example:
 | Crop | Crop crops an image given a bounding box and returns a file name of the cropped image. |
 | BboxArea | BboxArea returns the area of the bounding box in pixels normalized to 2 decimal places. |
 | SegArea | SegArea returns the area of the segmentation mask in pixels normalized to 2 decimal places. |
+| ExtractFrames | ExtractFrames extracts image frames from the input video. |
 
 
 It also has a basic set of calculate tools such as add, subtract, multiply and divide.

{vision_agent-0.0.37.dist-info → vision_agent-0.0.39.dist-info}/RECORD

@@ -18,8 +18,9 @@ vision_agent/lmm/__init__.py,sha256=I8mbeNUajTfWVNqLsuFQVOaNBDlkIhYp9DFU8H4kB7g,
 vision_agent/lmm/lmm.py,sha256=ARcbgkcyP83TbVVoXI9B-gtG0gJuTaG_MjcUGbams4U,8052
 vision_agent/tools/__init__.py,sha256=aX0pU3pXU1V0Cj9FzYCvdsX76TAglFMHx59kNhXHbPs,131
 vision_agent/tools/prompts.py,sha256=9RBbyqlNlExsGKlJ89Jkph83DAEJ8PCVGaHoNbyN7TM,1416
-vision_agent/tools/tools.py,sha256=Vlb8H9qm4rA5HxGw5p-gJES6jgPIkfrtVlM7jcxw7d8,19141
-vision_agent-0.0.37.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-vision_agent-0.0.37.dist-info/METADATA,sha256=Y9oIfWbRK-3EuNewrwK4WOnpHY2ca7FB8jDa5oucT5Y,4966
-vision_agent-0.0.37.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
-vision_agent-0.0.37.dist-info/RECORD,,
+vision_agent/tools/tools.py,sha256=2mmomPDbldXRpw3q5zAcazKJMjAGd0Jl9ak9JykHQYI,21211
+vision_agent/tools/video.py,sha256=KV_Wcat7DDGxpHSaGBu7s4lj4crlYaUu4YKpCO_86k4,7440
+vision_agent-0.0.39.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+vision_agent-0.0.39.dist-info/METADATA,sha256=_jugEQnOeNbLa3kSSo0zTn2bII3Rh5dfop9qyMWXPfw,5282
+vision_agent-0.0.39.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
+vision_agent-0.0.39.dist-info/RECORD,,