videopython 0.4.0__py3-none-any.whl → 0.5.0__py3-none-any.whl

videopython/ai/understanding/transcribe.py

@@ -1,37 +1,66 @@
-from typing import Literal
+from typing import Literal, Union
 
 import whisper
+from soundpython import Audio
 
-from videopython.base.transcription import Transcription, TranscriptionSegment
+from videopython.base.text.transcription import Transcription, TranscriptionSegment, TranscriptionWord
 from videopython.base.video import Video
 
 
-class VideoTranscription:
+class CreateTranscription:
+    """Unified transcription service for both audio and video."""
+
     def __init__(self, model_name: Literal["tiny", "base", "small", "medium", "large", "turbo"] = "small") -> None:
         self.model = whisper.load_model(name=model_name)
 
-    def transcribe_video(self, video: Video) -> Transcription:
-        """Transcribes video to text.
+    def _process_transcription_result(self, transcription_result: dict) -> Transcription:
+        """Process raw transcription result into Transcription object.
 
         Args:
-            video: Video to transcribe.
+            transcription_result: Raw result from whisper model
 
         Returns:
-            List of dictionaries with segments of text and their start and end times.
+            Processed Transcription object
         """
-        if video.audio.is_silent:
-            return Transcription(segments=[])
+        transcription_segments = []
+        for segment in transcription_result["segments"]:
+            transcription_words = [
+                TranscriptionWord(word=word["word"], start=float(word["start"]), end=float(word["end"]))
+                for word in segment["words"]
+            ]
+            transcription_segment = TranscriptionSegment(
+                start=segment["start"], end=segment["end"], text=segment["text"], words=transcription_words
+            )
+            transcription_segments.append(transcription_segment)
+
+        return Transcription(segments=transcription_segments)
+
+    def transcribe(self, media: Union[Audio, Video]) -> Transcription:
+        """Transcribe audio or video to text.
+
+        Args:
+            media: Audio or Video to transcribe.
+
+        Returns:
+            Transcription object with segments of text and their timestamps.
+        """
+        if isinstance(media, Video):
+            # Handle video transcription
+            if media.audio.is_silent:
+                return Transcription(segments=[])
+
+            audio = media.audio.to_mono().resample(whisper.audio.SAMPLE_RATE)
+            transcription_result = self.model.transcribe(audio=audio.data, word_timestamps=True)
 
-        audio = video.audio.to_mono()
-        audio = audio.resample(whisper.audio.SAMPLE_RATE)
-        audio_data = audio.data
+        elif isinstance(media, Audio):
+            # Handle audio transcription
+            if media.is_silent:
+                return Transcription(segments=[])
 
-        transcription = self.model.transcribe(audio=audio_data, word_timestamps=True)
+            audio = media.to_mono().resample(whisper.audio.SAMPLE_RATE)
+            transcription_result = self.model.transcribe(audio=audio.data, word_timestamps=True)
 
-        transcription_segments = [
-            TranscriptionSegment(start=segment["start"], end=segment["end"], text=segment["text"])
-            for segment in transcription["segments"]
-        ]
-        result = Transcription(segments=transcription_segments)
+        else:
+            raise TypeError(f"Unsupported media type: {type(media)}. Expected Audio or Video.")
 
-        return result
+        return self._process_transcription_result(transcription_result)

videopython/base/combine.py

@@ -0,0 +1,45 @@
+from typing import Literal
+
+import numpy as np
+
+from videopython.base.transforms import ResampleFPS, Resize
+from videopython.base.video import Video
+
+
+class StackVideos:
+    def __init__(self, mode: Literal["horizontal", "vertical"]) -> None:
+        self.mode = mode
+
+    def _validate(self, video1: Video, video2: Video) -> tuple[Video, Video]:
+        video1, video2 = self._align_shapes(video1, video2)
+        video1, video2 = self._align_fps(video1, video2)
+        video1, video2 = self._align_duration(video1, video2)
+        return video1, video2
+
+    def _align_fps(self, video1: Video, video2: Video) -> tuple[Video, Video]:
+        if video1.fps > video2.fps:
+            video1 = ResampleFPS(fps=video2.fps).apply(video1)
+        elif video1.fps < video2.fps:
+            video2 = ResampleFPS(fps=video1.fps).apply(video2)
+        return (video1, video2)
+
+    def _align_shapes(self, video1: Video, video2: Video) -> tuple[Video, Video]:
+        if self.mode == "horizontal":
+            video2 = Resize(height=video1.metadata.height).apply(video2)
+        elif self.mode == "vertical":
+            video2 = Resize(width=video1.metadata.width).apply(video2)
+        return (video1, video2)
+
+    def _align_duration(self, video1: Video, video2: Video) -> tuple[Video, Video]:
+        if len(video1.frames) > len(video2.frames):
+            video1 = video1[: len(video2.frames)]
+        elif len(video1.frames) < len(video2.frames):
+            video2 = video2[: len(video1.frames)]
+        return (video1, video2)
+
+    def apply(self, videos: tuple[Video, Video]) -> Video:
+        videos = self._validate(*videos)
+        axis = 1 if self.mode == "vertical" else 2
+        new_frames = np.concatenate((videos[0].frames, videos[1].frames), axis=axis)
+        new_audio = videos[0].audio.overlay(videos[1].audio)
+        return Video(frames=new_frames, fps=videos[0].fps, audio=new_audio)

videopython/{utils/text.py → base/text/overlay.py}

@@ -1,10 +1,21 @@
+"""
+Beware, the code below was heavily "vibe-coded".
+
+The main purpose of this file are 2 classes:
+1. `ImageText` class for creating RGBA image with rendered subtitles
+2. `TranscriptionOverlay` class, which takes the `Transcription` and `Video` objects and overlays subtitles on `Video`.
+"""
+
 from enum import Enum
 from typing import TypeAlias, Union
 
 import numpy as np
 from PIL import Image, ImageDraw, ImageFont
+from tqdm import tqdm
 
 from videopython.base.exceptions import OutOfBoundsError
+from videopython.base.text.transcription import Transcription, TranscriptionSegment
+from videopython.base.video import Video
 
 # Type aliases for clarity
 MarginType: TypeAlias = Union[int, tuple[int, int, int, int]]
@@ -319,6 +330,7 @@ class ImageText:
         font_filename: str,
         xy: PositionType,
         font_size: int | None = 11,
+        font_border_size: int = 0,
         color: RGBColor = (0, 0, 0),
         max_width: int | None = None,
         max_height: int | None = None,
@@ -333,6 +345,7 @@ class ImageText:
             font_filename: Path to the font file
             xy: Position (x,y) either as absolute pixels (int) or relative to frame (float 0-1)
             font_size: Size of the font in points, or None to auto-calculate
+            font_border_size: Size of border around text in pixels (0 for no border)
             color: RGB color of the text
             max_width: Maximum width for auto font sizing
             max_height: Maximum height for auto font sizing
@@ -355,6 +368,9 @@ class ImageText:
         if font_size is not None and font_size <= 0:
             raise ValueError("Font size must be positive")
 
+        if font_border_size < 0:
+            raise ValueError("Font border size cannot be negative")
+
         if font_size is None and (max_width is None or max_height is None):
             raise ValueError("Must set either `font_size`, or both `max_width` and `max_height`!")
         elif font_size is None:
@@ -371,6 +387,15 @@ class ImageText:
         if x < 0 or y < 0 or x + text_dimensions[0] > self.image_size[0] or y + text_dimensions[1] > self.image_size[1]:
             raise OutOfBoundsError(f"Text with size {text_dimensions} at position ({x}, {y}) is out of bounds!")
 
+        # Draw border if requested
+        if font_border_size > 0:
+            # Draw text border by drawing text in multiple positions around the main text
+            for border_x in range(-font_border_size, font_border_size + 1):
+                for border_y in range(-font_border_size, font_border_size + 1):
+                    if border_x != 0 or border_y != 0:  # Skip the center position
+                        self._draw.text((x + border_x, y + border_y), text, font=font, fill=(0, 0, 0))
+
+        # Draw the main text on top
         self._draw.text((x, y), text, font=font, fill=color)
         return text_dimensions
 
@@ -423,6 +448,46 @@ class ImageText:
         except Exception as e:
             raise ValueError(f"Error measuring text: {str(e)}")
 
+    def _get_font_baseline_offset(
+        self, base_font_filename: str, base_font_size: int, highlight_font_filename: str, highlight_font_size: int
+    ) -> int:
+        """
+        Calculate the vertical offset needed to align baselines of different fonts and sizes.
+
+        Args:
+            base_font_filename: Path to the base font file
+            base_font_size: Font size of normal text
+            highlight_font_filename: Path to the highlight font file
+            highlight_font_size: Font size of highlighted text
+
+        Returns:
+            Vertical offset in pixels to align highlighted text baseline with normal text baseline
+        """
+        base_font = self._get_font(base_font_filename, base_font_size)
+        highlight_font = self._get_font(highlight_font_filename, highlight_font_size)
+
+        # Use a reference character to get baseline metrics
+        # We use 'A' as it's a good reference for ascender height
+        ref_char = "A"
+
+        # Get bounding boxes for the reference character
+        base_bbox = base_font.getbbox(ref_char)
+        highlight_bbox = highlight_font.getbbox(ref_char)
+
+        if base_bbox is None or highlight_bbox is None:
+            return 0  # Fallback if bbox calculation fails
+
+        # The baseline offset is the difference in the top of the bounding box
+        # since getbbox returns (left, top, right, bottom) where top is negative for ascenders
+        base_ascent = -base_bbox[1]  # Distance from baseline to top of character
+        highlight_ascent = -highlight_bbox[1]  # Distance from baseline to top of character
+
+        # Calculate the offset needed to align baselines
+        # If highlighted text has a larger ascent, we need to move it down
+        baseline_offset = highlight_ascent - base_ascent
+
+        return baseline_offset
+
     def _split_lines_by_width(
         self,
         text: str,
@@ -499,12 +564,18 @@ class ImageText:
         xy: PositionType,
         box_width: Union[int, float] | None = None,
         font_size: int = 11,
+        font_border_size: int = 0,
         text_color: RGBColor = (0, 0, 0),
         background_color: RGBAColor | None = None,
         background_padding: int = 0,
         place: TextAlign = TextAlign.LEFT,
         anchor: AnchorPoint = AnchorPoint.TOP_LEFT,
         margin: MarginType = 0,
+        words: list[str] | None = None,
+        highlight_word_index: int | None = None,
+        highlight_color: RGBColor | None = None,
+        highlight_size_multiplier: float = 1.5,
+        highlight_bold_font: str | None = None,
     ) -> tuple[int, int]:
         """
         Write text in a box with advanced positioning and alignment options.
@@ -515,12 +586,18 @@ class ImageText:
             xy: Position (x,y) either as absolute pixels (int) or relative to frame (float 0-1)
             box_width: Width of the box in pixels (int) or relative to frame width (float 0-1)
             font_size: Font size in points
+            font_border_size: Size of border around text in pixels (0 for no border)
             text_color: RGB color of the text
             background_color: If set, adds background color to the text box. Expects RGBA values.
             background_padding: Number of padding pixels to add when adding text background color
             place: Text alignment within the box (TextAlign.LEFT, TextAlign.RIGHT, TextAlign.CENTER)
             anchor: Which part of the text box to anchor at the position
             margin: Margin in pixels (single value or [top, right, bottom, left])
+            words: All words occuring in text, helpful for highlighting.
+            highlight_word_index: Index of word to highlight (0-based, None to disable highlighting)
+            highlight_color: RGB color for the highlighted word (defaults to text_color if None)
+            highlight_size_multiplier: Font size multiplier for highlighted word
+            highlight_bold_font: Path to bold font file for highlighted word (defaults to font_filename if None)
 
         Returns:
             Coordinates of the lower-right corner of the written text box (x, y)
@@ -541,6 +618,25 @@ class ImageText:
         if background_padding < 0:
             raise ValueError("Background padding cannot be negative")
 
+        if font_border_size < 0:
+            raise ValueError("Font border size cannot be negative")
+
+        # Validate highlighting parameters
+        if highlight_word_index is not None:
+            if not words:
+                words = text.split()
+            if highlight_word_index < 0 or highlight_word_index >= len(words):
+                raise ValueError(
+                    f"highlight_word_index {highlight_word_index} out of range for text with {len(words)} words"
+                )
+
+        if highlight_size_multiplier <= 0:
+            raise ValueError("highlight_size_multiplier must be positive")
+
+        # Set default highlight color if not provided
+        if highlight_word_index is not None and highlight_color is None:
+            highlight_color = text_color
+
         # Process margins to determine available area
         margin_top, margin_right, margin_bottom, margin_left = self._process_margin(margin)
         available_width = self.image_size[0] - margin_left - margin_right
@@ -590,6 +686,7 @@ class ImageText:
 
         # Write lines
         current_text_height = y_pos
+        word_index_offset = 0  # Track global word index across lines
         for line in lines:
             line_dimensions = self.get_text_dimensions(font_filename, font_size, line)
 
@@ -604,14 +701,49 @@ class ImageText:
                 valid_places = [e.value for e in TextAlign]
                 raise ValueError(f"Place '{place}' is not supported. Must be one of: {', '.join(valid_places)}")
 
-            # Write the line
-            self.write_text(
-                text=line,
-                font_filename=font_filename,
-                xy=(x_left, current_text_height),
-                font_size=font_size,
-                color=text_color,
-            )
+            # Check if highlighting is needed for this line
+            if highlight_word_index is not None:
+                line_words = line.split()
+                line_start_word_index = word_index_offset
+                line_end_word_index = word_index_offset + len(line_words) - 1
+
+                # Check if the highlighted word is in this line
+                if line_start_word_index <= highlight_word_index <= line_end_word_index:
+                    self._write_line_with_highlight(
+                        line=line,
+                        font_filename=font_filename,
+                        font_size=font_size,
+                        font_border_size=font_border_size,
+                        text_color=text_color,
+                        highlight_color=highlight_color or (255, 255, 255),
+                        highlight_size_multiplier=highlight_size_multiplier,
+                        highlight_word_local_index=highlight_word_index - line_start_word_index,
+                        highlight_bold_font=highlight_bold_font,
+                        x_left=int(x_left),
+                        y_top=int(current_text_height),
+                    )
+                else:
+                    # Write normal line without highlighting
+                    self.write_text(
+                        text=line,
+                        font_filename=font_filename,
+                        xy=(x_left, current_text_height),
+                        font_size=font_size,
+                        font_border_size=font_border_size,
+                        color=text_color,
+                    )
+
+                word_index_offset += len(line_words)
+            else:
+                # Write normal line without highlighting
+                self.write_text(
+                    text=line,
+                    font_filename=font_filename,
+                    xy=(x_left, current_text_height),
+                    font_size=font_size,
+                    font_border_size=font_border_size,
+                    color=text_color,
+                )
 
             # Increment vertical position for next line
             current_text_height += line_dimensions[1]
@@ -690,6 +822,88 @@ class ImageText:
 
         return (int(x_pos + box_width), int(current_text_height))
 
+    def _write_line_with_highlight(
+        self,
+        line: str,
+        font_filename: str,
+        font_size: int,
+        font_border_size: int,
+        text_color: RGBColor,
+        highlight_color: RGBColor,
+        highlight_size_multiplier: float,
+        highlight_word_local_index: int,
+        highlight_bold_font: str | None,
+        x_left: int,
+        y_top: int,
+    ) -> None:
+        """
+        Write a line of text with one word highlighted using word-by-word rendering with baseline alignment.
+
+        Args:
+            line: The text line to render
+            font_filename: Path to the font file
+            font_size: Base font size in points
+            font_border_size: Size of border around text in pixels (0 for no border)
+            text_color: RGB color for normal text
+            highlight_color: RGB color for highlighted word
+            highlight_size_multiplier: Font size multiplier for highlighted word
+            highlight_word_local_index: Index of word to highlight within this line (0-based)
+            highlight_bold_font: Path to bold font file for highlighted word (defaults to font_filename if None)
+            x_left: Left x position for the line
+            y_top: Top y position for the line
+        """
+        # Split line into words
+        words = line.split()
+        if highlight_word_local_index >= len(words):
+            return  # Safety check
+
+        # Calculate highlighted font size and determine font files
+        highlight_font_size = int(font_size * highlight_size_multiplier)
+        highlight_font_file = highlight_bold_font if highlight_bold_font is not None else font_filename
+
+        # Calculate baseline offset for highlighted words (using the appropriate font files)
+        baseline_offset = self._get_font_baseline_offset(
+            font_filename, font_size, highlight_font_file, highlight_font_size
+        )
+
+        # Render words one by one with proper spacing
+        current_x = x_left
+
+        for i, word in enumerate(words):
+            # Determine if this is the highlighted word
+            is_highlighted = i == highlight_word_local_index
+
+            # Choose font file, size, and color based on highlighting
+            word_font_file = highlight_font_file if is_highlighted else font_filename
+            word_font_size = highlight_font_size if is_highlighted else font_size
+            word_color = highlight_color if is_highlighted else text_color
+
+            # Calculate y position with baseline alignment
+            word_y = y_top
+            if is_highlighted:
+                word_y += baseline_offset
+
+            # Render the word
+            self.write_text(
+                text=word,
+                font_filename=word_font_file,
+                xy=(current_x, word_y),
+                font_size=word_font_size,
+                font_border_size=font_border_size,
+                color=word_color,
+            )
+
+            # Calculate the width of this word for spacing
+            word_width = self.get_text_dimensions(word_font_file, word_font_size, word)[0]
+
+            # Update current_x for next word (add word width plus space)
+            current_x += word_width
+
+            # Add space between words (except after the last word)
+            if i < len(words) - 1:
+                space_width = self.get_text_dimensions(font_filename, font_size, " ")[0]
+                current_x += space_width
+
     def _find_smallest_bounding_rect(self, mask: np.ndarray) -> tuple[int, int, int, int]:
         """
         Find the smallest bounding rectangle containing non-zero values in the mask.
@@ -725,3 +939,164 @@ class ImageText:
         xmin, xmax = col_indices[[0, -1]]
 
         return xmin, xmax, ymin, ymax
+
+
+class TranscriptionOverlay:
+    def __init__(
+        self,
+        font_filename: str,
+        font_size: int = 40,
+        font_border_size: int = 2,
+        text_color: RGBColor = (255, 235, 59),
+        background_color: RGBAColor | None = (0, 0, 0, 100),
+        background_padding: int = 15,
+        position: PositionType = (0.5, 0.7),
+        box_width: Union[int, float] = 0.6,
+        text_align: TextAlign = TextAlign.CENTER,
+        anchor: AnchorPoint = AnchorPoint.CENTER,
+        margin: MarginType = 20,
+        highlight_color: RGBColor = (76, 175, 80),
+        highlight_size_multiplier: float = 1.2,
+        highlight_bold_font: str | None = None,
+    ):
+        """
+        Initialize TranscriptionOverlay effect.
+
+        Args:
+            font_filename: Path to font file for text rendering
+            font_size: Base font size for text
+            text_color: RGB color for normal text
+            font_border_size: Size of border around text in pixels (0 for no border)
+            background_color: RGBA background color (None for no background)
+            background_padding: Padding around text background
+            position: Position of text box (relative 0-1 or absolute pixels)
+            box_width: Width of text box (relative 0-1 or absolute pixels)
+            text_align: Text alignment within box
+            anchor: Anchor point for text positioning
+            margin: Margin around text box
+            highlight_color: RGB color for highlighted words
+            highlight_size_multiplier: Size multiplier for highlighted words
+            highlight_bold_font: Optional bold font for highlighting
+        """
+        self.font_filename = font_filename
+        self.font_size = font_size
+        self.text_color = text_color
+        self.font_border_size = font_border_size
+        self.background_color = background_color
+        self.background_padding = background_padding
+        self.position = position
+        self.box_width = box_width
+        self.text_align = text_align
+        self.anchor = anchor
+        self.margin = margin
+        self.highlight_color = highlight_color
+        self.highlight_size_multiplier = highlight_size_multiplier
+        self.highlight_bold_font = highlight_bold_font
+
+        # Cache for text overlays to avoid regenerating identical frames
+        self._overlay_cache: dict[tuple[str, int | None], np.ndarray] = {}
+
+    def _get_active_segment(self, transcription: Transcription, timestamp: float) -> TranscriptionSegment | None:
+        """Get the transcription segment active at the given timestamp."""
+        for segment in transcription.segments:
+            if segment.start <= timestamp <= segment.end:
+                return segment
+        return None
+
+    def _get_active_word_index(self, segment: TranscriptionSegment, timestamp: float) -> int | None:
+        """Get the index of the word being spoken at the given timestamp within a segment."""
+        for i, word in enumerate(segment.words):
+            if word.start <= timestamp <= word.end:
+                return i
+        return None
+
+    def _create_text_overlay(
+        self, video_shape: tuple[int, int, int], segment: TranscriptionSegment, highlight_word_index: int | None
+    ) -> np.ndarray:
+        """Create a text overlay image for the given segment and highlight."""
+        # Use video frame dimensions for overlay
+        height, width = video_shape[:2]
+
+        # Create cache key based on segment text and highlight
+        cache_key = (segment.text, highlight_word_index)
+        if cache_key in self._overlay_cache:
+            return self._overlay_cache[cache_key]
+
+        # Create ImageText with video dimensions
+        img_text = ImageText(image_size=(width, height), background=(0, 0, 0, 0))
+
+        # Write text with highlighting
+        img_text.write_text_box(
+            text=segment.text,
+            font_filename=self.font_filename,
+            xy=self.position,
+            box_width=self.box_width,
+            font_size=self.font_size,
+            font_border_size=self.font_border_size,
+            text_color=self.text_color,
+            background_color=self.background_color,
+            background_padding=self.background_padding,
+            place=self.text_align,
+            anchor=self.anchor,
+            margin=self.margin,
+            words=[w.word for w in segment.words],
+            highlight_word_index=highlight_word_index,
+            highlight_color=self.highlight_color,
+            highlight_size_multiplier=self.highlight_size_multiplier,
+            highlight_bold_font=self.highlight_bold_font,
+        )
+
+        overlay_image = img_text.img_array
+
+        # Cache the overlay
+        self._overlay_cache[cache_key] = overlay_image
+
+        return overlay_image
+
+    def apply(self, video: Video, transcription: Transcription) -> Video:
+        """Apply transcription overlay to video frames."""
+        print("Applying transcription overlay...")
+
+        new_frames = []
+
+        for frame_idx, frame in enumerate(tqdm(video.frames)):
+            # Calculate timestamp for this frame
+            timestamp = frame_idx / video.fps
+
+            # Get active segment at this timestamp
+            active_segment = self._get_active_segment(transcription, timestamp)
+
+            if active_segment is None:
+                # No active transcription, keep original frame
+                new_frames.append(frame)
+                continue
+
+            # Get active word index for highlighting
+            highlight_word_index = self._get_active_word_index(active_segment, timestamp)
+
+            # Create text overlay
+            text_overlay = self._create_text_overlay(video.frame_shape, active_segment, highlight_word_index)
+
+            # Apply overlay to frame
+            overlaid_frame = self._apply_overlay_to_frame(frame, text_overlay)
+            new_frames.append(overlaid_frame)
+
+        # Create new video with overlaid frames
+        new_video = Video.from_frames(np.array(new_frames), fps=video.fps)
+        new_video.audio = video.audio  # Preserve audio
+
+        return new_video
+
+    def _apply_overlay_to_frame(self, frame: np.ndarray, overlay: np.ndarray) -> np.ndarray:
+        """Apply a text overlay to a single frame."""
+
+        # Convert frame to PIL Image
+        frame_pil = Image.fromarray(frame)
+
+        # Convert overlay to PIL Image
+        overlay_pil = Image.fromarray(overlay)
+
+        # Paste overlay onto frame using alpha channel
+        frame_pil.paste(overlay_pil, (0, 0), overlay_pil)
+
+        return np.array(frame_pil)