yta-video-opengl 0.0.14__py3-none-any.whl → 0.0.16__py3-none-any.whl

yta_video_opengl/complete/blend.py

@@ -0,0 +1,83 @@
+"""
+TODO: I don't like the name nor the
+location of this file, but it is here
+to encapsulate some functionality 
+related to combining video frames.
+"""
+import numpy as np
+
+
+def blend_alpha(
+    bottom,
+    top,
+    alpha = 0.5
+):
+    return (alpha * top + (1 - alpha) * bottom).astype(np.uint8)
+
+def blend_add(
+    bottom,
+    top
+):
+    """
+    Aclara la imagen combinada, como si superpusieras dos proyectores de luz.
+    """
+    return np.clip(bottom.astype(np.int16) + top.astype(np.int16), 0, 255).astype(np.uint8)
+
+def blend_multiply(
+    bottom,
+    top
+):
+    """
+    Oscurece, como proyectar dos transparencias juntas.
+    """
+    return ((bottom.astype(np.float32) * top.astype(np.float32)) / 255).astype(np.uint8)
+
+def blend_screen(
+    bottom,
+    top
+):
+    """
+    Hace lo contrario a Multiply, aclara la imagen.
+    """
+    return (255 - ((255 - bottom.astype(np.float32)) * (255 - top.astype(np.float32)) / 255)).astype(np.uint8)
+
+def blend_overlay(
+    bottom,
+    top
+):
+    """
+    Mezcla entre Multiply y Screen según el brillo de cada píxel.
+    """
+    b = bottom.astype(np.float32) / 255
+    t = top.astype(np.float32) / 255
+    mask = b < 0.5
+    result = np.zeros_like(b)
+    result[mask] = 2 * b[mask] * t[mask]
+    result[~mask] = 1 - 2 * (1 - b[~mask]) * (1 - t[~mask])
+    return (result * 255).astype(np.uint8)
+
+def blend_difference(
+    bottom,
+    top
+):
+    """
+    Resalta las diferencias entre los dos frames.
+    """
+    return np.abs(bottom.astype(np.int16) - top.astype(np.int16)).astype(np.uint8)
+
+# TODO: This one needs a mask, thats why
+# it is commented
+# def blend_mask(
+#     bottom,
+#     top,
+#     mask
+# ):
+#     """
+#     En lugar de un alpha fijo, puedes pasar una máscara (por ejemplo, un degradado o un canal alfa real)
+
+#     mask: array float32 entre 0 y 1, mismo tamaño que frame.
+#     """
+#     return (mask * top + (1 - mask) * bottom).astype(np.uint8)
+
+
+

yta_video_opengl/complete/timeline.py

@@ -14,9 +14,14 @@ from yta_video_opengl.complete.track import Track
 from yta_video_opengl.video import Video
 from yta_video_opengl.t import get_ts, fps_to_time_base, T
 from yta_validation.parameter import ParameterValidator
+from av.video.frame import VideoFrame
+from av.audio.frame import AudioFrame
+from av.audio.resampler import AudioResampler
 from quicktions import Fraction
 from typing import Union
 
+import numpy as np
+
 
 class Timeline:
     """
@@ -129,8 +134,25 @@ class Timeline:
         # other frame in other track, or to know if
         # I want them as transparent or something
 
-        # TODO: Combinate them, I send first by now
-        return next(frames)
+        # TODO: This is just a test function
+        from yta_video_opengl.complete.blend import blend_add
+
+        # TODO: Combinate frames, we force them to
+        # rgb24 to obtain them with the same shape,
+        # but maybe we have to change this because
+        # we also need to handle alphas
+        output_frame = next(frames).to_ndarray(format = 'rgb24')
+        for frame in frames:
+            # Combine them
+            # TODO: We need to ignore the frames that
+            # are just empty black frames and use them
+            # not in the combination process
+            output_frame = blend_add(output_frame, frame.to_ndarray(format = 'rgb24'))
+
+        # TODO: How to build this VideoFrame correctly
+        # and what about the 'format' (?)
+        # We don't handle pts here, just the image
+        return VideoFrame.from_ndarray(output_frame, format = 'rgb24')
     
     def get_audio_frames_at(
         self,
@@ -138,20 +160,151 @@ class Timeline:
     ):
         # TODO: What if the different audio streams
         # have also different fps (?)
-        frames = []
+        audio_frames = []
         for track in self.tracks:
             # TODO: Make this work properly
-            audio_frames = track.get_audio_frames_at(t)
+            audio_frames.append(list(track.get_audio_frames_at(t)))
 
             # TODO: Combine them
-            if audio_frames is not None:
-                frames = audio_frames
-                break
+            # TODO: We need to ignore the frames that
+            # are just empty black frames and use them
+            # not in the combination process
+
+        def mix_audio_frames_by_index(
+            tracks_frames,
+            layout = 'stereo'
+        ):
+            """
+            Combine all the columns of the given
+            matrix of audio frames 'tracks_frames'.
+            The rows are the different tracks and
+            the columns are the frame at that 't'
+            moment of each of those tracks.
+
+            The 'tracks_frames' matrix needs to be
+            pre-processed to have only 1 single
+            frame to combine, so we concatenate
+            all the frames if more than 1 per 
+            column.
+            """
+            # TODO: Please, improve and clean all this
+            # code is so sh*tty, and make utils to
+            # combine and those things, not here...
+            # Also the formats, make them dynamic and
+            # based on the output that is defined here
+            # in the Timeline class.
+            mixed_frames = []
+
+            # Iterate by columns (each row is a track)
+            for frames_at_index in zip(*tracks_frames):
+                arrays = []
+                for f in frames_at_index:
+                    # Resample to output expected values
+                    # TODO: This must be dynamic depending
+                    # on the track values
+                    resampler = AudioResampler(format = 'fltp', layout = 'stereo', rate = self.audio_fps)
+                    arr = resampler.resample(f)
+
+                    arr = f.to_ndarray()
+
+                    # TODO: This below must change depending
+                    # on the expected output, for us and now
+                    # it is float32, fltp, stereo, 44_100
+                    # Same format
+                    if arr.dtype == np.int16:
+                        arr = arr.astype(np.float32) / 32768.0
+
+                    # Same layout (number of channels)
+                    if arr.shape[0] == 1:
+                        return np.repeat(arr, 2, axis = 0)
+                    # elif arr.dtype == np.float32:
+                    #     # Ya está en [-1,1], no lo toques
+                    #     pass
+
+                    arrays.append(arr)
+
+                # Alinear longitudes
+                max_len = max(a.shape[1] for a in arrays)
+                stacked = []
+                for a in arrays:
+                    buf = np.zeros((a.shape[0], max_len), dtype = np.float32)
+                    buf[:, :a.shape[1]] = a
+                    stacked.append(buf)
+
+                # Mezcla
+                mix = np.sum(stacked, axis = 0) / len(stacked)
+                #mix = np.sum(stacked, axis = 0) 
+
+                # Limitar al rango [-1,1]
+                mix = np.clip(mix, -1.0, 1.0)
+
+                # Crear frame de salida
+                # TODO: What about the 'format' if they
+                # are all different (?)
+                out = AudioFrame.from_ndarray(mix, format = 'fltp', layout = layout)
+                out.sample_rate = self.audio_fps
+                # TODO: This will be written later when
+                # encoding
+                # out.pts = frames_at_index[0].pts
+                # out.time_base = frames_at_index[0].time_base
+
+                print(mix.min(), mix.max())
+
+                mixed_frames.append(out)
+
+            return mixed_frames
+        
+        def combine_audio_frames(frames):
+            """
+            Combina varios AudioFrames consecutivos en uno solo.
+            - Convierte a float32
+            - Concatena muestras a lo largo del tiempo
+            - Devuelve un AudioFrame nuevo
+            """
+            if not frames:
+                # TODO: This should not happen
+                return None
             
-        #from yta_video_opengl.utils import get_silent_audio_frame
-        #make_silent_audio_frame()
-        for frame in frames:
-            yield frame
+            if len(frames) == 1:
+                return frames
+
+            # Verificamos consistencia básica
+            sample_rate = frames[0].sample_rate
+            layout = frames[0].layout.name
+            channels = frames[0].layout.channels
+
+            arrays = []
+            for f in frames:
+                if f.sample_rate != sample_rate or f.layout.name != layout:
+                    raise ValueError("Los frames deben tener mismo sample_rate y layout")
+
+                # arr = f.to_ndarray()  # (channels, samples)
+                # if arr.dtype == np.int16:
+                #     arr = arr.astype(np.float32) / 32768.0
+                # elif arr.dtype != np.float32:
+                #     arr = arr.astype(np.float32)
+
+                arrays.append(f.to_ndarray())
+
+            # Concatenamos por eje de samples
+            combined = np.concatenate(arrays, axis = 1)
+
+            # Creamos un frame nuevo
+            out = AudioFrame.from_ndarray(combined, format = frames[0].format, layout = layout)
+            out.sample_rate = sample_rate
+
+            return [out]
+
+        # We need only 1 single audio frame per column
+        collapsed = []
+        for frames in audio_frames:
+            collapsed.append(combine_audio_frames(frames))
+
+        # Now, mix column by column (track by track)
+        frames = mix_audio_frames_by_index(collapsed)
+
+        for audio_frame in frames:
+            yield audio_frame
             
     def render(
         self,
@@ -168,10 +321,8 @@ class Timeline:
         project will be rendered.
         """
         ParameterValidator.validate_mandatory_string('filename', filename, do_accept_empty = False)
-        # TODO: We need to accept Fraction as number
-        #ParameterValidator.validate_mandatory_positive_number('start', start, do_include_zero = True)
-        # TODO: We need to accept Fraction as number
-        #ParameterValidator.validate_positive_number('end', end, do_include_zero = False)
+        ParameterValidator.validate_mandatory_positive_number('start', start, do_include_zero = True)
+        ParameterValidator.validate_positive_number('end', end, do_include_zero = False)
 
         # TODO: Limitate 'end' a bit...
         end = (

yta_video_opengl/complete/track.py

@@ -39,10 +39,8 @@ class _Part:
         end: Union[int, float, Fraction],
         video: Union[VideoOnTrack, None] = None
     ):
-        # TODO: We need to accept Fraction as number
-        # ParameterValidator.validate_mandatory_positive_number('start', start, do_include_zero = True)
-        # TODO: We need to accept Fraction as number
-        # ParameterValidator.validate_mandatory_positive_number('end', end, do_include_zero = False)
+        ParameterValidator.validate_mandatory_positive_number('start', start, do_include_zero = True)
+        ParameterValidator.validate_mandatory_positive_number('end', end, do_include_zero = False)
         ParameterValidator.validate_instance_of('video', video, VideoOnTrack)
 
         self._track: Track = track
@@ -200,6 +198,17 @@ class Track:
                 for video in self.videos
             )
         )
+    
+    @property
+    def videos(
+        self
+    ) -> list[VideoOnTrack]:
+        """
+        The list of videos we have in the track
+        but ordered using the 'start' attribute
+        from first to last.
+        """
+        return sorted(self._videos, key = lambda video: video.start)
 
     def __init__(
         self,
@@ -212,7 +221,7 @@ class Track:
         # TODO: Where does it come from (?)
         audio_samples_per_frame: int
     ):
-        self.videos: list[VideoOnTrack] = []
+        self._videos: list[VideoOnTrack] = []
         """
         The list of 'VideoOnTrack' instances that
         must play on this track.
@@ -354,7 +363,7 @@ class Track:
         else:
             t = self.end
         
-        self.videos.append(VideoOnTrack(
+        self._videos.append(VideoOnTrack(
             video,
             t
         ))

yta_video_opengl/complete/video_on_track.py

@@ -46,10 +46,7 @@ class VideoOnTrack:
         start: Union[int, float, Fraction] = 0.0
     ):
         ParameterValidator.validate_mandatory_instance_of('video', video, Video)
-        # TODO: Now we need to accept 'Fraction',
-        # from 'fractions' or 'quicktions', as a
-        # number
-        #ParameterValidator.validate_mandatory_positive_number('start', start, do_include_zero = True)
+        ParameterValidator.validate_mandatory_positive_number('start', start, do_include_zero = True)
 
         self.video: Video = video
         """

yta_video_opengl/reader/__init__.py

@@ -2,7 +2,8 @@
 A video reader using the PyAv (av) library
 that, using ffmpeg, detects the video.
 """
-from yta_video_opengl.reader.cache import VideoFrameCache
+from yta_video_opengl.reader.cache.video import VideoFrameCache
+from yta_video_opengl.reader.cache.audio import AudioFrameCache
 from yta_video_opengl.utils import iterate_stream_frames_demuxing
 from yta_video_opengl.t import T
 from yta_validation import PythonValidator
@@ -435,7 +436,7 @@ class VideoReader:
         The video frame cache system to optimize
         the way we access to the frames.
         """
-        self.audio_cache: VideoFrameCache = None
+        self.audio_cache: AudioFrameCache = None
         """
         The audio frame cache system to optimize
         the way we access to the frames.
@@ -490,7 +491,7 @@ class VideoReader:
                 raise Exception(f'No video nor audio stream found in the "{self.filename}" file.')
 
             self.video_cache = VideoFrameCache(self.container, self.video_stream)
-            self.audio_cache = VideoFrameCache(self.container, self.audio_stream)
+            self.audio_cache = AudioFrameCache(self.container, self.audio_stream)
 
     def seek(
         self,
@@ -620,16 +621,28 @@ class VideoReader:
         ):
             yield frame
 
-    def get_frame_from_t(
+    def get_frame(
         self,
         t: Union[int, float, Fraction]
-    ) -> 'VideoFrame':
+    ) -> VideoFrame:
         """
-        Get the video frame with the given 't' time
-        moment, using the video cache system.
+        Get the video frame that is in the 't' time
+        moment provided.
         """
-        return self.video_cache.get_video_frame(t)
-    
+        return self.video_cache.get_frame(t)
+
+    def get_frames(
+        self,
+        start: Union[int, float, Fraction] = 0.0,
+        end: Union[int, float, Fraction, None] = None
+    ):
+        """
+        Iterator to get the video frames in between
+        the provided 'start' and 'end' time moments.
+        """
+        for frame in self.video_cache.get_frames(start, end):
+            yield frame
+
     def get_audio_frame_from_t(
         self,
         t: Union[int, float, Fraction]
@@ -638,7 +651,7 @@ class VideoReader:
         Get the audio frame with the given 't' time
         moment, using the audio cache system.
         """
-        return self.audio_cache.get_audio_frame_from_t(t)
+        return self.audio_cache.get_frame(t)
     
     def get_audio_frames_from_t(
         self,
@@ -659,19 +672,7 @@ class VideoReader:
         # We want all the audios that must be played
         # during the video frame that starts in the
         # 't' time moment
-        for frame in self.audio_cache.get_audio_frames(t.truncated, t.next(1).truncated):
-            yield frame
-    
-    def get_frames(
-        self,
-        start: Union[int, float, Fraction] = 0.0,
-        end: Union[int, float, Fraction, None] = None
-    ):
-        """
-        Iterator to get the video frames in between
-        the provided 'start' and 'end' time moments.
-        """
-        for frame in self.video_cache.get_frames(start, end):
+        for frame in self.get_audio_frames(t.truncated, t.next(1).truncated):
             yield frame
 
     def get_audio_frames(
@@ -683,7 +684,7 @@ class VideoReader:
         Iterator to get the audio frames in between
         the provided 'start' and 'end' time moments.
         """
-        for frame in self.audio_cache.get_audio_frames(start, end):
+        for frame in self.audio_cache.get_frames(start, end):
             yield frame
     
     def close(