yta-video-opengl 0.0.16__py3-none-any.whl → 0.0.18__py3-none-any.whl

yta_video_opengl/complete/timeline.py

@@ -13,11 +13,14 @@ video written).
 from yta_video_opengl.complete.track import Track
 from yta_video_opengl.video import Video
 from yta_video_opengl.t import get_ts, fps_to_time_base, T
+from yta_video_opengl.complete.frame_wrapper import VideoFrameWrapped, AudioFrameWrapped
+from yta_video_opengl.complete.frame_combinator import AudioFrameCombinator
 from yta_validation.parameter import ParameterValidator
+from yta_validation import PythonValidator
 from av.video.frame import VideoFrame
 from av.audio.frame import AudioFrame
-from av.audio.resampler import AudioResampler
 from quicktions import Fraction
+from functools import reduce
 from typing import Union
 
 import numpy as np
@@ -51,7 +54,10 @@ class Timeline:
         audio_fps: Union[int, Fraction] = 44_100.0, # 48_000.0 for aac
         # TODO: I don't like this name
         # TODO: Where does this come from (?)
-        audio_samples_per_frame: int = 1024
+        audio_samples_per_frame: int = 1024,
+        video_codec: str = 'h264',
+        video_pixel_format: str = 'yuv420p',
+        audio_codec: str = 'aac'
     ):
         # TODO: By now we are using just two video
         # tracks to test the composition
@@ -79,10 +85,36 @@ class Timeline:
         """
         All the video tracks we are handling.
         """
-        # TODO: Handle the other properties
-        self.size = size
-        self.fps = fps
-        self.audio_fps = audio_fps
+        
+        self.size: tuple[int, int] = size
+        """
+        The size that the final video must have.
+        """
+        self.fps: Union[int, float, Fraction] = fps
+        """
+        The fps of the output video.
+        """
+        self.audio_fps: Union[int, Fraction] = audio_fps
+        """
+        The fps of the output audio.
+        """
+        self.audio_samples_per_frame: int = audio_samples_per_frame
+        """
+        The audio samples each audio frame must
+        have.
+        """
+        self.video_codec: str = video_codec
+        """
+        The video codec for the video exported.
+        """
+        self.video_pixel_format: str = video_pixel_format
+        """
+        The pixel format for the video exported.
+        """
+        self.audio_codec: str = audio_codec
+        """
+        The audio codec for the audio exported.
+        """
 
     # TODO: Create 'add_track' method, but by now
     # we hare handling only one
@@ -122,7 +154,7 @@ class Timeline:
         Get all the frames that are played at the
         't' time provided, but combined in one.
         """
-        frames = (
+        frames = list(
             track.get_frame_at(t)
             for track in self.tracks
         )
@@ -134,20 +166,45 @@ class Timeline:
         # other frame in other track, or to know if
         # I want them as transparent or something
 
-        # TODO: This is just a test function
-        from yta_video_opengl.complete.blend import blend_add
-
         # TODO: Combinate frames, we force them to
         # rgb24 to obtain them with the same shape,
         # but maybe we have to change this because
         # we also need to handle alphas
-        output_frame = next(frames).to_ndarray(format = 'rgb24')
+
+        # TODO: We need to ignore the ones that are
+        # tagged with 
+        # .metadata['is_from_empty_part'] = 'True'
+
+        """
+        1. Only empty frames
+            -> Black background, keep one
+        2. Empty frames but other frames:
+            -> Skip all empty frames and apply
+               track orders
+        """
+
+        output_frame = frames[0]._frame.to_ndarray(format = 'rgb24')
+
         for frame in frames:
-            # Combine them
-            # TODO: We need to ignore the frames that
-            # are just empty black frames and use them
-            # not in the combination process
-            output_frame = blend_add(output_frame, frame.to_ndarray(format = 'rgb24'))
+            # We just need the first non-empty frame,
+            # that must be from the track with the
+            # bigger priority
+            # TODO: I assume, by now, that the frames
+            # come in order (bigger priority first)
+            if not frame.is_from_empty_part:
+                # TODO: By now I'm just returning the first
+                # one but we will need to check the alpha
+                # layer to combine if possible
+                output_frame = frame._frame.to_ndarray(format = 'rgb24')
+                break
+
+            # # TODO: This code below is to combine the
+            # # frames but merging all of them, that is
+            # # unexpected in a video editor but we have
+            # # the way to do it
+            # from yta_video_opengl.complete.frame_combinator import VideoFrameCombinator
+            # # TODO: What about the 'format' (?)
+            # output_frame = VideoFrameCombinator.blend_add(output_frame, frame.to_ndarray(format = 'rgb24'))
 
         # TODO: How to build this VideoFrame correctly
         # and what about the 'format' (?)
@@ -158,150 +215,57 @@ class Timeline:
         self,
         t: float
     ):
+        audio_frames: list[AudioFrameWrapped] = []
+        """
+        Matrix in which the rows are the different
+        tracks we have, and the column includes all
+        the audio frames for this 't' time moment
+        for the track of that row. We can have more
+        than one frame per column per row (track)
+        but we need a single frame to combine all
+        the tracks.
+        """
         # TODO: What if the different audio streams
         # have also different fps (?)
-        audio_frames = []
         for track in self.tracks:
             # TODO: Make this work properly
             audio_frames.append(list(track.get_audio_frames_at(t)))
-
-            # TODO: Combine them
             # TODO: We need to ignore the frames that
             # are just empty black frames and use them
             # not in the combination process
 
-        def mix_audio_frames_by_index(
-            tracks_frames,
-            layout = 'stereo'
-        ):
-            """
-            Combine all the columns of the given
-            matrix of audio frames 'tracks_frames'.
-            The rows are the different tracks and
-            the columns are the frame at that 't'
-            moment of each of those tracks.
-
-            The 'tracks_frames' matrix needs to be
-            pre-processed to have only 1 single
-            frame to combine, so we concatenate
-            all the frames if more than 1 per 
-            column.
-            """
-            # TODO: Please, improve and clean all this
-            # code is so sh*tty, and make utils to
-            # combine and those things, not here...
-            # Also the formats, make them dynamic and
-            # based on the output that is defined here
-            # in the Timeline class.
-            mixed_frames = []
-
-            # Iterate by columns (each row is a track)
-            for frames_at_index in zip(*tracks_frames):
-                arrays = []
-                for f in frames_at_index:
-                    # Resample to output expected values
-                    # TODO: This must be dynamic depending
-                    # on the track values
-                    resampler = AudioResampler(format = 'fltp', layout = 'stereo', rate = self.audio_fps)
-                    arr = resampler.resample(f)
-
-                    arr = f.to_ndarray()
-
-                    # TODO: This below must change depending
-                    # on the expected output, for us and now
-                    # it is float32, fltp, stereo, 44_100
-                    # Same format
-                    if arr.dtype == np.int16:
-                        arr = arr.astype(np.float32) / 32768.0
-
-                    # Same layout (number of channels)
-                    if arr.shape[0] == 1:
-                        return np.repeat(arr, 2, axis = 0)
-                    # elif arr.dtype == np.float32:
-                    #     # Ya está en [-1,1], no lo toques
-                    #     pass
-
-                    arrays.append(arr)
-
-                # Alinear longitudes
-                max_len = max(a.shape[1] for a in arrays)
-                stacked = []
-                for a in arrays:
-                    buf = np.zeros((a.shape[0], max_len), dtype = np.float32)
-                    buf[:, :a.shape[1]] = a
-                    stacked.append(buf)
-
-                # Mezcla
-                mix = np.sum(stacked, axis = 0) / len(stacked)
-                #mix = np.sum(stacked, axis = 0) 
-
-                # Limitar al rango [-1,1]
-                mix = np.clip(mix, -1.0, 1.0)
-
-                # Crear frame de salida
-                # TODO: What about the 'format' if they
-                # are all different (?)
-                out = AudioFrame.from_ndarray(mix, format = 'fltp', layout = layout)
-                out.sample_rate = self.audio_fps
-                # TODO: This will be written later when
-                # encoding
-                # out.pts = frames_at_index[0].pts
-                # out.time_base = frames_at_index[0].time_base
-
-                print(mix.min(), mix.max())
-
-                mixed_frames.append(out)
-
-            return mixed_frames
-        
-        def combine_audio_frames(frames):
-            """
-            Combina varios AudioFrames consecutivos en uno solo.
-            - Convierte a float32
-            - Concatena muestras a lo largo del tiempo
-            - Devuelve un AudioFrame nuevo
-            """
-            if not frames:
-                # TODO: This should not happen
-                return None
-            
-            if len(frames) == 1:
-                return frames
-
-            # Verificamos consistencia básica
-            sample_rate = frames[0].sample_rate
-            layout = frames[0].layout.name
-            channels = frames[0].layout.channels
-
-            arrays = []
-            for f in frames:
-                if f.sample_rate != sample_rate or f.layout.name != layout:
-                    raise ValueError("Los frames deben tener mismo sample_rate y layout")
-
-                # arr = f.to_ndarray()  # (channels, samples)
-                # if arr.dtype == np.int16:
-                #     arr = arr.astype(np.float32) / 32768.0
-                # elif arr.dtype != np.float32:
-                #     arr = arr.astype(np.float32)
-
-                arrays.append(f.to_ndarray())
-
-            # Concatenamos por eje de samples
-            combined = np.concatenate(arrays, axis = 1)
-
-            # Creamos un frame nuevo
-            out = AudioFrame.from_ndarray(combined, format = frames[0].format, layout = layout)
-            out.sample_rate = sample_rate
+        # We need only 1 single audio frame per column
+        collapsed_frames = [
+            concatenate_audio_frames(frames)
+            for frames in audio_frames
+        ]
 
-            return [out]
+        # TODO: What about the lenghts and those
+        # things? They should be ok because they are
+        # based on our output but I'm not completely
+        # sure here..
+
+        # We keep only the non-silent frames because
+        # we will sum them after and keeping them
+        # will change the results.
+        collapsed_frames = [
+            frame._frame
+            for frame in collapsed_frames
+            if not frame.is_from_empty_part
+        ]
 
-        # We need only 1 single audio frame per column
-        collapsed = []
-        for frames in audio_frames:
-            collapsed.append(combine_audio_frames(frames))
+        if len(collapsed_frames) == 0:
+            # If they were all silent, just keep one
+            collapsed_frames = [collapsed_frames[0]._frame]
 
         # Now, mix column by column (track by track)
-        frames = mix_audio_frames_by_index(collapsed)
+        # TODO: I do this to have an iterator, but 
+        # maybe we need more than one single audio
+        # frame because of the size at the original
+        # video or something...
+        frames = [
+            AudioFrameCombinator.sum_tracks_frames(collapsed_frames, self.audio_fps)
+        ]
 
         for audio_frame in frames:
             yield audio_frame
@@ -416,4 +380,84 @@ class Timeline:
 
         writer.mux_video_frame(None)
         writer.mux_audio_frame(None)
-        writer.output.close()
+        writer.output.close()
+
+def _is_empty_part_frame(
+    frame: Union['VideoFrameWrapped', 'AudioFrameWrapped']
+) -> bool:
+    """
+    Flag to indicate if the frame comes from
+    an empty part or not.
+
+    TODO: The 'metadata' is included in our
+    wrapper class, not in VideoFrame or
+    AudioFrame classes. I should be sending
+    the wrapper in all the code, but by now
+    I'm doing it just in specific cases.
+    """
+    return (
+        hasattr(frame, 'metadata') and
+        frame.is_from_empty_part
+    )
+
+# TODO: Refactor and move please
+# TODO: This has to work for AudioFrame
+# also, but I need it working for Wrapped
+def concatenate_audio_frames(
+    frames: list[AudioFrameWrapped]
+) -> AudioFrameWrapped:
+    """
+    Concatenate all the given 'frames' in one
+    single audio frame and return it.
+
+    The audio frames must have the same layout
+    and sample rate.
+    """
+    if not frames:
+        # TODO: This should not happen
+        return None
+    
+    if len(frames) == 1:
+        return frames[0]
+
+    # We need to preserve the metadata
+    is_from_empty_part = all(
+        frame.is_from_empty_part
+        for frame in frames
+    )
+    metadata = reduce(lambda key_values, frame: {**key_values, **frame.metadata}, frames, {})
+    
+    sample_rate = frames[0]._frame.sample_rate
+    layout = frames[0]._frame.layout.name
+
+    arrays = []
+    # TODO: What about 'metadata' (?)
+    for frame in frames:
+        if (
+            frame._frame.sample_rate != sample_rate or
+            frame._frame.layout.name != layout
+        ):
+            raise ValueError("Los frames deben tener mismo sample_rate y layout")
+
+        # arr = frame.to_ndarray()  # (channels, samples)
+        # if arr.dtype == np.int16:
+        #     arr = arr.astype(np.float32) / 32768.0
+        # elif arr.dtype != np.float32:
+        #     arr = arr.astype(np.float32)
+
+        arrays.append(frame._frame.to_ndarray())
+
+    combined = np.concatenate(arrays, axis = 1)
+
+    out = AudioFrame.from_ndarray(
+        array = combined,
+        format = frames[0].format,
+        layout = layout
+    )
+    out.sample_rate = sample_rate
+
+    return AudioFrameWrapped(
+        frame = out,
+        metadata = metadata,
+        is_from_empty_part = is_from_empty_part
+    )

yta_video_opengl/complete/track.py

@@ -1,8 +1,11 @@
 from yta_video_opengl.complete.video_on_track import VideoOnTrack
 from yta_video_opengl.video import Video
 from yta_video_opengl.t import T
-from yta_video_opengl.utils import get_black_background_video_frame, get_silent_audio_frame, audio_frames_and_remainder_per_video_frame
+from yta_video_opengl.complete.frame_wrapper import VideoFrameWrapped
+from yta_video_opengl.utils import audio_frames_and_remainder_per_video_frame
 from yta_video_opengl.t import fps_to_time_base
+from yta_video_opengl.complete.frame_wrapper import AudioFrameWrapped
+from yta_video_opengl.complete.frame_generator import VideoFrameGenerator, AudioFrameGenerator
 from yta_validation.parameter import ParameterValidator
 from quicktions import Fraction
 from typing import Union
@@ -48,6 +51,20 @@ class _Part:
         The instance of the track this part belongs
         to.
         """
+        # TODO: I would like to avoid this 2 instances
+        # here, and I think I've done it with static
+        # properties in other project, but as I don't
+        # remember how and where by now, here it is...
+        self._video_frame_generator: VideoFrameGenerator = VideoFrameGenerator()
+        """
+        Useful internal tool to generate background
+        frames for the empty parts.
+        """
+        self._audio_frame_generator: AudioFrameGenerator = AudioFrameGenerator()
+        """
+        Useful internal tool to generate silent
+        audio frames for the empty parts.
+        """
         self.start: Fraction = Fraction(start)
         """
         The start 't' time moment of the part.
@@ -67,27 +84,38 @@ class _Part:
     def get_frame_at(
         self,
         t: Union[int, float, Fraction]
-    ) -> 'VideoFrame':
+    ) -> 'VideoFrameWrapped':
         """
         Get the frame that must be displayed at 
         the given 't' time moment.
         """
-        if self.is_empty_part:
+        frame = (
             # TODO: What about the 'format' (?)
             # TODO: Maybe I shouldn't set the 'time_base'
             # here and do it just in the Timeline 'render'
             #return get_black_background_video_frame(self._track.size)
             # TODO: This 'time_base' maybe has to be related
             # to a Timeline general 'time_base' and not the fps
-            return get_black_background_video_frame(self._track.size, time_base = fps_to_time_base(self._track.fps))
-
-        frame = self.video.get_frame_at(t)
+            VideoFrameWrapped(
+                frame = self._video_frame_generator.background.full_black(
+                    size = self._track.size,
+                    time_base = fps_to_time_base(self._track.fps)
+                ),
+                is_from_empty_part = True
+            )
+            if self.is_empty_part else
+            VideoFrameWrapped(
+                frame = self.video.get_frame_at(t),
+                is_from_empty_part = False
+            )
+            
+        )
 
         # TODO: This should not happen because of
         # the way we handle the videos here but the
         # video could send us a None frame here, so
         # do we raise exception (?)
-        if frame is None:
+        if frame._frame is None:
             #frame = get_black_background_video_frame(self._track.size)
             # TODO: By now I'm raising exception to check if
             # this happens or not because I think it would
@@ -96,13 +124,20 @@ class _Part:
         
         return frame
     
-    # TODO: I'm not sure if we need this
     def get_audio_frames_at(
         self,
         t: Union[int, float, Fraction]
     ):
+        """
+        Iterate over all the audio frames that
+        exist at the time moment 't' provided.
+        """
         if not self.is_empty_part:
-            frames = self.video.get_audio_frames_at(t)
+            for frame in self.video.get_audio_frames_at(t):
+                yield AudioFrameWrapped(
+                    frame = frame,
+                    is_from_empty_part = False
+                )
         else:
             # TODO: Transform this below to a utils in
             # which I obtain the array directly
@@ -115,15 +150,22 @@ class _Part:
 
             # TODO: I need to set the pts, but here (?)
             # The complete silent frames we need
+            silent_frame = self._audio_frame_generator.silent(
+                sample_rate = self._track.audio_fps,
+                # TODO: Check where do we get this value from
+                layout = 'stereo',
+                number_of_samples = self._track.audio_samples_per_frame,
+                # TODO: Check where do we get this value from
+                format = 'fltp',
+                pts = None,
+                time_base = None
+            )
+            
             frames = (
                 [
-                    get_silent_audio_frame(
-                        sample_rate = self._track.audio_fps,
-                        # TODO: Check where do we get this value from
-                        layout = 'stereo',
-                        number_of_samples = self._track.audio_samples_per_frame,
-                        # TODO: Check where do we get this value from
-                        format = 'fltp'
+                    AudioFrameWrapped(
+                        frame = silent_frame,
+                        is_from_empty_part = True
                     )
                 ] * number_of_frames
                 if number_of_frames > 0 else
@@ -132,21 +174,26 @@ class _Part:
 
             # The remaining partial silent frames we need
             if number_of_remaining_samples > 0:
+                silent_frame = self._audio_frame_generator.silent(
+                    sample_rate = self._track.audio_fps,
+                    # TODO: Check where do we get this value from
+                    layout = 'stereo',
+                    number_of_samples = number_of_remaining_samples,
+                    # TODO: Check where do we get this value from
+                    format = 'fltp',
+                    pts = None,
+                    time_base = None
+                )
+                
                 frames.append(
-                    get_silent_audio_frame(
-                        sample_rate = self._track.audio_fps,
-                        # TODO: Check where do we get this value from
-                        layout = 'stereo',
-                        number_of_samples = number_of_remaining_samples,
-                        # TODO: Check where do we get this value from
-                        format = 'fltp'
+                    AudioFrameWrapped(
+                        frame = silent_frame,
+                        is_from_empty_part = True
                     )
                 )
 
-        # TODO: Return or yield (?)
-        for frame in frames:
-            yield frame
-        #return frames
+            for frame in frames:
+                yield frame
 
 # TODO: I don't like using t as float,
 # we need to implement fractions.Fraction
@@ -288,7 +335,7 @@ class Track:
     def get_frame_at(
         self,
         t: Union[int, float, Fraction]
-    ) -> 'VideoFrame':
+    ) -> 'VideoFrameWrapped':
         """
         Get the frame that must be displayed at
         the 't' time moment provided, which is

yta_video_opengl/complete/video_on_track.py

@@ -127,7 +127,7 @@ class VideoOnTrack:
     def get_audio_frames_at(
         self,
         t: Union[int, float, Fraction]
-    ) -> Union[any, None]:
+    ):
         """
         Get the audio frames that must be played at
         the 't' time moment provided, that could be

yta_video_opengl/reader/cache/video.py

@@ -51,7 +51,10 @@ class VideoFrameCache(FrameCache):
         Get the video frame that is in the 't'
         time moment provided.
         """
+        print(f'Getting frame from {str(float(t))}')
+        print(f'FPS: {str(self.fps)}')
         t: T = T.from_fps(t, self.fps)
+        print(f'So the actual frame is from {str(float(t.truncated))} to {str(float(t.next(1).truncated))}')
         for frame in self.get_frames(t.truncated, t.next(1).truncated):
             return frame
 

yta_video_opengl/tests.py

@@ -602,9 +602,14 @@ def video_modified_stored():
     video = Video(VIDEO_PATH, 0.25, 0.75)
     timeline = Timeline()
     timeline.add_video(Video(VIDEO_PATH, 0.25, 1.0), 0.75)
+    timeline.add_video(Video('C:/Users/dania/Downloads/Y2meta.app-10 Smooth Transitions Green Screen Template For Kinemaster, Alight Motion, Filmora, premiere pro-(1080p).mp4', 1.5, 2.0), 3.0)
     timeline.add_video(Video(VIDEO_PATH, 0.5, 1.0), 2.0)
     timeline.add_video(Video(VIDEO_PATH, 0.5, 1.0), 2.1, do_use_second_track = True)
-    timeline.add_video(Video('C:/Users/dania/Downloads/Y2meta.app-TOP 12 SIMPLE LIQUID TRANSITION _ GREEN SCREEN TRANSITION PACK-(1080p60).mp4', 0.25, 1.5), 0.25, do_use_second_track = True)
+    # 30fps
+    # timeline.add_video(Video('C:/Users/dania/Downloads/Y2meta.app-TOP 12 SIMPLE LIQUID TRANSITION _ GREEN SCREEN TRANSITION PACK-(1080p60).mp4', 0.25, 1.5), 0.25, do_use_second_track = True)
+    # 29.97fps
+    # timeline.add_video(Video('C:/Users/dania/Downloads/Y_una_porra_los_simpsons_castellano.mp4', 5.8, 6.8), 3.6, do_use_second_track = True)
+    timeline.add_video(Video('C:/Users/dania/Downloads/Y_una_porra_los_simpsons_castellano_60fps.mp4', 5.8, 7.8), 3.6, do_use_second_track = True)
     timeline.render(OUTPUT_PATH)
 
     return

yta_video_opengl/video.py

@@ -167,8 +167,9 @@ class Video:
         actually the 1.5s of the video because of
         the subclipped time range.
         """
-        t = self.start + t
+        t += self.start
         
+        print(f'Video real t is {str(float(t))}')
         if t >= self.end:
             raise Exception(f'The "t" ({str(t)}) provided is out of range. This video lasts from [{str(self.start)}, {str(self.end)}).')
         
@@ -182,7 +183,9 @@ class Video:
         Get the video frame with the given 't' time
         moment, using the video cache system.
         """
-        return self.reader.video_cache.get_frame(self._get_real_t(t))
+        print(f'Getting frame from {str(float(t))} that is actually {str(float(self._get_real_t(t)))}')
+        return self.reader.get_frame(self._get_real_t(t))
+        #return self.reader.video_cache.get_frame(self._get_real_t(t))
 
     def get_audio_frame_from_t(
         self,
@@ -214,6 +217,7 @@ class Video:
         (remember that a video frame is associated
         with more than 1 audio frame).
         """
+        print(f'Getting audio frames from {str(float(t))} that is actually {str(float(self._get_real_t(t)))}')
         for frame in self.reader.get_audio_frames_from_t(self._get_real_t(t)):
             yield frame
 

{yta_video_opengl-0.0.16.dist-info → yta_video_opengl-0.0.18.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: yta-video-opengl
-Version: 0.0.16
+Version: 0.0.18
 Summary: Youtube Autonomous Video OpenGL Module
 Author: danialcala94
 Author-email: danielalcalavalera@gmail.com