yta-video-opengl 0.0.11__py3-none-any.whl → 0.0.12__py3-none-any.whl

yta_video_opengl/complete/timeline.py

@@ -1,12 +1,21 @@
+"""
+When we are reading from a source, the reader
+has its own time base and properties. When we
+are writing, the writer has different time
+base and properties. We need to adjust our
+writer to be able to write, because the videos
+we read can be different, and the video we are
+writing is defined by us. The 'time_base' is
+an important property or will make ffmpeg
+become crazy and deny packets (that means no
+video written).
+"""
 from yta_video_opengl.complete.track import Track
 from yta_video_opengl.video import Video
 from yta_validation.parameter import ParameterValidator
 from typing import Union
 from fractions import Fraction
 
-import numpy as np
-import av
-
 
 class Timeline:
     """
@@ -24,25 +33,50 @@ class Timeline:
         that lasts longer. This is the last time
         moment that has to be rendered.
         """
-        return max(track.end for track in self.tracks)
+        return max(
+            track.end
+            for track in self.tracks
+        )
 
     def __init__(
         self,
-        size: tuple[int, int] = (1920, 1080),
-        fps: float = 60.0
+        size: tuple[int, int] = (1_920, 1_080),
+        fps: float = 60.0,
+        audio_fps: float = 44_100.0, # 48_000.0 for aac
+        # TODO: I don't like this name
+        # TODO: Where does this come from (?)
+        audio_nb_samples: int = 1024
     ):
         # TODO: By now we are using just two video
         # tracks to test the composition
         # TODO: We need to be careful with the
         # priority, by now its defined by its
         # position in the array
-        self.tracks: list[Track] = [Track(), Track()]
+        self.tracks: list[Track] = [
+            Track(
+                size = size,
+                fps = fps,
+                audio_fps = audio_fps,
+                # TODO: I need more info about the audio
+                # I think
+                audio_nb_samples = audio_nb_samples
+            ),
+            Track(
+                size = size,
+                fps = fps,
+                audio_fps = audio_fps,
+                # TODO: I need more info about the audio
+                # I think
+                audio_nb_samples = audio_nb_samples
+            )
+        ]
         """
         All the video tracks we are handling.
         """
-        # TODO: Handle size and fps
+        # TODO: Handle the other properties
         self.size = size
         self.fps = fps
+        self.audio_fps = audio_fps
 
     # TODO: Create 'add_track' method, but by now
     # we hare handling only one
@@ -61,19 +95,23 @@ class Timeline:
         TODO: The 'do_use_second_track' parameter
         is temporary.
         """
+        # TODO: This is temporary logic by now 
+        # just to be able to test mixing frames
+        # from 2 different tracks at the same
+        # time
         index = 1 * do_use_second_track
 
         self.tracks[index].add_video(video, t)
 
         return self
-
+    
     # TODO: This method is not for the Track but
     # for the timeline, as one track can only
     # have consecutive elements
     def get_frame_at(
         self,
         t: float
-    ) -> Union['VideoFrame', None]:
+    ) -> 'VideoFrame':
         """
         Get all the frames that are played at the
         't' time provided, but combined in one.
@@ -82,24 +120,38 @@ class Timeline:
             track.get_frame_at(t)
             for track in self.tracks
         )
+        # TODO: Here I receive black frames because
+        # it was empty, but I don't have a way to
+        # detect those black empty frames because
+        # they are just VideoFrame instances... I
+        # need a way to know so I can skip them if
+        # other frame in other track, or to know if
+        # I want them as transparent or something
 
-        frames = [
-            frame
-            for frame in frames
-            if frame is not None
-        ]
-
-        return (
-            # TODO: Combinate them, I send first by now
-            frames[0]
-            if len(frames) > 0 else
-            # TODO: Should I send None or a full
-            # black (or transparent) frame? I think
-            # None is better because I don't know
-            # the size here (?)
-            None
-        )
+        # TODO: Combinate them, I send first by now
+        return next(frames)
     
+    def get_audio_frames_at(
+        self,
+        t: float
+    ):
+        # TODO: What if the different audio streams
+        # have also different fps (?)
+        frames = []
+        for track in self.tracks:
+            # TODO: Make this work properly
+            audio_frames = track.get_audio_frames_at(t)
+
+            # TODO: Combine them
+            if audio_frames is not None:
+                frames = audio_frames
+                break
+            
+        #from yta_video_opengl.utils import get_silent_audio_frame
+        #make_silent_audio_frame()
+        for frame in frames:
+            yield frame
+            
     def render(
         self,
         filename: str,
@@ -127,47 +179,34 @@ class Timeline:
 
         if start >= end:
             raise Exception('The provided "start" cannot be greater or equal to the "end" provided.')
-        # TODO: Obtain all the 't', based on 'fps'
-        # that we need to render from 'start' to
-        # 'end'
-        # TODO: I don't want to have this here
-        def generate_times(start: float, end: float, fps: int):
-            dt = 1.0 / fps
-            times = []
-
-            t = start
-            while t <= end:
-                times.append(t + 0.000001)
-                t += dt
-
-            return times
 
         from yta_video_opengl.writer import VideoWriter
+        from yta_video_opengl.utils import get_black_background_video_frame, get_silent_audio_frame
 
         writer = VideoWriter('test_files/output_render.mp4')
         # TODO: This has to be dynamic according to the
         # video we are writing
         writer.set_video_stream(
             codec_name = 'h264',
-            fps = 60,
-            size = (1920, 1080),
+            fps = self.fps,
+            size = self.size,
             pixel_format = 'yuv420p'
         )
         
-        for t in generate_times(start, end, self.fps):
+        writer.set_audio_stream(
+            codec_name = 'aac',
+            fps = self.audio_fps
+        )
+        
+        audio_pts = 0
+        for t in get_ts(start, end, self.fps):
             frame = self.get_frame_at(t)
 
-            if frame is None:
-                # Replace with black background if no frame
-                frame = av.VideoFrame.from_ndarray(
-                    array = np.zeros((1920, 1080, 3), dtype = np.uint8),
-                    format = 'rgb24'
-                )
-
             # We need to adjust our output elements to be
             # consecutive and with the right values
             # TODO: We are using int() for fps but its float...
             frame.time_base = Fraction(1, int(self.fps))
+            #frame.pts = int(video_frame_index / frame.time_base)
             frame.pts = int(t / frame.time_base)
 
             # TODO: We need to handle the audio
@@ -175,5 +214,58 @@ class Timeline:
                 frame = frame
             )
 
+            #print(f'    [VIDEO] Here in t:{str(t)} -> pts:{str(frame.pts)} - dts:{str(frame.dts)}')
+
+            num_of_audio_frames = 0
+            for audio_frame in self.get_audio_frames_at(t):
+                # TODO: The track gives us empty (black)
+                # frames by default but maybe we need a
+                # @dataclass in the middle to handle if
+                # we want transparent frames or not and/or
+                # to detect them here because, if not,
+                # they are just simple VideoFrames and we
+                # don't know they are 'empty' frames
+
+                # We need to adjust our output elements to be
+                # consecutive and with the right values
+                # TODO: We are using int() for fps but its float...
+                audio_frame.time_base = Fraction(1, int(self.audio_fps))
+                #audio_frame.pts = int(audio_frame_index / audio_frame.time_base)
+                audio_frame.pts = audio_pts
+                # We increment for the next iteration
+                audio_pts += audio_frame.samples
+                #audio_frame.pts = int(t + (audio_frame_index * audio_frame.time_base) / audio_frame.time_base)
+
+                #print(f'[AUDIO] Here in t:{str(t)} -> pts:{str(audio_frame.pts)} - dts:{str(audio_frame.dts)}')
+
+                num_of_audio_frames += 1
+                print(audio_frame)
+                writer.mux_audio_frame(audio_frame)
+            print(f'Num of audio frames: {str(num_of_audio_frames)}')
+
         writer.mux_video_frame(None)
-        writer.output.close()
+        writer.mux_audio_frame(None)
+        writer.output.close()
+
+
+# TODO: I don't want to have this here
+def get_ts(
+    start: float,
+    end: float,
+    fps: int
+):
+    """
+    Obtain, without using a Progression class and
+    importing the library, a list of 't' time
+    moments from the provided 'start' to the also
+    given 'end', with the 'fps' given as parameter.
+    """
+    dt = 1.0 / fps
+    times = []
+
+    t = start
+    while t <= end:
+        times.append(t + 0.000001)
+        t += dt
+
+    return times

yta_video_opengl/complete/track.py

@@ -1,9 +1,148 @@
 from yta_video_opengl.complete.video_on_track import VideoOnTrack
 from yta_video_opengl.video import Video
+from yta_video_frame_time import T
+from yta_video_opengl.utils import get_black_background_video_frame, get_silent_audio_frame, audio_frames_and_remainder_per_video_frame
 from yta_validation.parameter import ParameterValidator
 from typing import Union
 
 
+NON_LIMITED_EMPTY_PART_END = 999
+"""
+A value to indicate that the empty part
+has no end because it is in the last
+position and there is no video after it.
+"""
+class _Part:
+    """
+    Class to represent an element that is on the
+    track, that can be an empty space or a video
+    (with audio).
+    """
+
+    @property
+    def is_empty_part(
+        self
+    ) -> bool:
+        """
+        Flag to indicate if the part is an empty part,
+        which means that there is no video associated
+        but an empty space.
+        """
+        return self.video is None
+
+    def __init__(
+        self,
+        track: 'Track',
+        start: float,
+        end: float,
+        video: Union[VideoOnTrack, None] = None
+    ):
+        ParameterValidator.validate_mandatory_positive_number('start', start, do_include_zero = True)
+        ParameterValidator.validate_mandatory_positive_number('end', end, do_include_zero = False)
+        ParameterValidator.validate_instance_of('video', video, VideoOnTrack)
+
+        self._track: Track = track
+        """
+        The instance of the track this part belongs
+        to.
+        """
+        self.start: float = float(start)
+        """
+        The start 't' time moment of the part.
+        """
+        self.end: float = float(end)
+        """
+        The end 't' time moment of the part.
+        """
+        self.video: Union[VideoOnTrack, None] = video
+        """
+        The video associated, if existing, or
+        None if it is an empty space that we need
+        to fulfill with a black background and
+        silent audio.
+        """
+
+    def get_frame_at(
+        self,
+        t: float
+    ) -> 'VideoFrame':
+        """
+        Get the frame that must be displayed at 
+        the given 't' time moment.
+        """
+        if self.is_empty_part:
+            # TODO: What about the 'format' (?)
+            return get_black_background_video_frame(self._track.size)
+
+        frame = self.video.get_frame_at(t)
+
+        # TODO: This should not happen because of
+        # the way we handle the videos here but the
+        # video could send us a None frame here, so
+        # do we raise exception (?)
+        if frame is None:
+            #frame = get_black_background_video_frame(self._track.size)
+            # TODO: By now I'm raising exception to check if
+            # this happens or not because I think it would
+            # be malfunctioning
+            raise Exception(f'Video is returning None frame at t={str(t)}.')
+        
+        return frame
+    
+    # TODO: I'm not sure if we need this
+    def get_audio_frames_at(
+        self,
+        t: float
+    ):
+        if not self.is_empty_part:
+            frames = self.video.get_audio_frames_at(t)
+        else:
+            # TODO: Transform this below to a utils in
+            # which I obtain the array directly
+            # Check many full and partial silent frames we need
+            number_of_frames, number_of_remaining_samples = audio_frames_and_remainder_per_video_frame(
+                fps = self._track.fps,
+                sample_rate = self._track.audio_fps,
+                nb_samples = self._track.audio_nb_samples
+            )
+
+            # TODO: I need to set the pts, but here (?)
+            # The complete silent frames we need
+            frames = (
+                [
+                    get_silent_audio_frame(
+                        sample_rate = self._track.audio_fps,
+                        # TODO: Check where do we get this value from
+                        layout = 'stereo',
+                        nb_samples = self._track.audio_nb_samples,
+                        # TODO: Check where do we get this value from
+                        format = 'fltp'
+                    )
+                ] * number_of_frames
+                if number_of_frames > 0 else
+                []
+            )
+
+            # The remaining partial silent frames we need
+            if number_of_remaining_samples > 0:
+                frames.append(
+                    get_silent_audio_frame(
+                        sample_rate = self._track.audio_fps,
+                        # TODO: Check where do we get this value from
+                        layout = 'stereo',
+                        nb_samples = number_of_remaining_samples,
+                        # TODO: Check where do we get this value from
+                        format = 'fltp'
+                    )
+                )
+
+        # TODO: Return or yield (?)
+        for frame in frames:
+            yield frame
+        #return frames
+
+# TODO: I don't like using t as float,
+# we need to implement fractions.Fraction
 # TODO: This is called Track but it is
 # handling videos only. Should I have
 # VideoTrack and AudioTrack (?)
@@ -14,6 +153,26 @@ class Track:
     project.
     """
 
+    @property
+    def parts(
+        self
+    ) -> list[_Part]:
+        """
+        The list of parts that build this track,
+        but with the empty parts detected to 
+        be fulfilled with black frames and silent
+        audios.
+
+        A part can be a video or an empty space.
+        """
+        if (
+            not hasattr(self, '_parts') or
+            self._parts is None
+        ):
+            self._recalculate_parts()
+
+        return self._parts
+
     @property
     def end(
         self
@@ -34,13 +193,40 @@ class Track:
         )
 
     def __init__(
-        self
+        self,
+        # TODO: I need the general settings of the
+        # project to be able to make audio also, not
+        # only the empty frames
+        size: tuple[int, int],
+        fps: float,
+        audio_fps: float,
+        # TODO: Change the name
+        audio_nb_samples: int
     ):
         self.videos: list[VideoOnTrack] = []
         """
         The list of 'VideoOnTrack' instances that
         must play on this track.
         """
+        self.size: tuple[int, int] = size
+        """
+        The size of the videos of this track.
+        """
+        self.fps: float = fps
+        """
+        The fps of the track, needed to calculate
+        the base t time moments to be precise and
+        to obtain or generate the frames.
+        """
+        self.audio_fps: float = audio_fps
+        """
+        The fps of the audio track, needed to 
+        generate silent audios for the empty parts.
+        """
+        self.audio_nb_samples: int = audio_nb_samples
+        """
+        The number of samples per audio frame.
+        """
 
     def _is_free(
         self,
@@ -60,43 +246,73 @@ class Track:
             for video in self.videos
         )
     
-    def _get_video_at_t(
+    def _get_part_at_t(
         self,
         t: float
-    ) -> Union[VideoOnTrack, None]:
+    ) -> _Part:
         """
-        Get the video that is being played at
-        the 't' time moment provided.
+        Get the part at the given 't' time
+        moment, that will always exist because
+        we have an special non ended last 
+        empty part that would be returned if
+        accessing to an empty 't'.
         """
-        for video in self.videos:
-            if video.start <= t < video.end:
-                return video
+        for part in self.parts:
+            if part.start <= t < part.end:
+                return part
             
+        # TODO: This will only happen if they are
+        # asking for a value greater than the
+        # NON_LIMITED_EMPTY_PART_END...
+        raise Exception('NON_LIMITED_EMPTY_PART_END exceeded.')
         return None
-    
+
     def get_frame_at(
         self,
         t: float
-    ) -> Union['VideoFrame', None]:
+    ) -> 'VideoFrame':
         """
         Get the frame that must be displayed at
         the 't' time moment provided, which is
-        a frame from the video that is being
-        played at that time moment.
+        a frame from the video audio that is
+        being played at that time moment.
 
         Remember, this 't' time moment provided
         is about the track, and we make the
         conversion to the actual video 't' to
         get the frame.
         """
-        video = self._get_video_at_t(t)
+        # TODO: What if the frame, that comes from
+        # a video, doesn't have the expected size (?)
+        return self._get_part_at_t(t).get_frame_at(t)
+    
+    # TODO: This is not working well...
+    def get_audio_frames_at(
+        self,
+        t: float
+    ):
+        """
+        Get the sequence of audio frames that
+        must be displayed at the 't' time 
+        moment provided, which the collection
+        of audio frames corresponding to the
+        video frame that is being played at
+        that time moment.
 
-        return (
-            video.get_frame_at(t)
-            if video is not None else
-            None
-        )
+        Remember, this 't' time moment provided
+        is about the track, and we make the
+        conversion to the actual video 't' to
+        get the frame.
 
+        This is useful when we want to write a
+        video frame with its audio, so we obtain
+        all the audio frames associated to it
+        (remember that a video frame is associated
+        with more than 1 audio frame).
+        """
+        for frame in self._get_part_at_t(t).get_audio_frames_at(t):
+            yield frame
+    
     def add_video(
         self,
         video: Video,
@@ -117,11 +333,14 @@ class Track:
         if no video, or the end of the last video.
         """
         ParameterValidator.validate_mandatory_instance_of('video', video, Video)
-        ParameterValidator.validate_positive_float('t', t, do_include_zero = True)
+        ParameterValidator.validate_positive_number('t', t, do_include_zero = True)
 
         if t is not None:
             # TODO: We can have many different strategies
             # that we could define in the '__init__' maybe
+            # TODO: I don't like using float 't', but we
+            # need to make sure it is a multiple of 1 / fps
+            t = T.get_frame_time_base(float(t), self.fps)
             if not self._is_free(t, (t + video.end)):
                 raise Exception('The video cannot be added at the "t" time moment, something blocks it.')
         else:
@@ -132,5 +351,53 @@ class Track:
             t
         ))
 
+        self._recalculate_parts()
+
         # TODO: Maybe return the VideoOnTrack instead (?)
+        return self
+    
+    def _recalculate_parts(
+        self
+    ) -> 'Track':
+        """
+        Check the track and get all the parts. A
+        part can be empty (non video nor audio on
+        that time period, which means black 
+        background and silence audio), or a video
+        with (or without) audio.
+        """
+        parts = []
+        cursor = 0.0
+
+        for video in self.videos:
+            # Empty space between cursor and start of
+            # the next clip
+            if video.start > cursor:
+                parts.append(_Part(
+                    track = self,
+                    start = cursor,
+                    end = video.start,
+                    video = None
+                ))
+
+            # The video itself
+            parts.append(_Part(
+                track = self,
+                start = video.start,
+                end = video.end,
+                video = video
+            ))
+            
+            cursor = video.end
+
+        # Add the non limited last empty part
+        parts.append(_Part(
+            track = self,
+            start = cursor,
+            end = NON_LIMITED_EMPTY_PART_END,
+            video = None
+        ))
+
+        self._parts = parts
+
         return self