yta-video-opengl 0.0.11__tar.gz → 0.0.13__tar.gz

{yta_video_opengl-0.0.11 → yta_video_opengl-0.0.13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: yta-video-opengl
-Version: 0.0.11
+Version: 0.0.13
 Summary: Youtube Autonomous Video OpenGL Module
 Author: danialcala94
 Author-email: danielalcalavalera@gmail.com
@@ -10,6 +10,7 @@ Classifier: Programming Language :: Python :: 3.9
 Requires-Dist: av (>=0.0.1,<19.0.0)
 Requires-Dist: moderngl (>=0.0.1,<9.0.0)
 Requires-Dist: numpy (>=0.0.1,<9.0.0)
+Requires-Dist: quicktions (>=0.0.1,<9.0.0)
 Requires-Dist: yta_timer (>0.0.1,<1.0.0)
 Requires-Dist: yta_validation (>=0.0.1,<1.0.0)
 Requires-Dist: yta_video_frame_time (>=0.0.1,<1.0.0)

{yta_video_opengl-0.0.11 → yta_video_opengl-0.0.13}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "yta-video-opengl"
-version = "0.0.11"
+version = "0.0.13"
 description = "Youtube Autonomous Video OpenGL Module"
 authors = [
     {name = "danialcala94",email = "danielalcalavalera@gmail.com"}
@@ -14,6 +14,7 @@ dependencies = [
     "av (>=0.0.1,<19.0.0)",
     "moderngl (>=0.0.1,<9.0.0)",
     "numpy (>=0.0.1,<9.0.0)",
+    "quicktions (>=0.0.1,<9.0.0)"
 ]
 
 [tool.poetry]

yta_video_opengl-0.0.13/src/yta_video_opengl/complete/timeline.py

@@ -0,0 +1,267 @@
+"""
+When we are reading from a source, the reader
+has its own time base and properties. When we
+are writing, the writer has different time
+base and properties. We need to adjust our
+writer to be able to write, because the videos
+we read can be different, and the video we are
+writing is defined by us. The 'time_base' is
+an important property or will make ffmpeg
+become crazy and deny packets (that means no
+video written).
+"""
+from yta_video_opengl.complete.track import Track
+from yta_video_opengl.video import Video
+from yta_video_opengl.t import get_ts, fps_to_time_base, T
+from yta_validation.parameter import ParameterValidator
+from quicktions import Fraction
+from typing import Union
+
+
+class Timeline:
+    """
+    Class to represent all the tracks that
+    exist on the project and to handle the
+    combination of all their frames.
+    """
+
+    @property
+    def end(
+        self
+    ) -> Fraction:
+        """
+        The end of the last video of the track
+        that lasts longer. This is the last time
+        moment that has to be rendered.
+        """
+        return max(
+            track.end
+            for track in self.tracks
+        )
+
+    def __init__(
+        self,
+        size: tuple[int, int] = (1_920, 1_080),
+        fps: Union[int, float, Fraction] = 60.0,
+        audio_fps: Union[int, Fraction] = 44_100.0, # 48_000.0 for aac
+        # TODO: I don't like this name
+        # TODO: Where does this come from (?)
+        audio_samples_per_frame: int = 1024
+    ):
+        # TODO: By now we are using just two video
+        # tracks to test the composition
+        # TODO: We need to be careful with the
+        # priority, by now its defined by its
+        # position in the array
+        self.tracks: list[Track] = [
+            Track(
+                size = size,
+                fps = fps,
+                audio_fps = audio_fps,
+                # TODO: I need more info about the audio
+                # I think
+                audio_samples_per_frame = audio_samples_per_frame
+            ),
+            Track(
+                size = size,
+                fps = fps,
+                audio_fps = audio_fps,
+                # TODO: I need more info about the audio
+                # I think
+                audio_samples_per_frame = audio_samples_per_frame
+            )
+        ]
+        """
+        All the video tracks we are handling.
+        """
+        # TODO: Handle the other properties
+        self.size = size
+        self.fps = fps
+        self.audio_fps = audio_fps
+
+    # TODO: Create 'add_track' method, but by now
+    # we hare handling only one
+    def add_video(
+        self,
+        video: Video,
+        t: Union[int, float, Fraction],
+        # TODO: This is for testing, it has to
+        # disappear
+        do_use_second_track: bool = False
+    ) -> 'Timeline':
+        """
+        Add the provided 'video' to the timeline,
+        starting at the provided 't' time moment.
+
+        TODO: The 'do_use_second_track' parameter
+        is temporary.
+        """
+        # TODO: This is temporary logic by now 
+        # just to be able to test mixing frames
+        # from 2 different tracks at the same
+        # time
+        index = 1 * do_use_second_track
+
+        self.tracks[index].add_video(video, t)
+
+        return self
+    
+    # TODO: This method is not for the Track but
+    # for the timeline, as one track can only
+    # have consecutive elements
+    def get_frame_at(
+        self,
+        t: Union[int, float, Fraction]
+    ) -> 'VideoFrame':
+        """
+        Get all the frames that are played at the
+        't' time provided, but combined in one.
+        """
+        frames = (
+            track.get_frame_at(t)
+            for track in self.tracks
+        )
+        # TODO: Here I receive black frames because
+        # it was empty, but I don't have a way to
+        # detect those black empty frames because
+        # they are just VideoFrame instances... I
+        # need a way to know so I can skip them if
+        # other frame in other track, or to know if
+        # I want them as transparent or something
+
+        # TODO: Combinate them, I send first by now
+        return next(frames)
+    
+    def get_audio_frames_at(
+        self,
+        t: float
+    ):
+        # TODO: What if the different audio streams
+        # have also different fps (?)
+        frames = []
+        for track in self.tracks:
+            # TODO: Make this work properly
+            audio_frames = track.get_audio_frames_at(t)
+
+            # TODO: Combine them
+            if audio_frames is not None:
+                frames = audio_frames
+                break
+            
+        #from yta_video_opengl.utils import get_silent_audio_frame
+        #make_silent_audio_frame()
+        for frame in frames:
+            yield frame
+            
+    def render(
+        self,
+        filename: str,
+        start: Union[int, float, Fraction] = 0.0,
+        end: Union[int, float, Fraction, None] = None
+    ) -> 'Timeline':
+        """
+        Render the time range in between the given
+        'start' and 'end' and store the result with
+        the also provided 'fillename'.
+
+        If no 'start' and 'end' provided, the whole
+        project will be rendered.
+        """
+        ParameterValidator.validate_mandatory_string('filename', filename, do_accept_empty = False)
+        # TODO: We need to accept Fraction as number
+        #ParameterValidator.validate_mandatory_positive_number('start', start, do_include_zero = True)
+        # TODO: We need to accept Fraction as number
+        #ParameterValidator.validate_positive_number('end', end, do_include_zero = False)
+
+        # TODO: Limitate 'end' a bit...
+        end = (
+            self.end
+            if end is None else
+            end
+        )
+
+        if start >= end:
+            raise Exception('The provided "start" cannot be greater or equal to the "end" provided.')
+
+        from yta_video_opengl.writer import VideoWriter
+
+        writer = VideoWriter('test_files/output_render.mp4')
+        # TODO: This has to be dynamic according to the
+        # video we are writing
+        writer.set_video_stream(
+            codec_name = 'h264',
+            fps = self.fps,
+            size = self.size,
+            pixel_format = 'yuv420p'
+        )
+        
+        writer.set_audio_stream(
+            codec_name = 'aac',
+            fps = self.audio_fps
+        )
+
+        time_base = fps_to_time_base(self.fps)
+        audio_time_base = fps_to_time_base(self.audio_fps)
+
+        """
+        We are trying to render this:
+        -----------------------------
+        [0 a 0.5) => Frames negros
+        [0.5 a 1.25) => [0.25 a 1.0) de Video1
+        [1.25 a 1.75) => Frames negros
+        [1.75 a 2.25) => [0.25 a 0.75) de Video1
+        [2.25 a 3.0) => Frames negros
+        [3.0 a 3.75) => [2.25 a 3.0) de Video2
+        """
+        
+        audio_pts = 0
+        for t in get_ts(start, end, self.fps):
+            frame = self.get_frame_at(t)
+
+            #print(frame)
+
+            # We need to adjust our output elements to be
+            # consecutive and with the right values
+            # TODO: We are using int() for fps but its float...
+            frame.time_base = time_base
+            #frame.pts = int(video_frame_index / frame.time_base)
+            frame.pts = T(t, time_base).truncated_pts
+
+            # TODO: We need to handle the audio
+            writer.mux_video_frame(
+                frame = frame
+            )
+
+            #print(f'    [VIDEO] Here in t:{str(t)} -> pts:{str(frame.pts)} - dts:{str(frame.dts)}')
+
+            # TODO: Uncomment all this below for the audio
+            num_of_audio_frames = 0
+            for audio_frame in self.get_audio_frames_at(t):
+                # TODO: The track gives us empty (black)
+                # frames by default but maybe we need a
+                # @dataclass in the middle to handle if
+                # we want transparent frames or not and/or
+                # to detect them here because, if not,
+                # they are just simple VideoFrames and we
+                # don't know they are 'empty' frames
+
+                # We need to adjust our output elements to be
+                # consecutive and with the right values
+                # TODO: We are using int() for fps but its float...
+                audio_frame.time_base = audio_time_base
+                #audio_frame.pts = int(audio_frame_index / audio_frame.time_base)
+                audio_frame.pts = audio_pts
+                # We increment for the next iteration
+                audio_pts += audio_frame.samples
+                #audio_frame.pts = int(t + (audio_frame_index * audio_frame.time_base) / audio_frame.time_base)
+
+                #print(f'[AUDIO] Here in t:{str(t)} -> pts:{str(audio_frame.pts)} - dts:{str(audio_frame.dts)}')
+
+                #num_of_audio_frames += 1
+                #print(audio_frame)
+                writer.mux_audio_frame(audio_frame)
+            #print(f'Num of audio frames: {str(num_of_audio_frames)}')
+
+        writer.mux_video_frame(None)
+        writer.mux_audio_frame(None)
+        writer.output.close()

yta_video_opengl-0.0.13/src/yta_video_opengl/complete/track.py

@@ -0,0 +1,411 @@
+from yta_video_opengl.complete.video_on_track import VideoOnTrack
+from yta_video_opengl.video import Video
+from yta_video_opengl.t import T
+from yta_video_opengl.utils import get_black_background_video_frame, get_silent_audio_frame, audio_frames_and_remainder_per_video_frame
+from yta_video_opengl.t import fps_to_time_base
+from yta_validation.parameter import ParameterValidator
+from quicktions import Fraction
+from typing import Union
+
+
+NON_LIMITED_EMPTY_PART_END = 999
+"""
+A value to indicate that the empty part
+has no end because it is in the last
+position and there is no video after it.
+"""
+class _Part:
+    """
+    Class to represent an element that is on the
+    track, that can be an empty space or a video
+    (with audio).
+    """
+
+    @property
+    def is_empty_part(
+        self
+    ) -> bool:
+        """
+        Flag to indicate if the part is an empty part,
+        which means that there is no video associated
+        but an empty space.
+        """
+        return self.video is None
+
+    def __init__(
+        self,
+        track: 'Track',
+        start: Union[int, float, Fraction],
+        end: Union[int, float, Fraction],
+        video: Union[VideoOnTrack, None] = None
+    ):
+        # TODO: We need to accept Fraction as number
+        # ParameterValidator.validate_mandatory_positive_number('start', start, do_include_zero = True)
+        # TODO: We need to accept Fraction as number
+        # ParameterValidator.validate_mandatory_positive_number('end', end, do_include_zero = False)
+        ParameterValidator.validate_instance_of('video', video, VideoOnTrack)
+
+        self._track: Track = track
+        """
+        The instance of the track this part belongs
+        to.
+        """
+        self.start: Fraction = Fraction(start)
+        """
+        The start 't' time moment of the part.
+        """
+        self.end: Fraction = Fraction(end)
+        """
+        The end 't' time moment of the part.
+        """
+        self.video: Union[VideoOnTrack, None] = video
+        """
+        The video associated, if existing, or
+        None if it is an empty space that we need
+        to fulfill with a black background and
+        silent audio.
+        """
+
+    def get_frame_at(
+        self,
+        t: Union[int, float, Fraction]
+    ) -> 'VideoFrame':
+        """
+        Get the frame that must be displayed at 
+        the given 't' time moment.
+        """
+        if self.is_empty_part:
+            # TODO: What about the 'format' (?)
+            # TODO: Maybe I shouldn't set the 'time_base'
+            # here and do it just in the Timeline 'render'
+            #return get_black_background_video_frame(self._track.size)
+            # TODO: This 'time_base' maybe has to be related
+            # to a Timeline general 'time_base' and not the fps
+            return get_black_background_video_frame(self._track.size, time_base = fps_to_time_base(self._track.fps))
+
+        frame = self.video.get_frame_at(t)
+
+        # TODO: This should not happen because of
+        # the way we handle the videos here but the
+        # video could send us a None frame here, so
+        # do we raise exception (?)
+        if frame is None:
+            #frame = get_black_background_video_frame(self._track.size)
+            # TODO: By now I'm raising exception to check if
+            # this happens or not because I think it would
+            # be malfunctioning
+            raise Exception(f'Video is returning None frame at t={str(t)}.')
+        
+        return frame
+    
+    # TODO: I'm not sure if we need this
+    def get_audio_frames_at(
+        self,
+        t: Union[int, float, Fraction]
+    ):
+        if not self.is_empty_part:
+            frames = self.video.get_audio_frames_at(t)
+        else:
+            # TODO: Transform this below to a utils in
+            # which I obtain the array directly
+            # Check many full and partial silent frames we need
+            number_of_frames, number_of_remaining_samples = audio_frames_and_remainder_per_video_frame(
+                video_fps = self._track.fps,
+                sample_rate = self._track.audio_fps,
+                number_of_samples_per_audio_frame = self._track.audio_samples_per_frame
+            )
+
+            # TODO: I need to set the pts, but here (?)
+            # The complete silent frames we need
+            frames = (
+                [
+                    get_silent_audio_frame(
+                        sample_rate = self._track.audio_fps,
+                        # TODO: Check where do we get this value from
+                        layout = 'stereo',
+                        number_of_samples = self._track.audio_samples_per_frame,
+                        # TODO: Check where do we get this value from
+                        format = 'fltp'
+                    )
+                ] * number_of_frames
+                if number_of_frames > 0 else
+                []
+            )
+
+            # The remaining partial silent frames we need
+            if number_of_remaining_samples > 0:
+                frames.append(
+                    get_silent_audio_frame(
+                        sample_rate = self._track.audio_fps,
+                        # TODO: Check where do we get this value from
+                        layout = 'stereo',
+                        number_of_samples = number_of_remaining_samples,
+                        # TODO: Check where do we get this value from
+                        format = 'fltp'
+                    )
+                )
+
+        # TODO: Return or yield (?)
+        for frame in frames:
+            yield frame
+        #return frames
+
+# TODO: I don't like using t as float,
+# we need to implement fractions.Fraction
+# TODO: This is called Track but it is
+# handling videos only. Should I have
+# VideoTrack and AudioTrack (?)
+class Track:
+    """
+    Class to represent a track in which we place
+    videos, images and audio to build a video
+    project.
+    """
+
+    @property
+    def parts(
+        self
+    ) -> list[_Part]:
+        """
+        The list of parts that build this track,
+        but with the empty parts detected to 
+        be fulfilled with black frames and silent
+        audios.
+
+        A part can be a video or an empty space.
+        """
+        if (
+            not hasattr(self, '_parts') or
+            self._parts is None
+        ):
+            self._recalculate_parts()
+
+        return self._parts
+
+    @property
+    def end(
+        self
+    ) -> Fraction:
+        """
+        The end of the last video of this track,
+        which is also the end of the track. This
+        is the last time moment that has to be
+        rendered.
+        """
+        return Fraction(
+            0.0
+            if len(self.videos) == 0 else
+            max(
+                video.end
+                for video in self.videos
+            )
+        )
+
+    def __init__(
+        self,
+        # TODO: I need the general settings of the
+        # project to be able to make audio also, not
+        # only the empty frames
+        size: tuple[int, int],
+        fps: float,
+        audio_fps: float,
+        # TODO: Where does it come from (?)
+        audio_samples_per_frame: int
+    ):
+        self.videos: list[VideoOnTrack] = []
+        """
+        The list of 'VideoOnTrack' instances that
+        must play on this track.
+        """
+        self.size: tuple[int, int] = size
+        """
+        The size of the videos of this track.
+        """
+        self.fps: float = float(fps)
+        """
+        The fps of the track, needed to calculate
+        the base t time moments to be precise and
+        to obtain or generate the frames.
+        """
+        self.audio_fps: float = float(audio_fps)
+        """
+        The fps of the audio track, needed to 
+        generate silent audios for the empty parts.
+        """
+        self.audio_samples_per_frame: int = audio_samples_per_frame
+        """
+        The number of samples per audio frame.
+        """
+
+    def _is_free(
+        self,
+        start: Union[int, float, Fraction],
+        end: Union[int, float, Fraction]
+    ) -> bool:
+        """
+        Check if the time range in between the 
+        'start' and 'end' time given is free or
+        there is some video playing at any moment.
+        """
+        return not any(
+            (
+                video.video.start < end and
+                video.video.end > start
+            )
+            for video in self.videos
+        )
+    
+    def _get_part_at_t(
+        self,
+        t: Union[int, float, Fraction]
+    ) -> _Part:
+        """
+        Get the part at the given 't' time
+        moment, that will always exist because
+        we have an special non ended last 
+        empty part that would be returned if
+        accessing to an empty 't'.
+        """
+        for part in self.parts:
+            if part.start <= t < part.end:
+                return part
+            
+        # TODO: This will only happen if they are
+        # asking for a value greater than the
+        # NON_LIMITED_EMPTY_PART_END...
+        raise Exception('NON_LIMITED_EMPTY_PART_END exceeded.')
+        return None
+
+    def get_frame_at(
+        self,
+        t: Union[int, float, Fraction]
+    ) -> 'VideoFrame':
+        """
+        Get the frame that must be displayed at
+        the 't' time moment provided, which is
+        a frame from the video audio that is
+        being played at that time moment.
+
+        Remember, this 't' time moment provided
+        is about the track, and we make the
+        conversion to the actual video 't' to
+        get the frame.
+        """
+        # TODO: What if the frame, that comes from
+        # a video, doesn't have the expected size (?)
+        return self._get_part_at_t(t).get_frame_at(t)
+    
+    # TODO: This is not working well...
+    def get_audio_frames_at(
+        self,
+        t: Union[int, float, Fraction]
+    ):
+        """
+        Get the sequence of audio frames that
+        must be displayed at the 't' time 
+        moment provided, which the collection
+        of audio frames corresponding to the
+        video frame that is being played at
+        that time moment.
+
+        Remember, this 't' time moment provided
+        is about the track, and we make the
+        conversion to the actual video 't' to
+        get the frame.
+
+        This is useful when we want to write a
+        video frame with its audio, so we obtain
+        all the audio frames associated to it
+        (remember that a video frame is associated
+        with more than 1 audio frame).
+        """
+        for frame in self._get_part_at_t(t).get_audio_frames_at(t):
+            yield frame
+    
+    def add_video(
+        self,
+        video: Video,
+        t: Union[int, float, Fraction, None] = None
+    ) -> 'Track':
+        """
+        Add the 'video' provided to the track. If
+        a 't' time moment is provided, the video
+        will be added to that time moment if 
+        possible. If there is no other video 
+        placed in the time gap between the given
+        't' and the provided 'video' duration, it
+        will be added succesfully. In the other
+        case, an exception will be raised.
+
+        If 't' is None, the first available 't'
+        time moment will be used, that will be 0.0
+        if no video, or the end of the last video.
+        """
+        ParameterValidator.validate_mandatory_instance_of('video', video, Video)
+        ParameterValidator.validate_positive_number('t', t, do_include_zero = True)
+
+        if t is not None:
+            # TODO: We can have many different strategies
+            # that we could define in the '__init__' maybe
+            t: T = T.from_fps(t, self.fps)
+            if not self._is_free(t.truncated, t.next(1).truncated):
+                raise Exception('The video cannot be added at the "t" time moment, something blocks it.')
+            t = t.truncated
+        else:
+            t = self.end
+        
+        self.videos.append(VideoOnTrack(
+            video,
+            t
+        ))
+
+        self._recalculate_parts()
+
+        # TODO: Maybe return the VideoOnTrack instead (?)
+        return self
+    
+    def _recalculate_parts(
+        self
+    ) -> 'Track':
+        """
+        Check the track and get all the parts. A
+        part can be empty (non video nor audio on
+        that time period, which means black 
+        background and silence audio), or a video
+        with (or without) audio.
+        """
+        parts = []
+        cursor = 0.0
+
+        for video in self.videos:
+            # Empty space between cursor and start of
+            # the next clip
+            if video.start > cursor:
+                parts.append(_Part(
+                    track = self,
+                    start = cursor,
+                    end = video.start,
+                    video = None
+                ))
+
+            # The video itself
+            parts.append(_Part(
+                track = self,
+                start = video.start,
+                end = video.end,
+                video = video
+            ))
+            
+            cursor = video.end
+
+        # Add the non limited last empty part
+        parts.append(_Part(
+            track = self,
+            start = cursor,
+            end = NON_LIMITED_EMPTY_PART_END,
+            video = None
+        ))
+
+        self._parts = parts
+
+        return self