yta-video-opengl 0.0.9__py3-none-any.whl → 0.0.11__py3-none-any.whl

yta_video_opengl/classes.py

@@ -132,6 +132,7 @@ class _Uniforms:
         for key, value in self.uniforms.items():
             print(f'"{key}": {str(value)}')
 
+# TODO: Moved to 'nodes.opengl.py'
 class BaseNode:
     """
     The basic class of a node to manipulate frames

yta_video_opengl/complete/timeline.py

@@ -0,0 +1,179 @@
+from yta_video_opengl.complete.track import Track
+from yta_video_opengl.video import Video
+from yta_validation.parameter import ParameterValidator
+from typing import Union
+from fractions import Fraction
+
+import numpy as np
+import av
+
+
+class Timeline:
+    """
+    Class to represent all the tracks that
+    exist on the project and to handle the
+    combination of all their frames.
+    """
+
+    @property
+    def end(
+        self
+    ) -> float:
+        """
+        The end of the last video of the track
+        that lasts longer. This is the last time
+        moment that has to be rendered.
+        """
+        return max(track.end for track in self.tracks)
+
+    def __init__(
+        self,
+        size: tuple[int, int] = (1920, 1080),
+        fps: float = 60.0
+    ):
+        # TODO: By now we are using just two video
+        # tracks to test the composition
+        # TODO: We need to be careful with the
+        # priority, by now its defined by its
+        # position in the array
+        self.tracks: list[Track] = [Track(), Track()]
+        """
+        All the video tracks we are handling.
+        """
+        # TODO: Handle size and fps
+        self.size = size
+        self.fps = fps
+
+    # TODO: Create 'add_track' method, but by now
+    # we hare handling only one
+    def add_video(
+        self,
+        video: Video,
+        t: float,
+        # TODO: This is for testing, it has to
+        # disappear
+        do_use_second_track: bool = False
+    ) -> 'Timeline':
+        """
+        Add the provided 'video' to the timeline,
+        starting at the provided 't' time moment.
+
+        TODO: The 'do_use_second_track' parameter
+        is temporary.
+        """
+        index = 1 * do_use_second_track
+
+        self.tracks[index].add_video(video, t)
+
+        return self
+
+    # TODO: This method is not for the Track but
+    # for the timeline, as one track can only
+    # have consecutive elements
+    def get_frame_at(
+        self,
+        t: float
+    ) -> Union['VideoFrame', None]:
+        """
+        Get all the frames that are played at the
+        't' time provided, but combined in one.
+        """
+        frames = (
+            track.get_frame_at(t)
+            for track in self.tracks
+        )
+
+        frames = [
+            frame
+            for frame in frames
+            if frame is not None
+        ]
+
+        return (
+            # TODO: Combinate them, I send first by now
+            frames[0]
+            if len(frames) > 0 else
+            # TODO: Should I send None or a full
+            # black (or transparent) frame? I think
+            # None is better because I don't know
+            # the size here (?)
+            None
+        )
+    
+    def render(
+        self,
+        filename: str,
+        start: float = 0.0,
+        end: Union[float, None] = None
+    ) -> 'Timeline':
+        """
+        Render the time range in between the given
+        'start' and 'end' and store the result with
+        the also provided 'fillename'.
+
+        If no 'start' and 'end' provided, the whole
+        project will be rendered.
+        """
+        ParameterValidator.validate_mandatory_string('filename', filename, do_accept_empty = False)
+        ParameterValidator.validate_mandatory_positive_number('start', start, do_include_zero = True)
+        ParameterValidator.validate_positive_number('end', end, do_include_zero = False)
+
+        # TODO: Limitate 'end' a bit...
+        end = (
+            self.end
+            if end is None else
+            end
+        )
+
+        if start >= end:
+            raise Exception('The provided "start" cannot be greater or equal to the "end" provided.')
+        # TODO: Obtain all the 't', based on 'fps'
+        # that we need to render from 'start' to
+        # 'end'
+        # TODO: I don't want to have this here
+        def generate_times(start: float, end: float, fps: int):
+            dt = 1.0 / fps
+            times = []
+
+            t = start
+            while t <= end:
+                times.append(t + 0.000001)
+                t += dt
+
+            return times
+
+        from yta_video_opengl.writer import VideoWriter
+
+        writer = VideoWriter('test_files/output_render.mp4')
+        # TODO: This has to be dynamic according to the
+        # video we are writing
+        writer.set_video_stream(
+            codec_name = 'h264',
+            fps = 60,
+            size = (1920, 1080),
+            pixel_format = 'yuv420p'
+        )
+        
+        for t in generate_times(start, end, self.fps):
+            frame = self.get_frame_at(t)
+
+            if frame is None:
+                # Replace with black background if no frame
+                frame = av.VideoFrame.from_ndarray(
+                    array = np.zeros((1920, 1080, 3), dtype = np.uint8),
+                    format = 'rgb24'
+                )
+
+            # We need to adjust our output elements to be
+            # consecutive and with the right values
+            # TODO: We are using int() for fps but its float...
+            frame.time_base = Fraction(1, int(self.fps))
+            frame.pts = int(t / frame.time_base)
+
+            # TODO: We need to handle the audio
+            writer.mux_video_frame(
+                frame = frame
+            )
+
+        writer.mux_video_frame(None)
+        writer.output.close()

yta_video_opengl/complete/track.py

@@ -0,0 +1,136 @@
+from yta_video_opengl.complete.video_on_track import VideoOnTrack
+from yta_video_opengl.video import Video
+from yta_validation.parameter import ParameterValidator
+from typing import Union
+
+
+# TODO: This is called Track but it is
+# handling videos only. Should I have
+# VideoTrack and AudioTrack (?)
+class Track:
+    """
+    Class to represent a track in which we place
+    videos, images and audio to build a video
+    project.
+    """
+
+    @property
+    def end(
+        self
+    ) -> float:
+        """
+        The end of the last video of this track,
+        which is also the end of the track. This
+        is the last time moment that has to be
+        rendered.
+        """
+        return (
+            0.0
+            if len(self.videos) == 0 else
+            max(
+                video.end
+                for video in self.videos
+            )
+        )
+
+    def __init__(
+        self
+    ):
+        self.videos: list[VideoOnTrack] = []
+        """
+        The list of 'VideoOnTrack' instances that
+        must play on this track.
+        """
+
+    def _is_free(
+        self,
+        start: float,
+        end: float
+    ) -> bool:
+        """
+        Check if the time range in between the 
+        'start' and 'end' time given is free or
+        there is some video playing at any moment.
+        """
+        return not any(
+            (
+                video.video.start < end and
+                video.video.end > start
+            )
+            for video in self.videos
+        )
+    
+    def _get_video_at_t(
+        self,
+        t: float
+    ) -> Union[VideoOnTrack, None]:
+        """
+        Get the video that is being played at
+        the 't' time moment provided.
+        """
+        for video in self.videos:
+            if video.start <= t < video.end:
+                return video
+            
+        return None
+    
+    def get_frame_at(
+        self,
+        t: float
+    ) -> Union['VideoFrame', None]:
+        """
+        Get the frame that must be displayed at
+        the 't' time moment provided, which is
+        a frame from the video that is being
+        played at that time moment.
+
+        Remember, this 't' time moment provided
+        is about the track, and we make the
+        conversion to the actual video 't' to
+        get the frame.
+        """
+        video = self._get_video_at_t(t)
+
+        return (
+            video.get_frame_at(t)
+            if video is not None else
+            None
+        )
+
+    def add_video(
+        self,
+        video: Video,
+        t: Union[float, None] = None
+    ) -> 'Track':
+        """
+        Add the 'video' provided to the track. If
+        a 't' time moment is provided, the video
+        will be added to that time moment if 
+        possible. If there is no other video 
+        placed in the time gap between the given
+        't' and the provided 'video' duration, it
+        will be added succesfully. In the other
+        case, an exception will be raised.
+
+        If 't' is None, the first available 't'
+        time moment will be used, that will be 0.0
+        if no video, or the end of the last video.
+        """
+        ParameterValidator.validate_mandatory_instance_of('video', video, Video)
+        ParameterValidator.validate_positive_float('t', t, do_include_zero = True)
+
+        if t is not None:
+            # TODO: We can have many different strategies
+            # that we could define in the '__init__' maybe
+            if not self._is_free(t, (t + video.end)):
+                raise Exception('The video cannot be added at the "t" time moment, something blocks it.')
+        else:
+            t = self.end
+        
+        self.videos.append(VideoOnTrack(
+            video,
+            t
+        ))
+
+        # TODO: Maybe return the VideoOnTrack instead (?)
+        return self

yta_video_opengl/complete/video_on_track.py

@@ -0,0 +1,100 @@
+"""
+If we have a video placed in a timeline,
+starting at the t=2s and the video lasts
+2 seconds, the `t` time range in which the
+video is playing is `[2s, 4s]`, so here 
+you have some examples with global `t` 
+values:
+- `t=1`, the video is not playing because
+it starts at `t=2`
+- `t=3`, the video is playing, it started
+at `t=2` and it has been playing during 1s
+- `t=5`, the video is not playing because
+it started at `t=2`, lasting 2s, so it
+finished at `t=4`
+"""
+from yta_video_opengl.video import Video
+from yta_validation.parameter import ParameterValidator
+from av.video.frame import VideoFrame
+from typing import Union
+
+
+class VideoOnTrack:
+    """
+    A video in the timeline.
+    """
+
+    @property
+    def end(
+        self
+    ) -> float:
+        """
+        The end time moment 't' of the video once
+        once its been placed on the track, which
+        is affected by the video duration and its
+        start time moment on the track.
+
+        This end is different from the video end.
+        """
+        return self.start + self.video.duration
+
+    def __init__(
+        self,
+        video: Video,
+        start: float = 0.0
+    ):
+        ParameterValidator.validate_mandatory_instance_of('video', video, Video)
+        ParameterValidator.validate_mandatory_positive_number('start', start, do_include_zero = True)
+
+        self.video: Video = video
+        """
+        The video source, with all its properties,
+        that is placed in the timeline.
+        """
+        self.start: float = float(start)
+        """
+        The time moment in which the video should
+        start playing, within the timeline.
+
+        This is the time respect to the timeline
+        and its different from the video `start`
+        time, which is related to the file.
+        """
+
+    def _get_video_t(
+        self,
+        t: float
+    ) -> float:
+        """
+        The video 't' time moment for the given
+        global 't' time moment. This 't' is the one
+        to use inside the video content to display
+        its frame.
+        """
+        return t - self.start
+
+    def is_playing(
+        self,
+        t: float
+    ) -> bool:
+        """
+        Check if this video is playing at the general
+        't' time moment, which is a global time moment
+        for the whole project.
+        """
+        return self.start <= t < self.end
+
+    def get_frame_at(
+        self,
+        t: float
+    ) -> Union[VideoFrame, None]:
+        """
+        Get the frame for the 't' time moment provided,
+        that could be None if the video is not playing
+        in that moment.
+        """
+        return (
+            self.video.reader.get_frame_from_t(self._get_video_t(t))
+            if self.is_playing(t) else
+            None
+        )

yta_video_opengl/nodes/__init__.py

@@ -0,0 +1,119 @@
+from yta_validation.parameter import ParameterValidator
+from typing import Union
+from abc import ABC, abstractmethod
+
+import av
+import moderngl
+
+
+class Node(ABC):
+    """
+    Base class to represent a node, which
+    is an entity that processes frames
+    individually.
+
+    This class must be inherited by any 
+    video or audio node class.
+    """
+
+    # TODO: What about the types?
+    @abstractmethod
+    def process(
+        frame: Union[av.VideoFrame, av.AudioFrame, moderngl.Texture],
+        t: float
+        # TODO: Maybe we need 'fps' and 'number_of_frames'
+        # to calculate progressions or similar...
+    ) -> Union[av.VideoFrame, av.AudioFrame, moderngl.Texture]:
+        pass
+
+class TimedNode:
+    """
+    Class to represent a Node wrapper to
+    be able to specify the time range in
+    which we want the node to be applied.
+
+    If the 't' time moment is not inside
+    this range, the frame will be returned
+    as it is, with no change.
+
+    A 't' time moment inside the range has
+    this condition:
+    - `start <= t < end`
+
+    We are not including the end because
+    the next TimedNode could start on that
+    specific value, and remember that the
+    first time moment is 0.
+
+    This is the class that has to be applied
+    when working with videos and not a Node
+    directly.
+
+    The 'start' and 'end' values by default
+    """
+
+    def __init__(
+        self,
+        node: Node,
+        start: float = 0.0,
+        end: Union[float, None] = None
+    ):
+        ParameterValidator.validate_mandatory_positive_number('start', start, do_include_zero = True)
+        ParameterValidator.validate_positive_number('end', end, do_include_zero = False)
+
+        if (
+            end is not None and
+            end < start
+        ):
+            raise Exception('The "end" parameter provided must be greater or equal to the "start" parameter.')
+
+        self.node: Node = node
+        """
+        The node we are wrapping and we want to
+        apply as a modification of the frame in
+        which we are in a 't' time moment.
+        """
+        self.start: float = start
+        """
+        The 't' time moment in which the Node must
+        start being applied (including it).
+        """
+        self.end: Union[float, None] = end
+        """
+        The 't' time moment in which the Node must
+        stop being applied (excluding it).
+        """
+
+    def is_within_time(
+        self,
+        t: float
+    ) -> bool:
+        """
+        Flag to indicate if the 't' time moment provided
+        is in the range of this TimedNode instance, 
+        which means that it fits this condition:
+        - `start <= t < end`
+        """
+        return (
+            self.start <= t < self.end
+            if self.end is not None else
+            self.start <= t
+        )
+
+    def process(
+        self,
+        frame: Union[av.VideoFrame, av.AudioFrame, moderngl.Texture],
+        t: float
+        # TODO: Maybe we need 'fps' and 'number_of_frames'
+        # to calculate progressions or similar...
+    ) -> Union['VideoFrame', 'AudioFrame', 'Texture']:
+        """
+        Process the frame if the provided 't' time
+        moment is in the range of this TimedNode
+        instance.
+        """
+        return (
+            self.node.process(frame, t)
+            if self.is_within_time(t) else
+            frame
+        )

yta_video_opengl/nodes/audio/__init__.py

@@ -0,0 +1,115 @@
+"""
+When working with audio frames, we don't need
+to use the GPU because audios are 1D and the
+information can be processed perfectly with
+a library like numpy.
+
+If we need a very intense calculation for an
+audio frame (FFT, convolution, etc.) we can
+use CuPy or some DPS specific libraries, but
+90% is perfectly done with numpy.
+
+If you want to modify huge amounts of audio
+(some seconds at the same time), you can use
+CuPy, that has the same API as numpy but
+working in GPU. Doing this below most of the
+changes would work:
+- `import numpy as np` → `import cupy as np`
+"""
+from yta_video_opengl.nodes import TimedNode
+from abc import abstractmethod
+from typing import Union
+
+import numpy as np
+import av
+
+
+class AudioNode:
+    """
+    Base audio node class to implement a
+    change in an audio frame by using the
+    numpy library.
+    """
+
+    @abstractmethod
+    def process(
+        self,
+        frame: av.AudioFrame,
+        t: float
+    ):
+        """
+        Process the provided audio 'frame' that
+        is played on the given 't' time moment.
+        """
+        pass
+
+"""
+Here you have an example. The 'private'
+node class is the modifier, that we don't
+want to expose, and the 'public' class is
+the one that inherits from TimedNode and
+wraps the 'private' class to build the
+functionality.
+"""
+class VolumeAudioNode(TimedNode):
+    """
+    TimedNode to set the audio volume of a video
+    in a specific frame.
+    """
+
+    def __init__(
+        self,
+        factor_fn,
+        start: float = 0.0,
+        end: Union[float, None] = None
+    ):
+        super().__init__(
+            node = _SetVolumeAudioNode(factor_fn),
+            start = start,
+            end = end
+        )
+
+class _SetVolumeAudioNode(AudioNode):
+    """
+    Audio node to change the volume of an
+    audio frame.
+    """
+
+    def __init__(
+        self,
+        factor_fn
+    ):
+        """
+        factor_fn: function (t, index) -> factor volumen
+        """
+        self.factor_fn = factor_fn
+
+    def process(
+        self,
+        frame: av.AudioFrame,
+        t: float,
+    ) -> av.AudioFrame:
+        # TODO: Why index (?) Maybe 'total_frames'
+        factor = self.factor_fn(t, 0)
+
+        samples = frame.to_ndarray().astype(np.float32)
+        samples *= factor
+
+        # Determine dtype according to format
+        samples = (
+            samples.astype(np.int16)
+            # 'fltp', 's16', 's16p'
+            if 's16' in frame.format.name else
+            samples.astype(np.float32)
+        )
+
+        new_frame = av.AudioFrame.from_ndarray(
+            samples,
+            format = frame.format.name,
+            layout = frame.layout.name
+        )
+        new_frame.sample_rate = frame.sample_rate
+        new_frame.pts = frame.pts
+        new_frame.time_base = frame.time_base
+
+        return new_frame

yta_video_opengl/nodes/video/__init__.py

@@ -0,0 +1,5 @@
+"""
+Working with video frames has to be done
+with nodes that use OpenGL because this
+way, by using the GPU, is the best way.
+"""