yta-video-opengl 0.0.3__tar.gz → 0.0.5__tar.gz

{yta_video_opengl-0.0.3 → yta_video_opengl-0.0.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: yta-video-opengl
-Version: 0.0.3
+Version: 0.0.5
 Summary: Youtube Autonomous Video OpenGL Module
 Author: danialcala94
 Author-email: danielalcalavalera@gmail.com

{yta_video_opengl-0.0.3 → yta_video_opengl-0.0.5}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "yta-video-opengl"
-version = "0.0.3"
+version = "0.0.5"
 description = "Youtube Autonomous Video OpenGL Module"
 authors = [
     {name = "danialcala94",email = "danielalcalavalera@gmail.com"}

yta_video_opengl-0.0.5/src/yta_video_opengl/reader.py

@@ -0,0 +1,390 @@
+"""
+A video reader using the PyAv (av) library
+that, using ffmpeg, detects the video.
+"""
+from yta_validation import PythonValidator
+from av.video.frame import VideoFrame
+from av.audio.frame import AudioFrame
+from av.packet import Packet
+from av.video.stream import VideoStream
+from av.audio.stream import AudioStream
+from av.container.input import InputContainer
+from fractions import Fraction
+from av import open as av_open
+from typing import Union
+from dataclasses import dataclass
+
+
+@dataclass
+class VideoReaderFrame:
+    """
+    Class to wrap a frame of a video that is
+    being read, that can be a video or audio
+    frame, and has been decoded.
+    """
+
+    @property
+    def is_video(
+        self
+    ):
+        """
+        Flag to indicate if the instance is a video
+        frame.
+        """
+        return PythonValidator.is_instance_of(self.data, VideoFrame)
+    
+    @property
+    def is_audio(
+        self
+    ):
+        """
+        Flag to indicate if the instance is an audio
+        frame.
+        """
+        return PythonValidator.is_instance_of(self.data, AudioFrame)
+    
+    def __init__(
+        self,
+        # TODO: Add the type, please
+        data: any
+    ):
+        self.data: Union[AudioFrame, VideoFrame] = data
+        """
+        The frame content, that can be audio or video
+        frame.
+        """
+
+@dataclass
+class VideoReaderPacket:
+    """
+    Class to wrap a packet of a video that is
+    being read, that can contain video or audio
+    frames.
+    """
+
+    @property
+    def is_video(
+        self
+    ) -> bool:
+        """
+        Flag to indicate if the packet includes video
+        frames or not.
+        """
+        return self.data.stream.type == 'video'
+    
+    @property
+    def is_audio(
+        self
+    ) -> bool:
+        """
+        Flag to indicate if the packet includes audio
+        frames or not.
+        """
+        return self.data.stream.type == 'audio'
+
+    def __init__(
+        self,
+        data: Packet
+    ):
+        self.data: Packet = data
+        """
+        The packet, that can include video or audio
+        frames and can be decoded.
+        """
+
+    def decode(
+        self
+    ) -> list['SubtitleSet']:
+        """
+        Get the frames but decoded, perfect to make
+        modifications and encode to save them again.
+        """
+        return self.data.decode()
+        
+
+class VideoReader:
+    """
+    Class to read video files with the PyAv (av)
+    library that uses ffmpeg on the background.
+    """
+
+    @property
+    def frame_iterator(
+        self
+    ) -> 'Iterator[VideoFrame]':
+        """
+        Iterator to iterate over all the video frames
+        decodified.
+        """
+        return self.container.decode(self.video_stream)
+    
+    @property
+    def next_frame(
+        self
+    ) -> Union[VideoFrame, None]:
+        """
+        Get the next video frame (decoded) from the
+        iterator.
+        """
+        return next(self.frame_iterator)
+
+    @property
+    def audio_frame_iterator(
+        self
+    ) -> 'Iterator[AudioFrame]':
+        """
+        Iterator to iterate over all the audio frames
+        decodified.
+        """
+        return self.container.decode(self.audio_stream)
+        
+    @property
+    def next_audio_frame(
+        self
+    ) -> Union[AudioFrame, None]:
+        """
+        Get the next audio frame (decoded) from the
+        iterator.
+        """
+        return next(self.audio_frame_iterator)
+    
+    @property
+    def packet_iterator(
+        self
+    ) -> 'Iterator[Packet]':
+        """
+        Iterator to iterate over all the video frames
+        as packets (not decodified).
+        """
+        return self.container.demux(self.video_stream)
+    
+    @property
+    def next_packet(
+        self
+    ) -> Union[Packet, None]:
+        """
+        Get the next video packet (not decoded) from
+        the iterator.
+        """
+        return next(self.packet_iterator)
+    
+    @property
+    def audio_packet_iterator(
+        self
+    ) -> 'Iterator[Packet]':
+        """
+        Iterator to iterate over all the audio frames
+        as packets (not decodified).
+        """
+        return self.container.demux(self.audio_stream)
+    
+    @property
+    def next_audio_packet(
+        self
+    ) -> Union[Packet, None]:
+        """
+        Get the next audio packet (not decoded) from
+        the iterator.
+        """
+        return next(self.packet_iterator)
+    
+    @property
+    def packet_with_audio_iterator(
+        self
+    ) -> 'Iterator[Packet]':
+        """
+        Iterator to iterate over all the video frames
+        as packets (not decodified) including also the
+        audio as packets.
+        """
+        return self.container.demux((self.video_stream, self.audio_stream))
+
+    @property
+    def next_packet_with_audio(
+        self
+    ) -> Union[Packet, None]:
+        """
+        Get the next video frames packet (or audio
+        frames packet) from the iterator. Depending
+        on the position, the packet can be video or
+        audio.
+        """
+        return next(self.packet_with_audio_iterator)
+
+    @property
+    def codec_name(
+        self
+    ) -> str:
+        """
+        Get the name of the video codec.
+        """
+        return self.video_stream.codec_context.name
+    
+    @property
+    def audio_codec_name(
+        self
+    ) -> str:
+        """
+        Get the name of the audio codec.
+        """
+        return self.audio_stream.codec_context.name
+
+    @property
+    def number_of_frames(
+        self
+    ) -> int:
+        """
+        The number of frames in the video.
+        """
+        return self.video_stream.frames
+    
+    @property
+    def number_of_audio_frames(
+        self
+    ) -> int:
+        """
+        The number of frames in the audio.
+        """
+        return self.audio_stream.frames
+    
+    @property
+    def fps(
+        self
+    ) -> Fraction:
+        """
+        The fps of the video.
+        """
+        # They return it as a Fraction but...
+        return self.video_stream.average_rate
+    
+    @property
+    def audio_fps(
+        self
+    ) -> Fraction:
+        """
+        The fps of the audio.
+        """
+        # TODO: What if no audio (?)
+        return self.audio_stream.average_rate
+    
+    @property
+    def size(
+        self
+    ) -> tuple[int, int]:
+        """
+        The size of the video in a (width, height) format.
+        """
+        return (
+            self.video_stream.width,
+            self.video_stream.height
+        )
+    
+    @property
+    def width(
+        self
+    ) -> int:
+        """
+        The width of the video, in pixels.
+        """
+        return self.size[0]
+    
+    @property
+    def height(
+        self
+    ) -> int:
+        """
+        The height of the video, in pixels.
+        """
+        return self.size[1]
+    
+    # Any property related to audio has to
+    # start with 'audio_property_name'
+
+    def __init__(
+        self,
+        filename: str
+    ):
+        self.filename: str = filename
+        """
+        The filename of the video source.
+        """
+        self.container: InputContainer = av_open(filename)
+        """
+        The av input general container of the
+        video (that also includes the audio) we
+        are reading.
+        """
+        self.video_stream: VideoStream = self.container.streams.video[0]
+        """
+        The stream that includes the video.
+        """
+        self.video_stream.thread_type = 'AUTO'
+        # TODO: What if no audio (?)
+        self.audio_stream: AudioStream = self.container.streams.audio[0]
+        """
+        The stream that includes the audio.
+        """
+        self.audio_stream.thread_type = 'AUTO'
+
+    def iterate(
+        self
+    ) -> 'Iterator[Union[VideoFrame, AudioFrame]]':
+        """
+        Iterator to iterate over the video frames
+        (already decoded).
+        """
+        for frame in self.frame_iterator:
+            yield VideoReaderFrame(frame)
+
+    def iterate_with_audio(
+        self,
+        do_decode_video: bool = True,
+        do_decode_audio: bool = False
+    ) -> 'Iterator[Union[VideoReaderFrame, VideoReaderPacket, None]]':
+        """
+        Iterator to iterate over the video and audio
+        packets, decoded only if the parameters are
+        set as True.
+
+        If the packet is decoded, it will return each
+        frame individually as a VideoReaderFrame 
+        instance. If not, the whole packet as a
+        VideoReaderPacket instance.
+        """
+        for packet in self.packet_with_audio_iterator:
+            is_video = packet.stream.type == 'video'
+
+            do_decode = (
+                (
+                    is_video and
+                    do_decode_video
+                ) or
+                (
+                    not is_video and
+                    do_decode_audio
+                )
+            )
+
+            if do_decode:
+                for frame in packet.decode():
+                    # Return each frame decoded
+                    yield VideoReaderFrame(frame)
+            else:
+                # Return the packet as it is
+                yield VideoReaderPacket(packet)
+
+
+
+
+"""
+When reading packets directly from the stream
+we can receive packets with size=0, but we need
+to process them and decode (or yield them). It
+is only when we are passing packets to the mux
+when we need to ignore teh ones thar are empty
+(size=0).
+
+TODO: Do we need to ignore all? By now, ignoring
+not is causing exceptions, and ignoring them is
+making it work perfectly.
+"""

yta_video_opengl-0.0.5/src/yta_video_opengl/tests.py

@@ -0,0 +1,179 @@
+"""
+Manual tests that are working and are interesting
+to learn about the code, refactor and build
+classes.
+"""
+from yta_validation import PythonValidator
+from yta_video_opengl.reader import VideoReader
+from yta_video_opengl.writer import VideoWriter
+from yta_timer import Timer
+from yta_video_frame_time import T
+
+import av
+import moderngl
+import numpy as np
+
+
+def video_modified_stored():
+    VIDEO_PATH = "test_files/test_1.mp4"
+    OUTPUT_PATH = "test_files/output.mp4"
+    AMP = 0.05
+    FREQ = 10.0
+    SPEED = 2.0
+
+    # ModernGL context without window
+    context = moderngl.create_standalone_context()
+
+    # Wave shader vertex and fragment
+    program = context.program(
+        vertex_shader = '''
+        #version 330
+        in vec2 in_pos;
+        in vec2 in_uv;
+        out vec2 v_uv;
+        void main() {
+            v_uv = in_uv;
+            gl_Position = vec4(in_pos, 0.0, 1.0);
+        }
+        ''',
+        fragment_shader = '''
+        #version 330
+        uniform sampler2D tex;
+        uniform float time;
+        uniform float amp;
+        uniform float freq;
+        uniform float speed;
+        in vec2 v_uv;
+        out vec4 f_color;
+        void main() {
+            float wave = sin(v_uv.x * freq + time * speed) * amp;
+            vec2 uv = vec2(v_uv.x, v_uv.y + wave);
+            f_color = texture(tex, uv);
+        }
+        '''
+    )
+
+    # Quad
+    vertices = np.array([
+        -1, -1, 0.0, 0.0,
+        1, -1, 1.0, 0.0,
+        -1,  1, 0.0, 1.0,
+        1,  1, 1.0, 1.0,
+    ], dtype = 'f4')
+    vbo = context.buffer(vertices.tobytes())
+    vao = context.simple_vertex_array(program, vbo, 'in_pos', 'in_uv')
+
+    video = VideoReader(VIDEO_PATH)
+
+    # TODO: This has to be dynamic, but
+    # according to what (?)
+    NUMPY_FORMAT = 'rgb24'
+    # TODO: Where do we obtain this from (?)
+    VIDEO_CODEC_NAME = 'libx264'
+    # TODO: Where do we obtain this from (?)
+    PIXEL_FORMAT = 'yuv420p'
+
+    # Framebuffer to render
+    fbo = context.simple_framebuffer(video.size)
+    fbo.use()
+
+    # Decode first frame and use as texture
+    first_frame = video.next_frame
+    # We need to reset it to being again pointing
+    # to the first frame...
+    # TODO: Improve this by, maybe, storing the first
+    # frame in memory so we can append it later, or
+    # using the '.seek(0)' even when it could be not
+    # accurate
+    video = VideoReader(VIDEO_PATH)
+
+    # Most of OpenGL textures expect origin in lower
+    # left corner
+    # TODO: What if alpha (?)
+    image = np.flipud(first_frame.to_ndarray(format = NUMPY_FORMAT))
+    texture = context.texture((image.shape[1], image.shape[0]), 3, image.tobytes())
+
+    texture.build_mipmaps()
+
+    # Uniforms
+    program['amp'].value = AMP
+    program['freq'].value = FREQ
+    program['speed'].value = SPEED
+
+    # Writer with H.264 codec
+    video_writer = (
+        VideoWriter(OUTPUT_PATH)
+        .set_video_stream(VIDEO_CODEC_NAME, video.fps, video.size, PIXEL_FORMAT)
+        .set_audio_stream_from_template(video.audio_stream)
+    )
+
+    frame_index = 0
+    for frame_or_packet in video.iterate_with_audio(
+        do_decode_video = True,
+        do_decode_audio = False
+    ):
+        # This below is because of the parameters we
+        # passed to the method
+        is_video_frame = PythonValidator.is_instance_of(frame_or_packet, 'VideoReaderFrame')
+        is_audio_packet = PythonValidator.is_instance_of(frame_or_packet, 'VideoReaderPacket')
+
+        # To simplify the process
+        if frame_or_packet is not None:
+            frame_or_packet = frame_or_packet.data
+
+        if is_audio_packet:
+            video_writer.mux(frame_or_packet)
+        elif is_video_frame:
+            with Timer(is_silent_as_context = True) as timer:
+
+                def process_frame(
+                    frame: 'VideoFrame'
+                ):
+                    # Add some variables if we need, for the
+                    # opengl change we are applying (check the
+                    # program code)
+                    program['time'].value = T.video_frame_index_to_video_frame_time(frame_index, float(video.fps))
+                    
+                    # To numpy RGB inverted for OpenGL
+                    img_array = np.flipud(
+                        frame.to_ndarray(format = NUMPY_FORMAT)
+                    )
+
+                    # Create texture
+                    texture = context.texture((img_array.shape[1], img_array.shape[0]), 3, img_array.tobytes())
+                    texture.use()
+
+                    # Render with shader to frame buffer
+                    fbo.use()
+                    vao.render(moderngl.TRIANGLE_STRIP)
+
+                    # Processed GPU result to numpy
+                    processed_data = np.frombuffer(
+                        fbo.read(components = 3, alignment = 1), dtype = np.uint8
+                    )
+                    # Invert numpy to normal frame
+                    processed_data = np.flipud(
+                        processed_data.reshape((img_array.shape[0], img_array.shape[1], 3))
+                    )
+
+                    # To VideoFrame and to buffer
+                    frame = av.VideoFrame.from_ndarray(processed_data, format = NUMPY_FORMAT)
+                    # TODO: What is this for (?)
+                    #out_frame.pict_type = 'NONE'
+                    return frame
+
+                video_writer.mux_video_frame(process_frame(frame_or_packet))
+
+            print(f'Frame {str(frame_index)}: {timer.time_elapsed_str}s')
+            frame_index += 1
+
+    # While this code can be finished, the work in
+    # the muxer could be not finished and have some
+    # packets waiting to be written. Here we tell
+    # the muxer to process all those packets.
+    video_writer.mux_video_frame(None)
+
+    # TODO: Maybe move this to the '__del__' (?)
+    video_writer.output.close()
+    video.container.close()
+    print(f'Saved as "{OUTPUT_PATH}".')