yta-video-opengl 0.0.6__tar.gz → 0.0.7__tar.gz

{yta_video_opengl-0.0.6 → yta_video_opengl-0.0.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: yta-video-opengl
-Version: 0.0.6
+Version: 0.0.7
 Summary: Youtube Autonomous Video OpenGL Module
 Author: danialcala94
 Author-email: danielalcalavalera@gmail.com

{yta_video_opengl-0.0.6 → yta_video_opengl-0.0.7}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "yta-video-opengl"
-version = "0.0.6"
+version = "0.0.7"
 description = "Youtube Autonomous Video OpenGL Module"
 authors = [
     {name = "danialcala94",email = "danielalcalavalera@gmail.com"}

{yta_video_opengl-0.0.6 → yta_video_opengl-0.0.7}/src/yta_video_opengl/classes.py

@@ -8,7 +8,8 @@ so we use different triangles to build
 our shapes (quad normally).
 """
 from yta_validation.parameter import ParameterValidator
-from yta_video_opengl.utils import frame_to_texture
+from yta_validation import PythonValidator
+from yta_video_opengl.utils import frame_to_texture, get_fullscreen_quad_vao
 from abc import ABC, abstractmethod
 from typing import Union
 
@@ -24,29 +25,48 @@ class _Uniforms:
     """
 
     @property
-    def program(
+    def uniforms(
         self
-    ):
+    ) -> dict:
         """
-        Shortcut to the FrameShader program.
+        The uniforms in the program, as a dict, in
+        the format `{key, value}`.
         """
-        return self._shader_instance.program
+        return {
+            key: self.program[key].value
+            for key in self.program
+            if PythonValidator.is_instance_of(self.program[key], moderngl.Uniform)
+        }
 
     def __init__(
         self,
-        shader_instance: 'FrameShaderBase'
+        program: moderngl.Program
     ):
-        self._shader_instance: 'FrameShaderBase' = shader_instance
+        self.program: moderngl.Program = program
+        """
+        The program instance this handler class
+        belongs to.
+        """
+
+    def get(
+        self,
+        name: str
+    ) -> Union[any, None]:
         """
-        The instance of the FrameShader class these
-        uniforms belong to.
+        Get the value of the uniform with the
+        given 'name'.
         """
+        return self.uniforms.get(name, None)
 
+    # TODO: I need to refactor these method to
+    # accept a **kwargs maybe, or to auto-detect
+    # the type and add the uniform as it must be
+    # done
     def set(
         self,
         name: str,
         value
-    ) -> 'FrameShaderBase':
+    ) -> '_Uniforms':
         """
         Set the provided 'value' to the normal type
         uniform with the given 'name'. Here you have
@@ -59,13 +79,13 @@ class _Uniforms:
         if name in self.program:
             self.program[name].value = value
 
-        return self._shader_instance
+        return self
     
     def set_vec(
         self,
         name: str,
         values
-    ) -> 'FrameShaderBase':
+    ) -> '_Uniforms':
         """
         Set the provided 'value' to the normal type
         uniform with the given 'name'. Here you have
@@ -78,11 +98,13 @@ class _Uniforms:
         if name in self.program:
             self.program[name].write(np.array(values, dtype = 'f4').tobytes())
 
+        return self
+
     def set_mat(
         self,
         name: str,
         value
-    ) -> 'FrameShaderBase':
+    ) -> '_Uniforms':
         """
         Set the provided 'value' to a `matN` type
         uniform with the given 'name'. The 'value'
@@ -99,16 +121,26 @@ class _Uniforms:
         if name in self.program:
             self.program[name].write(value)
 
-        return self._shader_instance
+        return self
+    
+    def print(
+        self
+    ) -> '_Uniforms':
+        """
+        Print the defined uniforms in console.
+        """
+        for key, value in self.uniforms.items():
+            print(f'"{key}": {str(value)}')
 
-class FrameShaderBase(ABC):
+class BaseNode:
     """
-    Class to be inherited by any of our own
-    custom opengl program classes.
+    The basic class of a node to manipulate frames
+    as opengl textures. This node will process the
+    frame as an input texture and will generate 
+    also a texture as the output.
 
-    This shader base class must be used by all
-    the classes that are modifying the frames
-    one by one.
+    Nodes can be chained and the result from one
+    node can be applied on another node.
     """
 
     @property
@@ -117,7 +149,7 @@ class FrameShaderBase(ABC):
         self
     ) -> str:
         """
-        Source code of the vertex shader.
+        The code of the vertex shader.
         """
         pass
 
@@ -127,47 +159,211 @@ class FrameShaderBase(ABC):
         self
     ) -> str:
         """
-        Source code of the fragment shader.
+        The code of the fragment shader.
         """
         pass
 
+    def __init__(
+        self,
+        context: moderngl.Context,
+        size: tuple[int, int],
+        **kwargs
+    ):
+        ParameterValidator.validate_mandatory_instance_of('context', context, moderngl.Context)
+        # TODO: Validate size
+
+        self.context: moderngl.Context = context
+        """
+        The context of the program.
+        """
+        self.size: tuple[int, int] = size
+        """
+        The size we want to use for the frame buffer
+        in a (width, height) format.
+        """
+        # Compile shaders within the program
+        self.program: moderngl.Program = self.context.program(
+            vertex_shader = self.vertex_shader,
+            fragment_shader = self.fragment_shader
+        )
+
+        # Create the fullscreen quad
+        self.quad = get_fullscreen_quad_vao(
+            context = self.context,
+            program = self.program
+        )
+
+        # Create the output fbo
+        self.output_tex = self.context.texture(self.size, 4)
+        self.output_tex.filter = (moderngl.LINEAR, moderngl.LINEAR)
+        self.fbo = self.context.framebuffer(color_attachments = [self.output_tex])
+
+        self.uniforms: _Uniforms = _Uniforms(self.program)
+        """
+        Shortcut to the uniforms functionality.
+        """
+        # Auto set uniforms dynamically if existing
+        for key, value in kwargs.items():
+            self.uniforms.set(key, value)
+
+    def process(
+        self,
+        input: Union[moderngl.Texture, 'VideoFrame', 'np.ndarray']
+    ) -> moderngl.Texture:
+        """
+        Apply the shader to the 'input', that
+        must be a frame or a texture, and return
+        the new resulting texture.
+
+        We use and return textures to maintain
+        the process in GPU and optimize it.
+        """
+        # TODO: Maybe we can accept a VideoFrame
+        # or a numpy array and transform it here
+        # into a texture, ready to be used:
+        # frame_to_texture(
+        #     # TODO: Do not use Pillow
+        #     frame = np.array(Image.open("input.jpg").convert("RGBA")),
+        #     context = self.context,
+        #     numpy_format = 'rgba'
+        # )
+        if PythonValidator.is_instance_of(input, ['VideoFrame', 'ndarray']):
+            # TODO: What about the numpy format (?)
+            input = frame_to_texture(input, self.context)
+
+        self.fbo.use()
+        self.context.clear(0.0, 0.0, 0.0, 0.0)
+
+        input.use(location = 0)
+
+        if 'texture' in self.program:
+            self.program['texture'] = 0
+
+        self.quad.render()
+
+        return self.output_tex
+    
+class WavingNode(BaseNode):
+    """
+    Just an example, without the shaders code
+    actually, to indicate that we can use
+    custom parameters to make it work.
+    """
+
     @property
-    def vertices(
+    def vertex_shader(
         self
-    ) -> 'np.ndarray':
+    ) -> str:
+        return (
+            '''
+            #version 330
+            in vec2 in_vert;
+            in vec2 in_texcoord;
+            out vec2 v_uv;
+            void main() {
+                v_uv = in_texcoord;
+                gl_Position = vec4(in_vert, 0.0, 1.0);
+            }
+            '''
+        )
+
+    @property
+    def fragment_shader(
+        self
+    ) -> str:
+        return (
+            '''
+            #version 330
+            uniform sampler2D tex;
+            uniform float time;
+            uniform float amplitude;
+            uniform float frequency;
+            uniform float speed;
+            in vec2 v_uv;
+            out vec4 f_color;
+            void main() {
+                float wave = sin(v_uv.x * frequency + time * speed) * amplitude;
+                vec2 uv = vec2(v_uv.x, v_uv.y + wave);
+                f_color = texture(tex, uv);
+            }
+            '''
+        )
+
+    def __init__(
+        self,
+        context: moderngl.Context,
+        size: tuple[int, int],
+        amplitude: float = 0.05,
+        frequency: float = 10.0,
+        speed: float = 2.0
+    ):
+        super().__init__(
+            context = context,
+            size = size,
+            amplitude = amplitude,
+            frequency = frequency,
+            speed = speed
+        )
+
+    # This is just an example and we are not
+    # using the parameters actually, but we
+    # could set those specific uniforms to be
+    # processed by the code
+    def process(
+        self,
+        input: Union[moderngl.Texture, 'VideoFrame', 'np.ndarray'],
+        t: float = 0.0,
+    ) -> moderngl.Texture:
+        """
+        Apply the shader to the 'input', that
+        must be a frame or a texture, and return
+        the new resulting texture.
+
+        We use and return textures to maintain
+        the process in GPU and optimize it.
+        """
+        self.uniforms.set('time', t)
+
+        return super().process(input)
+
+
+
+"""
+TODO: I should try to use the Node classes
+to manipulate the frames because this is how
+Davinci Resolve and other editors work.
+"""
+
+
+class FrameShaderBase(ABC):
+    """
+    Class to be inherited by any of our own
+    custom opengl program classes.
+
+    This shader base class must be used by all
+    the classes that are modifying the frames
+    one by one.
+    """
+
+    @property
+    @abstractmethod
+    def vertex_shader(
+        self
+    ) -> str:
         """
-        The UV coordinates to build the quad we
-        will use to represent the frame by 
-        applying it as a texture.
+        Source code of the vertex shader.
         """
-        return np.array([
-            # vertex 0 - bottom left
-            -1.0, -1.0,  0.0, 0.0,
-            # vertex 1 - bottom right
-            1.0, -1.0,  1.0, 0.0,
-            # vertex 2 - top left
-            -1.0,  1.0,  0.0, 1.0,
-            # vertex 3 - top right
-            1.0,  1.0,  1.0, 1.0
-        ], dtype = 'f4')
+        pass
 
     @property
-    def indexes(
+    @abstractmethod
+    def fragment_shader(
         self
-    ) -> 'np.ndarray':
+    ) -> str:
         """
-        The indexes of the vertices (see 'vertices'
-        property) to build the 2 opengl triangles
-        that will represent the quad we need for
-        the frame.
+        Source code of the fragment shader.
         """
-        return np.array(
-            [
-                0, 1, 2,
-                2, 1, 3
-            ],
-            dtype = 'i4'
-        )
+        pass
 
     def __init__(
         self,
@@ -203,14 +399,6 @@ class FrameShaderBase(ABC):
         """
         The frame buffer object.
         """
-        self.vbo: moderngl.Buffer = None
-        """
-        The vertices buffer object.
-        """
-        self.ibo: moderngl.Buffer = None
-        """
-        The indexes buffer object.
-        """
         self.uniforms: _Uniforms = None
         """
         Shortcut to the uniforms functionality.
@@ -221,6 +409,12 @@ class FrameShaderBase(ABC):
     def _initialize_program(
         self
     ):
+        """
+        This method is to allow the effects to
+        change their '__init__' method to be able
+        to provide parameters that will be set as
+        uniforms.
+        """
         # Compile shaders within the program
         self.program: moderngl.Program = self.context.program(
             vertex_shader = self.vertex_shader,
@@ -229,18 +423,9 @@ class FrameShaderBase(ABC):
 
         # Create frame buffer
         self.fbo = self.context.simple_framebuffer(self.size)
-
-        # Create buffers
-        # TODO: I could have more than 1 vertices,
-        # so more than 1 vbo and more than 1 vao...
-        self.vbo: moderngl.Buffer = self.context.buffer(self.vertices.tobytes())
-        self.ibo: moderngl.Buffer = self.context.buffer(self.indexes.tobytes())
-        vao_content = [
-            (self.vbo, "2f 2f", "in_vert", "in_texcoord")
-        ]
-        self.vao: moderngl.VertexArray = self.context.vertex_array(self.program, vao_content, self.ibo)
-
-        self.uniforms: _Uniforms = _Uniforms(self)
+        # Create quad vertex array
+        self.vao: moderngl.VertexArray = get_fullscreen_quad_vao(self.context, self.program)
+        self.uniforms: _Uniforms = _Uniforms(self.program)
 
         # TODO: How do I manage these textures (?)
         self.textures = {}
@@ -270,8 +455,7 @@ class FrameShaderBase(ABC):
         tex = self.context.texture((image.shape[1], image.shape[0]), 4, image.tobytes())
         tex.use(texture_unit)
         self.textures[uniform_name] = tex
-        if uniform_name in self.program:
-            self.program[uniform_name].value = texture_unit
+        self.uniforms.set(uniform_name, texture_unit)
 
     @abstractmethod
     def _prepare_frame(

{yta_video_opengl-0.0.6 → yta_video_opengl-0.0.7}/src/yta_video_opengl/reader/__init__.py

@@ -32,7 +32,7 @@ class VideoReaderFrame:
         Flag to indicate if the instance is a video
         frame.
         """
-        return PythonValidator.is_instance_of(self.data, VideoFrame)
+        return PythonValidator.is_instance_of(self.value, VideoFrame)
     
     @property
     def is_audio(
@@ -42,18 +42,37 @@ class VideoReaderFrame:
         Flag to indicate if the instance is an audio
         frame.
         """
-        return PythonValidator.is_instance_of(self.data, AudioFrame)
+        return PythonValidator.is_instance_of(self.value, AudioFrame)
+    
+    @property
+    def as_numpy(
+        self
+    ):
+        """
+        The frame as a numpy array.
+        """
+        return self.value.to_ndarray(format = self.pixel_format)
     
     def __init__(
         self,
         # TODO: Add the type, please
-        data: any
+        frame: any,
+        t: float = None,
+        pixel_format: str = 'rgb24'
     ):
-        self.data: Union[AudioFrame, VideoFrame] = data
+        self.value: Union[AudioFrame, VideoFrame] = frame
         """
         The frame content, that can be audio or video
         frame.
         """
+        self.t: float = t
+        """
+        The 't' time moment of the frame.
+        """
+        self.pixel_format: str = pixel_format
+        """
+        The pixel format of the frame.
+        """
 
 @dataclass
 class VideoReaderPacket:
@@ -71,7 +90,7 @@ class VideoReaderPacket:
         Flag to indicate if the packet includes video
         frames or not.
         """
-        return self.data.stream.type == 'video'
+        return self.value.stream.type == 'video'
     
     @property
     def is_audio(
@@ -81,13 +100,13 @@ class VideoReaderPacket:
         Flag to indicate if the packet includes audio
         frames or not.
         """
-        return self.data.stream.type == 'audio'
+        return self.value.stream.type == 'audio'
 
     def __init__(
         self,
-        data: Packet
+        packet: Packet
     ):
-        self.data: Packet = data
+        self.value: Packet = packet
         """
         The packet, that can include video or audio
         frames and can be decoded.
@@ -100,7 +119,7 @@ class VideoReaderPacket:
         Get the frames but decoded, perfect to make
         modifications and encode to save them again.
         """
-        return self.data.decode()
+        return self.value.decode()
         
 
 class VideoReader:
@@ -268,6 +287,24 @@ class VideoReader:
         # TODO: What if no audio (?)
         return self.audio_stream.average_rate
     
+    @property
+    def time_base(
+        self
+    ) -> Fraction:
+        """
+        The time base of the video.
+        """
+        return self.video_stream.time_base
+
+    @property
+    def audio_time_base(
+        self
+    ) -> Fraction:
+        """
+        The time base of the audio.
+        """
+        return self.audio_stream.time_base
+    
     @property
     def size(
         self
@@ -303,12 +340,18 @@ class VideoReader:
 
     def __init__(
         self,
-        filename: str
+        filename: str,
+        # Use 'rgba' if alpha channel
+        pixel_format: str = 'rgb24'
     ):
         self.filename: str = filename
         """
         The filename of the video source.
         """
+        self.pixel_format: str = pixel_format
+        """
+        The pixel format.
+        """
         self.container: InputContainer = None
         """
         The av input general container of the
@@ -370,7 +413,11 @@ class VideoReader:
         (already decoded).
         """
         for frame in self.frame_iterator:
-            yield VideoReaderFrame(frame)
+            yield VideoReaderFrame(
+                frame = frame,
+                t = float(frame.pts * self.time_base),
+                pixel_format = self.pixel_format
+            )
 
     def iterate_with_audio(
         self,
@@ -419,6 +466,14 @@ class VideoReader:
         the cache system.
         """
         return self.cache.get_frame(index)
+    
+    def close(
+        self
+    ) -> None:
+        """
+        Close the container to free it.
+        """
+        self.container.close()
 
 
 

{yta_video_opengl-0.0.6 → yta_video_opengl-0.0.7}/src/yta_video_opengl/tests.py

@@ -579,7 +579,8 @@ def video_modified_stored():
     # TODO: Where do we obtain this from (?)
     PIXEL_FORMAT = 'yuv420p'
 
-    from yta_video_opengl.classes import WavingFrame, BreathingFrame, HandheldFrame, OrbitingFrame, RotatingInCenterFrame, StrangeTvFrame, GlitchRgbFrame
+    from yta_video_opengl.classes import WavingFrame, BreathingFrame, HandheldFrame, OrbitingFrame, RotatingInCenterFrame, StrangeTvFrame, GlitchRgbFrame, WavingNode
+    from yta_video_opengl.utils import texture_to_frame, frame_to_texture
 
     video = VideoReader(VIDEO_PATH)
     video_writer = (
@@ -604,6 +605,10 @@ def video_modified_stored():
         size = video.size,
         first_frame = video.next_frame
     )
+    context = moderngl.create_context(standalone = True)
+    
+    # New way, with nodes
+    node = WavingNode(context, video.size, amplitude = 0.2, frequency = 9, speed = 3)
     # We need to reset it to being again pointing
     # to the first frame...
     # TODO: Improve this by, maybe, storing the first
@@ -624,7 +629,7 @@ def video_modified_stored():
 
         # To simplify the process
         if frame_or_packet is not None:
-            frame_or_packet = frame_or_packet.data
+            frame_or_packet = frame_or_packet.value
 
         if is_audio_packet:
             video_writer.mux(frame_or_packet)
@@ -632,13 +637,27 @@ def video_modified_stored():
             with Timer(is_silent_as_context = True) as timer:
                 t = T.video_frame_index_to_video_frame_time(frame_index, float(video.fps))
 
+                # This is another way of getting 't'
+                #t = float(frame_or_packet.pts * video.time_base)
+                
+                # TODO: Pass the frame as a texture
+                
                 video_writer.mux_video_frame(
-                    effect.process_frame(
-                        frame = frame_or_packet,
-                        t = t,
-                        numpy_format = NUMPY_FORMAT
+                    frame = texture_to_frame(
+                        texture = node.process(
+                            input = frame_or_packet,
+                            t = t
+                        )
                     )
                 )
+                
+                # video_writer.mux_video_frame(
+                #     effect.process_frame(
+                #         frame = frame_or_packet,
+                #         t = t,
+                #         numpy_format = NUMPY_FORMAT
+                #     )
+                # )
 
                 frame_index += 1
 

yta_video_opengl-0.0.7/src/yta_video_opengl/utils.py

@@ -0,0 +1,100 @@
+from yta_validation import PythonValidator
+from typing import Union
+
+import av
+import numpy as np
+import moderngl
+
+
+def frame_to_texture(
+    frame: Union['VideoFrame', 'np.ndarray'],
+    context: moderngl.Context,
+    numpy_format: str = 'rgb24'
+):
+    """
+    Transform the given 'frame' to an opengl
+    texture. The frame can be a VideoFrame
+    instance (from pyav library) or a numpy
+    array.
+    """
+    # To numpy RGB inverted for opengl
+    frame: np.ndarray = (
+        np.flipud(frame.to_ndarray(format = numpy_format))
+        if PythonValidator.is_instance_of(frame, 'VideoFrame') else
+        np.flipud(frame)
+    )
+
+    return context.texture(
+        size = (frame.shape[1], frame.shape[0]),
+        components = frame.shape[2],
+        data = frame.tobytes()
+    )
+
+# TODO: I should make different methods to
+# obtain a VideoFrame or a numpy array frame
+def texture_to_frame(
+    texture: moderngl.Texture
+) -> 'VideoFrame':
+    """
+    Transform an opengl texture into a pyav
+    VideoFrame instance.
+    """
+    # RGBA8
+    data = texture.read(alignment = 1)
+    frame = np.frombuffer(data, dtype = np.uint8).reshape((texture.size[1], texture.size[0], 4))
+    # Opengl gives it with the y inverted
+    frame = np.flipud(frame)
+    # TODO: This can be returned as a numpy frame
+
+    # This is if we need an 'av' VideoFrame (to
+    # export through the demuxer, for example)
+    frame = av.VideoFrame.from_ndarray(frame, format = 'rgba')
+    # TODO: Make this customizable
+    frame = frame.reformat(format = 'yuv420p')
+
+    return frame
+
+def get_fullscreen_quad_vao(
+    context: moderngl.Context,
+    program: moderngl.Program
+) -> moderngl.VertexArray:
+    """
+    Get the vertex array object of a quad, by
+    using the vertices, the indexes, the vbo,
+    the ibo and the vao content.
+    """
+    # Quad vertices in NDC (-1..1) with texture
+    # coords (0..1)
+    """
+    The UV coordinates to build the quad we
+    will use to represent the frame by 
+    applying it as a texture.
+    """
+    vertices = np.array([
+        # pos.x, pos.y, tex.u, tex.v
+        -1.0, -1.0, 0.0, 0.0,  # vertex 0 - bottom left
+        1.0, -1.0, 1.0, 0.0,  # vertex 1 - bottom right
+        -1.0,  1.0, 0.0, 1.0,  # vertex 2 - top left
+        1.0,  1.0, 1.0, 1.0,  # vertex 3 - top right
+    ], dtype = 'f4')
+
+    """
+    The indexes of the vertices (see 'vertices'
+    property) to build the 2 opengl triangles
+    that will represent the quad we need for
+    the frame.
+    """
+    indices = np.array([
+        0, 1, 2,
+        2, 1, 3
+    ], dtype = 'i4')
+
+    vbo = context.buffer(vertices.tobytes())
+    ibo = context.buffer(indices.tobytes())
+
+    vao_content = [
+        # 2 floats position, 2 floats texcoords
+        (vbo, '2f 2f', 'in_vert', 'in_texcoord'),
+    ]
+
+    return context.vertex_array(program, vao_content, ibo)

{yta_video_opengl-0.0.6 → yta_video_opengl-0.0.7}/src/yta_video_opengl/writer.py

@@ -1,4 +1,3 @@
-from yta_validation import PythonValidator
 from yta_validation.parameter import ParameterValidator
 from av.stream import Stream
 from av.packet import Packet

yta_video_opengl-0.0.6/src/yta_video_opengl/utils.py

@@ -1,24 +0,0 @@
-from yta_validation import PythonValidator
-from typing import Union
-
-import numpy as np
-import moderngl
-
-
-def frame_to_texture(
-    frame: Union['VideoFrame', 'np.ndarray'],
-    context: moderngl.Context,
-    numpy_format: str = 'rgb24'
-):
-    """
-    Transform the given 'frame' to an opengl
-    texture.
-    """
-    # To numpy RGB inverted for opengl
-    frame: np.ndarray = (
-        np.flipud(frame.to_ndarray(format = numpy_format))
-        if PythonValidator.is_instance_of(frame, 'VideoFrame') else
-        np.flipud(frame)
-    )
-
-    return context.texture((frame.shape[1], frame.shape[0]), 3, frame.tobytes())