yta-video-opengl 0.0.7__tar.gz → 0.0.8__tar.gz

{yta_video_opengl-0.0.7 → yta_video_opengl-0.0.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: yta-video-opengl
-Version: 0.0.7
+Version: 0.0.8
 Summary: Youtube Autonomous Video OpenGL Module
 Author: danialcala94
 Author-email: danielalcalavalera@gmail.com

{yta_video_opengl-0.0.7 → yta_video_opengl-0.0.8}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "yta-video-opengl"
-version = "0.0.7"
+version = "0.0.8"
 description = "Youtube Autonomous Video OpenGL Module"
 authors = [
     {name = "danialcala94",email = "danielalcalavalera@gmail.com"}

{yta_video_opengl-0.0.7 → yta_video_opengl-0.0.8}/src/yta_video_opengl/reader/__init__.py

@@ -3,6 +3,7 @@ A video reader using the PyAv (av) library
 that, using ffmpeg, detects the video.
 """
 from yta_video_opengl.reader.cache import VideoFrameCache
+from yta_video_opengl.utils import iterate_stream_frames_demuxing
 from yta_validation import PythonValidator
 from av.video.frame import VideoFrame
 from av.audio.frame import AudioFrame
@@ -285,7 +286,7 @@ class VideoReader:
         The fps of the audio.
         """
         # TODO: What if no audio (?)
-        return self.audio_stream.average_rate
+        return self.audio_stream.rate
     
     @property
     def time_base(
@@ -303,8 +304,38 @@ class VideoReader:
         """
         The time base of the audio.
         """
+        # TODO: What if no audio (?)
         return self.audio_stream.time_base
     
+    @property
+    def duration(
+        self
+    ) -> Union[float, None]:
+        """
+        The duration of the video.
+        """
+        return (
+            float(self.video_stream.duration * self.video_stream.time_base)
+            if self.video_stream.duration else
+            # TODO: What to do in this case (?)
+            None
+        )
+
+    @property
+    def audio_duration(
+        self
+    ) -> Union[float, None]:
+        """
+        The duration of the audio.
+        """
+        # TODO: What if no audio (?)
+        return (
+            float(self.audio_stream.duration * self.audio_stream.time_base)
+            if self.audio_stream.duration else
+            # TODO: What to do in this case (?)
+            None
+        )
+    
     @property
     def size(
         self
@@ -405,6 +436,26 @@ class VideoReader:
             self.audio_stream.thread_type = 'AUTO'
             self.cache = VideoFrameCache(self)
 
+    def seek(
+        self,
+        pts,
+        stream = None
+    ) -> 'VideoReader':
+        """
+        Call the container '.seek()' method with
+        the given 'pts' packet time stamp.
+        """
+        stream = (
+            self.video_stream
+            if stream is None else
+            stream
+        )
+
+        # TODO: Is 'offset' actually a 'pts' (?)
+        self.container.seek(pts, stream = stream)
+
+        return self
+
     def iterate(
         self
     ) -> 'Iterator[Union[VideoFrame, AudioFrame]]':
@@ -454,7 +505,58 @@ class VideoReader:
                     yield VideoReaderFrame(frame)
             else:
                 # Return the packet as it is
-                yield VideoReaderPacket(packet)
+                yield VideoReaderPacket(packet)        
+
+    # These methods below are using the demux
+    def iterate_video_frames(
+        self,
+        start_pts: int = 0,
+        end_pts: Union[int, None] = None
+    ):
+        """
+        Iterate over the video stream packets and
+        decode only the ones in the expected range,
+        so only those frames are decoded (which is
+        an expensive process).
+
+        This method returns a tuple of 3 elements:
+        - `frame` as a `VideoFrame` instance
+        - `t` as the frame time moment
+        - `index` as the frame index
+        """
+        for frame in iterate_stream_frames_demuxing(
+            container = self.container,
+            video_stream = self.video_stream,
+            audio_stream = None,
+            start_pts = start_pts,
+            end_pts = end_pts
+        ):
+            yield frame
+
+    def iterate_audio_frames(
+        self,
+        start_pts: int = 0,
+        end_pts: Union[int, None] = None
+    ):
+        """
+        Iterate over the audio stream packets and
+        decode only the ones in the expected range,
+        so only those frames are decoded (which is
+        an expensive process).
+
+        This method returns a tuple of 3 elements:
+        - `frame` as a `AudioFrame` instance
+        - `t` as the frame time moment
+        - `index` as the frame index
+        """
+        for frame in iterate_stream_frames_demuxing(
+            container = self.container,
+            video_stream = None,
+            audio_stream = self.audio_stream,
+            start_pts = start_pts,
+            end_pts = end_pts
+        ):
+            yield frame
 
     # TODO: Will we use this (?)
     def get_frame(

{yta_video_opengl-0.0.7 → yta_video_opengl-0.0.8}/src/yta_video_opengl/tests.py

@@ -581,6 +581,11 @@ def video_modified_stored():
 
     from yta_video_opengl.classes import WavingFrame, BreathingFrame, HandheldFrame, OrbitingFrame, RotatingInCenterFrame, StrangeTvFrame, GlitchRgbFrame, WavingNode
     from yta_video_opengl.utils import texture_to_frame, frame_to_texture
+    from yta_video_opengl.video import Video
+
+    Video(VIDEO_PATH, 0, 0.5).save_as(OUTPUT_PATH)
+
+    return
 
     video = VideoReader(VIDEO_PATH)
     video_writer = (
@@ -636,12 +641,9 @@ def video_modified_stored():
         elif is_video_frame:
             with Timer(is_silent_as_context = True) as timer:
                 t = T.video_frame_index_to_video_frame_time(frame_index, float(video.fps))
-
                 # This is another way of getting 't'
                 #t = float(frame_or_packet.pts * video.time_base)
                 
-                # TODO: Pass the frame as a texture
-                
                 video_writer.mux_video_frame(
                     frame = texture_to_frame(
                         texture = node.process(

yta_video_opengl-0.0.8/src/yta_video_opengl/utils.py

@@ -0,0 +1,343 @@
+from yta_validation import PythonValidator
+from av.container import InputContainer
+from av.video.stream import VideoStream
+from av.audio.stream import AudioStream
+from av.video.frame import VideoFrame
+from typing import Union
+
+import av
+import numpy as np
+import moderngl
+
+
+def frame_to_texture(
+    frame: Union['VideoFrame', 'np.ndarray'],
+    context: moderngl.Context,
+    numpy_format: str = 'rgb24'
+):
+    """
+    Transform the given 'frame' to an opengl
+    texture. The frame can be a VideoFrame
+    instance (from pyav library) or a numpy
+    array.
+    """
+    # To numpy RGB inverted for opengl
+    frame: np.ndarray = (
+        np.flipud(frame.to_ndarray(format = numpy_format))
+        if PythonValidator.is_instance_of(frame, 'VideoFrame') else
+        np.flipud(frame)
+    )
+
+    return context.texture(
+        size = (frame.shape[1], frame.shape[0]),
+        components = frame.shape[2],
+        data = frame.tobytes()
+    )
+
+# TODO: I should make different methods to
+# obtain a VideoFrame or a numpy array frame
+def texture_to_frame(
+    texture: moderngl.Texture
+) -> 'VideoFrame':
+    """
+    Transform an opengl texture into a pyav
+    VideoFrame instance.
+    """
+    # RGBA8
+    data = texture.read(alignment = 1)
+    frame = np.frombuffer(data, dtype = np.uint8).reshape((texture.size[1], texture.size[0], 4))
+    # Opengl gives it with the y inverted
+    frame = np.flipud(frame)
+    # TODO: This can be returned as a numpy frame
+
+    # This is if we need an 'av' VideoFrame (to
+    # export through the demuxer, for example)
+    frame = av.VideoFrame.from_ndarray(frame, format = 'rgba')
+    # TODO: Make this customizable
+    frame = frame.reformat(format = 'yuv420p')
+
+    return frame
+
+def get_fullscreen_quad_vao(
+    context: moderngl.Context,
+    program: moderngl.Program
+) -> moderngl.VertexArray:
+    """
+    Get the vertex array object of a quad, by
+    using the vertices, the indexes, the vbo,
+    the ibo and the vao content.
+    """
+    # Quad vertices in NDC (-1..1) with texture
+    # coords (0..1)
+    """
+    The UV coordinates to build the quad we
+    will use to represent the frame by 
+    applying it as a texture.
+    """
+    vertices = np.array([
+        # pos.x, pos.y, tex.u, tex.v
+        -1.0, -1.0, 0.0, 0.0,  # vertex 0 - bottom left
+        1.0, -1.0, 1.0, 0.0,  # vertex 1 - bottom right
+        -1.0,  1.0, 0.0, 1.0,  # vertex 2 - top left
+        1.0,  1.0, 1.0, 1.0,  # vertex 3 - top right
+    ], dtype = 'f4')
+
+    """
+    The indexes of the vertices (see 'vertices'
+    property) to build the 2 opengl triangles
+    that will represent the quad we need for
+    the frame.
+    """
+    indices = np.array([
+        0, 1, 2,
+        2, 1, 3
+    ], dtype = 'i4')
+
+    vbo = context.buffer(vertices.tobytes())
+    ibo = context.buffer(indices.tobytes())
+
+    vao_content = [
+        # 2 floats position, 2 floats texcoords
+        (vbo, '2f 2f', 'in_vert', 'in_texcoord'),
+    ]
+
+    return context.vertex_array(program, vao_content, ibo)
+
+def iterate_streams_packets(
+    container: 'InputContainer',
+    video_stream: 'VideoStream',
+    audio_stream: 'AudioStream',
+    video_start_pts: int = 0,
+    video_end_pts: Union[int, None] = None,
+    audio_start_pts: int = 0,
+    audio_end_pts: Union[int, None] = None
+):
+    """
+    Iterate over the provided 'stream' packets
+    and yield the ones in the expected range.
+    This is nice when trying to copy a stream
+    without modifications.
+    """
+    # 'video_start_pts' and 'audio_start_pts' must
+    # be 0 or a positive tps
+
+    if (
+        video_stream is None and
+        audio_stream is None
+    ):
+        raise Exception('No streams provided.')
+    
+    # We only need to seek on video
+    if video_stream is not None:
+        container.seek(video_start_pts, stream = video_stream)
+    if audio_stream is not None:
+        container.seek(audio_start_pts, stream = audio_stream)
+    
+    stream = [
+        stream
+        for stream in (video_stream, audio_stream)
+        if stream
+    ]
+
+    """
+    Apparently, if we ignore some packets based
+    on the 'pts', we can be ignoring information
+    that is needed for the next frames to be 
+    decoded, so we need to decode them all...
+
+    If we can find some strategy to seek not for
+    the inmediate but some before and read from
+    that one to avoid reading all of the packets
+    we could save some time, but at what cost? 
+    We cannot skip any crucial frame so we need
+    to know how many we can skip, and that sounds
+    a bit difficult depending on the codec.
+    """
+    stream_finished: str = ''
+    for packet in container.demux(stream):
+        if packet.pts is None:
+            continue
+
+        # TODO: We cannot skip like this, we need to
+        # look for the nearest keyframe to be able 
+        # to decode the frames later. Take a look at
+        # the VideoFrameCache class and use it.
+
+        # start_pts = (
+        #     video_start_pts
+        #     if packet.stream.type == 'video' else
+        #     audio_start_pts
+        # )
+        # end_pts = (
+        #     video_end_pts
+        #     if packet.stream.type == 'video' else
+        #     audio_end_pts
+        # )
+
+        # if packet.pts < start_pts:
+        #     continue
+
+        # if (
+        #     end_pts is not None and
+        #     packet.pts > end_pts
+        # ):
+        #     if (
+        #         stream_finished != '' and
+        #         (
+        #             # Finish if only one stream
+        #             stream_finished != packet.stream.type or
+        #             video_stream is None or
+        #             audio_stream is None
+        #         )
+        #     ):
+        #         # We have yielded all the frames in the
+        #         # expected range, no more needed
+        #         return
+            
+        #     stream_finished = packet.stream.type
+        #     continue
+        
+        yield packet
+
+def iterate_stream_frames_demuxing(
+    container: 'InputContainer',
+    video_stream: 'VideoStream',
+    audio_stream: 'AudioStream',
+    video_start_pts : int = 0,
+    video_end_pts: Union[int, None] = None,
+    audio_start_pts: int = 0,
+    audio_end_pts: Union[int, None] = None
+):
+    """
+    Iterate over the provided 'stream' packets
+    and decode only the ones in the expected
+    range, so only those frames are decoded
+    (which is an expensive process).
+
+    This method returns a tuple of 3 elements:
+    - `frame` as a `VideoFrame` instance
+    - `t` as the frame time moment
+    - `index` as the frame index
+
+    You can easy transform the frame received
+    to a numpy array by using this:
+    - `frame.to_ndarray(format = format)`
+    """
+    # 'start_pts' must be 0 or a positive tps
+    # 'end_pts' must be None or a positive tps
+
+    # We cannot skip packets or we will lose
+    # information needed to build the video
+    for packet in iterate_streams_packets(
+        container = container,
+        video_stream = video_stream,
+        audio_stream = audio_stream,
+        video_start_pts = video_start_pts,
+        video_end_pts = video_end_pts,
+        audio_start_pts = audio_start_pts,
+        audio_end_pts = audio_end_pts
+    ):
+        # Only valid and in range packets here
+        # Here only the accepted ones
+        stream_finished: str = ''
+        for frame in packet.decode():
+            if frame.pts is None:
+                continue
+
+            time_base = (
+                video_stream.time_base
+                if PythonValidator.is_instance_of(frame, VideoFrame) else
+                audio_stream.time_base 
+            )
+
+            average_rate = (
+                video_stream.average_rate
+                if PythonValidator.is_instance_of(frame, VideoFrame) else
+                audio_stream.rate
+            )
+
+            start_pts = (
+                video_start_pts
+                if packet.stream.type == 'video' else
+                audio_start_pts
+            )
+
+            end_pts = (
+                video_end_pts
+                if packet.stream.type == 'video' else
+                audio_end_pts
+            )
+
+            if frame.pts < start_pts:
+                continue
+
+            if (
+                end_pts is not None and
+                frame.pts > end_pts
+            ):
+                if (
+                    stream_finished != '' and
+                    (
+                        # Finish if only one stream
+                        stream_finished != packet.stream.type or
+                        video_stream is None or
+                        audio_stream is None
+                    )
+                ):
+                    # We have yielded all the frames in the
+                    # expected range, no more needed
+                    return
+                
+                stream_finished = packet.stream.type
+                continue
+            
+            time_base = (
+                video_stream.time_base
+                if PythonValidator.is_instance_of(frame, VideoFrame) else
+                audio_stream.time_base 
+            )
+
+            average_rate = (
+                video_stream.average_rate
+                if PythonValidator.is_instance_of(frame, VideoFrame) else
+                audio_stream.rate
+            )
+
+            # TODO: Maybe send a @dataclass instead (?)
+            yield (
+                frame,
+                pts_to_t(frame.pts, time_base),
+                pts_to_index(frame.pts, time_base, average_rate)
+            )
+
+def t_to_pts(
+    t: float,
+    stream_time_base: 'Fraction'
+ ) -> int:
+    """
+    Transform a 't' time moment (in seconds) to
+    a packet timestamp (pts) understandable by
+    the pyav library.
+    """
+    return int((t + 0.000001) / stream_time_base)
+
+def pts_to_index(
+    pts: int,
+    stream_time_base: 'Fraction',
+    fps: float
+) -> int:
+    """
+    Transform a 'pts' packet timestamp to a 
+    frame index.
+    """
+    return int(round(pts_to_t(pts, stream_time_base) * fps))
+
+def pts_to_t(
+    pts: int,
+    stream_time_base: 'Fraction'
+) -> float:
+    """
+    Transform a 'pts' packet timestamp to a 't'
+    time moment.
+    """
+    return pts * stream_time_base

yta_video_opengl-0.0.8/src/yta_video_opengl/video.py

@@ -0,0 +1,164 @@
+from yta_video_opengl.reader import VideoReader
+from yta_video_opengl.writer import VideoWriter
+from yta_video_opengl.utils import iterate_stream_frames_demuxing
+from yta_validation import PythonValidator
+from typing import Union
+
+
+# TODO: Where can I obtain this dynamically (?)
+PIXEL_FORMAT = 'yuv420p'
+
+# TODO: Maybe rename to 'Media' (?)
+class Video:
+    """
+    Class to wrap the functionality related to
+    handling and modifying a video.
+    """
+
+    @property
+    def start_pts(
+        self
+    ) -> int:
+        """
+        The start packet time stamp (pts), needed 
+        to optimize the packet iteration process.
+        """
+        return int(self.start / self.reader.time_base)
+    
+    @property
+    def end_pts(
+        self
+    ) -> Union[int, None]:
+        """
+        The end packet time stamp (pts), needed to
+        optimize the packet iteration process.
+        """
+        return (
+            int(self.end / self.reader.time_base)
+            # TODO: What do we do if no duration (?)
+            if self.duration is not None else
+            None
+        )
+    
+    @property
+    def audio_start_pts(
+        self
+    ) -> int:
+        """
+        The start packet time stamp (pts), needed 
+        to optimize the packet iteration process.
+        """
+        return int(self.start / self.reader.audio_time_base)
+    
+    @property
+    def audio_end_pts(
+        self
+    ) -> Union[int, None]:
+        """
+        The end packet time stamp (pts), needed to
+        optimize the packet iteration process.
+        """
+        return (
+            int(self.end / self.reader.audio_time_base)
+            # TODO: What do we do if no duration (?)
+            if self.duration is not None else
+            None
+        )
+    
+    @property
+    def duration(
+        self
+    ):
+        """
+        The duration of the video.
+        """
+        return self.end - self.start
+
+    @property
+    def frames(
+        self
+    ):
+        """
+        Iterator to yield all the frames, one by
+        one, within the range defined by the
+        'start' and 'end' parameters provided when
+        instantiating it.
+
+        This method returns a tuple of 3 elements:
+        - `frame` as a `VideoFrame` instance
+        - `t` as the frame time moment
+        - `index` as the frame index
+        """
+        for frame in iterate_stream_frames_demuxing(
+            container = self.reader.container,
+            video_stream = self.reader.video_stream,
+            audio_stream = self.reader.audio_stream,
+            video_start_pts = self.start_pts,
+            video_end_pts = self.end_pts,
+            audio_start_pts = self.audio_start_pts,
+            audio_end_pts = self.audio_end_pts
+        ):
+            yield frame
+
+    def __init__(
+        self,
+        filename: str,
+        start: float = 0.0,
+        end: Union[float, None] = None
+    ):
+        self.filename: str = filename
+        """
+        The filename of the original video.
+        """
+        # TODO: Detect the 'pixel_format' from the
+        # extension (?)
+        self.reader: VideoReader = VideoReader(self.filename)
+        """
+        The pyav video reader.
+        """
+        self.start: float = start
+        """
+        The time moment 't' in which the video
+        should start.
+        """
+        self.end: Union[float, None] = (
+            # TODO: Is this 'end' ok (?)
+            self.reader.duration
+            if end is None else
+            end
+        )
+        """
+        The time moment 't' in which the video
+        should end.
+        """
+        
+    def save_as(
+        self,
+        filename: str
+    ) -> 'Video':
+        writer =  VideoWriter(filename)
+        #writer.set_video_stream(self.reader.video_stream.codec.name, self.reader.fps, self.reader.size, PIXEL_FORMAT)
+        writer.set_video_stream_from_template(self.reader.video_stream)
+        writer.set_audio_stream_from_template(self.reader.audio_stream)
+
+        # TODO: I need to process the audio also, so
+        # build a method that do the same but for 
+        # both streams at the same time
+        for frame, t, index in self.frames:
+            if PythonValidator.is_instance_of(frame, 'VideoFrame'):
+                print(f'Saving video frame {str(index)}, with t = {str(t)}')
+                writer.mux_video_frame(
+                    frame = frame
+                )
+            else:
+                print(f'Saving audio frame {str(index)} ({str(round(float(t * self.reader.fps), 2))}), with t = {str(t)}')
+                writer.mux_audio_frame(
+                    frame = frame
+                )
+
+        writer.mux_audio_frame(None)
+        writer.mux_video_frame(None)
+
+        # TODO: Maybe move this to the '__del__' (?)
+        writer.output.close()
+        self.reader.container.close()

{yta_video_opengl-0.0.7 → yta_video_opengl-0.0.8}/src/yta_video_opengl/writer.py

@@ -114,9 +114,23 @@ class VideoWriter:
         You can pass the audio stream as it was
         obtained from the reader.
         """
+        self.audio_stream: AudioStream = self.output.add_stream(
+            codec_name = template.codec_context.name,
+            rate = template.codec_context.rate
+        )
+        self.audio_stream.codec_context.format = template.codec_context.format
+        self.audio_stream.codec_context.layout = template.codec_context.layout
+        self.audio_stream.time_base = Fraction(1, template.codec_context.rate)
+
+        return self
+
+        # This below is not working
         self.audio_stream: AudioStream = self.output.add_stream_from_template(
             template
         )
+        # TODO: Is this actually needed (?)
+        # Force this 'rate'
+        self.audio_stream.time_base = Fraction(1, template.codec_context.rate)
 
         return self
 

yta_video_opengl-0.0.7/src/yta_video_opengl/utils.py

@@ -1,100 +0,0 @@
-from yta_validation import PythonValidator
-from typing import Union
-
-import av
-import numpy as np
-import moderngl
-
-
-def frame_to_texture(
-    frame: Union['VideoFrame', 'np.ndarray'],
-    context: moderngl.Context,
-    numpy_format: str = 'rgb24'
-):
-    """
-    Transform the given 'frame' to an opengl
-    texture. The frame can be a VideoFrame
-    instance (from pyav library) or a numpy
-    array.
-    """
-    # To numpy RGB inverted for opengl
-    frame: np.ndarray = (
-        np.flipud(frame.to_ndarray(format = numpy_format))
-        if PythonValidator.is_instance_of(frame, 'VideoFrame') else
-        np.flipud(frame)
-    )
-
-    return context.texture(
-        size = (frame.shape[1], frame.shape[0]),
-        components = frame.shape[2],
-        data = frame.tobytes()
-    )
-
-# TODO: I should make different methods to
-# obtain a VideoFrame or a numpy array frame
-def texture_to_frame(
-    texture: moderngl.Texture
-) -> 'VideoFrame':
-    """
-    Transform an opengl texture into a pyav
-    VideoFrame instance.
-    """
-    # RGBA8
-    data = texture.read(alignment = 1)
-    frame = np.frombuffer(data, dtype = np.uint8).reshape((texture.size[1], texture.size[0], 4))
-    # Opengl gives it with the y inverted
-    frame = np.flipud(frame)
-    # TODO: This can be returned as a numpy frame
-
-    # This is if we need an 'av' VideoFrame (to
-    # export through the demuxer, for example)
-    frame = av.VideoFrame.from_ndarray(frame, format = 'rgba')
-    # TODO: Make this customizable
-    frame = frame.reformat(format = 'yuv420p')
-
-    return frame
-
-def get_fullscreen_quad_vao(
-    context: moderngl.Context,
-    program: moderngl.Program
-) -> moderngl.VertexArray:
-    """
-    Get the vertex array object of a quad, by
-    using the vertices, the indexes, the vbo,
-    the ibo and the vao content.
-    """
-    # Quad vertices in NDC (-1..1) with texture
-    # coords (0..1)
-    """
-    The UV coordinates to build the quad we
-    will use to represent the frame by 
-    applying it as a texture.
-    """
-    vertices = np.array([
-        # pos.x, pos.y, tex.u, tex.v
-        -1.0, -1.0, 0.0, 0.0,  # vertex 0 - bottom left
-        1.0, -1.0, 1.0, 0.0,  # vertex 1 - bottom right
-        -1.0,  1.0, 0.0, 1.0,  # vertex 2 - top left
-        1.0,  1.0, 1.0, 1.0,  # vertex 3 - top right
-    ], dtype = 'f4')
-
-    """
-    The indexes of the vertices (see 'vertices'
-    property) to build the 2 opengl triangles
-    that will represent the quad we need for
-    the frame.
-    """
-    indices = np.array([
-        0, 1, 2,
-        2, 1, 3
-    ], dtype = 'i4')
-
-    vbo = context.buffer(vertices.tobytes())
-    ibo = context.buffer(indices.tobytes())
-
-    vao_content = [
-        # 2 floats position, 2 floats texcoords
-        (vbo, '2f 2f', 'in_vert', 'in_texcoord'),
-    ]
-
-    return context.vertex_array(program, vao_content, ibo)