yta-video-opengl 0.0.13__py3-none-any.whl → 0.0.15__py3-none-any.whl

yta_video_opengl/complete/timeline.py

@@ -168,10 +168,8 @@ class Timeline:
         project will be rendered.
         """
         ParameterValidator.validate_mandatory_string('filename', filename, do_accept_empty = False)
-        # TODO: We need to accept Fraction as number
-        #ParameterValidator.validate_mandatory_positive_number('start', start, do_include_zero = True)
-        # TODO: We need to accept Fraction as number
-        #ParameterValidator.validate_positive_number('end', end, do_include_zero = False)
+        ParameterValidator.validate_mandatory_positive_number('start', start, do_include_zero = True)
+        ParameterValidator.validate_positive_number('end', end, do_include_zero = False)
 
         # TODO: Limitate 'end' a bit...
         end = (
@@ -218,6 +216,7 @@ class Timeline:
         for t in get_ts(start, end, self.fps):
             frame = self.get_frame_at(t)
 
+            print(f'Getting t:{str(float(t))}')
             #print(frame)
 
             # We need to adjust our output elements to be

yta_video_opengl/complete/track.py

@@ -39,10 +39,8 @@ class _Part:
         end: Union[int, float, Fraction],
         video: Union[VideoOnTrack, None] = None
     ):
-        # TODO: We need to accept Fraction as number
-        # ParameterValidator.validate_mandatory_positive_number('start', start, do_include_zero = True)
-        # TODO: We need to accept Fraction as number
-        # ParameterValidator.validate_mandatory_positive_number('end', end, do_include_zero = False)
+        ParameterValidator.validate_mandatory_positive_number('start', start, do_include_zero = True)
+        ParameterValidator.validate_mandatory_positive_number('end', end, do_include_zero = False)
         ParameterValidator.validate_instance_of('video', video, VideoOnTrack)
 
         self._track: Track = track
@@ -94,7 +92,7 @@ class _Part:
             # TODO: By now I'm raising exception to check if
             # this happens or not because I think it would
             # be malfunctioning
-            raise Exception(f'Video is returning None frame at t={str(t)}.')
+            raise Exception(f'Video is returning None video frame at t={str(t)}.')
         
         return frame
     

yta_video_opengl/complete/video_on_track.py

@@ -46,10 +46,7 @@ class VideoOnTrack:
         start: Union[int, float, Fraction] = 0.0
     ):
         ParameterValidator.validate_mandatory_instance_of('video', video, Video)
-        # TODO: Now we need to accept 'Fraction',
-        # from 'fractions' or 'quicktions', as a
-        # number
-        #ParameterValidator.validate_mandatory_positive_number('start', start, do_include_zero = True)
+        ParameterValidator.validate_mandatory_positive_number('start', start, do_include_zero = True)
 
         self.video: Video = video
         """
@@ -150,10 +147,7 @@ class VideoOnTrack:
         )
 
         for frame in frames:
-            # TODO: I am generating a tuple in the
-            # src\yta_video_opengl\reader\cache.py
-            # get_frames method... maybe remove it (?)
-            yield frame[0]
+            yield frame
 
         # # TODO: This was a simple return before
         # return (

yta_video_opengl/reader/__init__.py

@@ -2,7 +2,8 @@
 A video reader using the PyAv (av) library
 that, using ffmpeg, detects the video.
 """
-from yta_video_opengl.reader.cache import VideoFrameCache
+from yta_video_opengl.reader.cache.video import VideoFrameCache
+from yta_video_opengl.reader.cache.audio import AudioFrameCache
 from yta_video_opengl.utils import iterate_stream_frames_demuxing
 from yta_video_opengl.t import T
 from yta_validation import PythonValidator
@@ -435,7 +436,7 @@ class VideoReader:
         The video frame cache system to optimize
         the way we access to the frames.
         """
-        self.audio_cache: VideoFrameCache = None
+        self.audio_cache: AudioFrameCache = None
         """
         The audio frame cache system to optimize
         the way we access to the frames.
@@ -490,7 +491,7 @@ class VideoReader:
                 raise Exception(f'No video nor audio stream found in the "{self.filename}" file.')
 
             self.video_cache = VideoFrameCache(self.container, self.video_stream)
-            self.audio_cache = VideoFrameCache(self.container, self.audio_stream)
+            self.audio_cache = AudioFrameCache(self.container, self.audio_stream)
 
     def seek(
         self,
@@ -620,36 +621,27 @@ class VideoReader:
         ):
             yield frame
 
-    # TODO: Will we use this (?)
     def get_frame(
-        self,
-        index: int
-    ) -> 'VideoFrame':
-        """
-        Get the video frame with the given 'index',
-        using the video cache system.
-        """
-        return self.video_cache.get_frame(index)
-    
-    def get_frame_from_t(
         self,
         t: Union[int, float, Fraction]
-    ) -> 'VideoFrame':
+    ) -> VideoFrame:
         """
-        Get the video frame with the given 't' time
-        moment, using the video cache system.
+        Get the video frame that is in the 't' time
+        moment provided.
         """
-        return self.video_cache.get_frame_from_t(t)
-    
-    def get_audio_frame(
+        return self.video_cache.get_frame(t)
+
+    def get_frames(
         self,
-        index: int
-    ) -> 'AudioFrame':
+        start: Union[int, float, Fraction] = 0.0,
+        end: Union[int, float, Fraction, None] = None
+    ):
         """
-        Get the audio frame with the given 'index',
-        using the audio cache system.
+        Iterator to get the video frames in between
+        the provided 'start' and 'end' time moments.
         """
-        return self.audio_cache.get_frame(index)
+        for frame in self.video_cache.get_frames(start, end):
+            yield frame
 
     def get_audio_frame_from_t(
         self,
@@ -659,7 +651,7 @@ class VideoReader:
         Get the audio frame with the given 't' time
         moment, using the audio cache system.
         """
-        return self.audio_cache.get_frame_from_t(t)
+        return self.audio_cache.get_frame(t)
     
     def get_audio_frames_from_t(
         self,
@@ -677,19 +669,10 @@ class VideoReader:
         with more than 1 audio frame).
         """
         t: T = T.from_fps(t, self.fps)
-        for frame in self.audio_cache.get_frames(t.truncated, t.next(1).truncated):
-            yield frame
-    
-    def get_frames(
-        self,
-        start: Union[int, float, Fraction] = 0.0,
-        end: Union[int, float, Fraction, None] = None
-    ):
-        """
-        Iterator to get the video frames in between
-        the provided 'start' and 'end' time moments.
-        """
-        for frame in self.video_cache.get_frames(start, end):
+        # We want all the audios that must be played
+        # during the video frame that starts in the
+        # 't' time moment
+        for frame in self.get_audio_frames(t.truncated, t.next(1).truncated):
             yield frame
 
     def get_audio_frames(

yta_video_opengl/reader/cache/__init__.py

@@ -0,0 +1,249 @@
+"""
+The pyav container stores the information based
+on the packets timestamps (called 'pts'). Some
+of the packets are considered key_frames because
+they include those key frames.
+
+Also, this library uses those key frames to start
+decodifying from there to the next one, obtaining
+all the frames in between able to be read and
+modified.
+
+This cache system will look for the range of 
+frames that belong to the key frame related to the
+frame we are requesting in the moment, keeping in
+memory all those frames to be handled fast. It
+will remove the old frames if needed to use only
+the 'size' we set when creating it.
+
+A stream can have 'fps = 60' but use another
+different time base that make the pts values go 0,
+ 256, 512... for example. The 'time_base' is the
+only accurate way to obtain the pts.
+
+Feel free to move this explanation to other
+place, its about the duration.
+
+The stream 'duration' parameter is measured
+on ticks, the amount of ticks that the
+stream lasts. Here below is an example:
+
+- Duration raw: 529200
+- Time base: 1/44100
+- Duration (seconds): 12.0
+"""
+from av.container import InputContainer
+from av.video.stream import VideoStream
+from av.audio.stream import AudioStream
+from av.video.frame import VideoFrame
+from av.audio.frame import AudioFrame
+from av.packet import Packet
+from yta_validation.parameter import ParameterValidator
+from quicktions import Fraction
+from collections import OrderedDict
+from typing import Union
+from abc import abstractmethod, ABC
+
+import numpy as np
+import math
+
+
+class FrameCache(ABC):
+    """
+    Class to manage the frames cache of a video
+    or audio.
+    """
+
+    @property
+    @abstractmethod
+    def fps(
+        self
+    ) -> Union[int, Fraction, None]:
+        """
+        The frames per second.
+        """
+        pass
+    
+    @property
+    def time_base(
+        self
+    ) -> Union[Fraction, None]:
+        """
+        The time base of the stream.
+        """
+        return self.stream.time_base
+
+    def __init__(
+        self,
+        container: InputContainer,
+        stream: Union[VideoStream, AudioStream],
+        size: Union[int, None] = None
+    ):
+        ParameterValidator.validate_mandatory_instance_of('container', container, InputContainer)
+        ParameterValidator.validate_mandatory_instance_of('stream', stream, [VideoStream, AudioStream])
+        ParameterValidator.validate_number_between('size', size, 1, 120)
+
+        self.container: InputContainer = container
+        """
+        The pyav container.
+        """
+        self.stream: Union[VideoStream, AudioStream] = stream
+        """
+        The pyav stream.
+        """
+        self.cache: OrderedDict = OrderedDict()
+        """
+        The cache ordered dictionary.
+        """
+        self.key_frames_pts: list[int] = []
+        """
+        The list that contains the timestamps of the
+        key frame packets, ordered from begining to
+        end.
+        """
+        self.size: int = size
+        """
+        The size of the cache.
+        """
+
+        self._last_packet_accessed: Union[Packet, None] = None
+        """
+        The last packet that has been accessed
+        """
+
+        self._prepare()
+
+    def _prepare(
+        self
+    ):
+        # Index key frames
+        for packet in self.container.demux(self.stream):
+            if packet.is_keyframe:
+                self.key_frames_pts.append(packet.pts)
+
+        # The cache size will be auto-calculated to
+        # use the amount of frames of the biggest
+        # interval of frames that belongs to a key
+        # frame, or a value by default
+        # TODO: Careful if this is too big
+        # Intervals, but in number of frames
+        intervals = np.diff(
+            # Intervals of time between keyframes
+            np.array(self.key_frames_pts) * self.time_base
+        ) * self.fps
+
+        self.size = (
+            math.ceil(np.max(intervals))
+            if intervals.size > 0 else
+            (
+                self.size
+                if self.size is not None else
+                # TODO: Make this a setting (?)
+                60
+            )
+        )
+        
+        self.container.seek(0)
+
+    def _get_nearest_keyframe_pts(
+        self,
+        pts: int
+    ):
+        """
+        Get the fps of the keyframe that is the
+        nearest to the provided 'pts'. Useful to
+        seek and start decoding frames from that
+        keyframe.
+        """
+        return max([
+            key_frame_pts
+            for key_frame_pts in self.key_frames_pts
+            if key_frame_pts <= pts
+        ])
+
+    def _store_frame_in_cache(
+        self,
+        frame: Union[VideoFrame, AudioFrame]
+    ) -> Union[VideoFrame, AudioFrame]:
+        """
+        Store the provided 'frame' in cache if it
+        is not on it, removing the first item of
+        the cache if full.
+        """
+        if frame.pts not in self.cache:
+            self.cache[frame.pts] = frame
+
+            # Clean cache if full
+            if len(self.cache) > self.size:
+                self.cache.popitem(last = False)
+
+        return frame
+    
+    def _seek(
+        self,
+        pts: int
+    ):
+        """
+        Seek to the given 'pts' This is useful
+        when working with 'container.demux' and
+        iterating over packets, not when using
+        'stream.decode' and getting frames 
+        directly.
+        """
+        self.container.seek(
+            offset = pts,
+            stream = self.stream
+        )
+
+    def clear(
+        self
+    ) -> 'VideoFrameCache':
+        """
+        Clear the cache by removing all the items.
+        """
+        self.cache.clear()
+
+        return self
+
+    def get_frame(
+        self,
+        t: Union[int, float, Fraction]
+    ) -> Union[VideoFrame, AudioFrame]:
+        """
+        Get the single frame that is in the 't'
+        time moment provided.
+        """
+        for frame in self.get_frames(t):
+            return frame
+        
+    @abstractmethod
+    def get_frames(
+        self,
+        start: Union[int, float, Fraction],
+        end: Union[int, float, Fraction]
+    ):
+        pass
+
+
+"""
+There is a way of editing videos being
+able to arbitrary access to frames, that
+is transforming the source videos to
+intra-frame videos. This is a ffmpeg
+command that can do it:
+
+- `ffmpeg -i input.mp4 -c:v libx264 -x264opts keyint=1 -preset fast -crf 18 -c:a copy output_intra.mp4`
+
+Once you have the 'output_intra.mp4',
+each packet can decodify its frame 
+depending not on the previous one, being
+able to seek and jump easy.
+
+There are 3 type of video codifications,
+the I-frame (intra-coded), in which any
+frame can be decoded by itself, P-frame
+(predicted), that need one or more 
+previous frames to be decoded, and 
+B-frame (bidirectional predicted), that
+needs previous and future frames.
+"""

yta_video_opengl/reader/cache/audio.py

@@ -0,0 +1,195 @@
+
+from yta_video_opengl.reader.cache import FrameCache
+from yta_video_opengl.reader.cache.utils import trim_audio_frame
+from yta_video_opengl.t import T
+from yta_validation.parameter import ParameterValidator
+from av.container import InputContainer
+from av.audio.stream import AudioStream
+from av.audio.frame import AudioFrame
+from quicktions import Fraction
+from typing import Union
+
+
+class AudioFrameCache(FrameCache):
+    """
+    Cache for the audio frames.
+    """
+
+    @property
+    def fps(
+        self
+    ) -> Union[Fraction, int]:
+        """
+        The frames per second.
+        """
+        return self.stream.rate
+
+    @property
+    def frame_duration(
+        self
+    ) -> int:
+        """
+        The frame duration in ticks, which is the
+        minimum amount of time, 1 / time_base.
+        """
+        return self.stream.frames
+    
+    def __init__(
+        self,
+        container: InputContainer,
+        stream: AudioStream,
+        size: Union[int, None] = None
+    ):
+        ParameterValidator.validate_mandatory_instance_of('stream', stream, AudioStream)
+
+        super().__init__(container, stream, size)
+
+    def _seek(
+        self,
+        pts: int
+    ):
+        """
+        Seek to the given 'pts' only if it is not
+        the next 'pts' to the last read, and it 
+        will also apply a pad to avoid problems
+        when reading audio frames.
+        """
+        # I found that it is recommended to
+        # read ~100ms before the pts we want to
+        # actually read so we obtain the frames
+        # clean (this is important in audio).
+        # This solves a problem I had related
+        # to some artifacts on the audio when
+        # trimming exactly without this pad.
+        pts_pad = int(0.1 / self.time_base)
+        self.container.seek(
+            offset = max(0, pts - pts_pad),
+            stream = self.stream
+        )
+
+    def get_frame(
+        self,
+        t: Union[int, float, Fraction]
+    ) -> AudioFrame:
+        """
+        Get the video frame that is in the 't'
+        time moment provided.
+        """
+        t: T = T.from_fps(t, self.fps)
+        for frame in self.get_frames(t.truncated, t.next(1).truncated):
+            return frame
+
+    def get_frames(
+        self,
+        start: Union[int, float, Fraction],
+        end: Union[int, float, Fraction]
+    ):
+        """
+        Get all the audio frames in the range
+        between the provided 'start' and 'end'
+        time (in seconds).
+
+        This method is an iterator that yields
+        the frame, its t and its index.
+        """
+        # TODO: Validate 'start' and 'end' are mandatory
+        # positive numbers
+        # Make sure the 'start' and 'end' time moments
+        # provided are truncated values based on the
+        # stream time base
+        start = T(start, self.time_base).truncated
+        end = T(end, self.time_base).truncated
+        
+        if end <= start:
+            raise Exception(f'The time range start:{str(float(start))} - end:{str(float(end))}) is not valid.')
+
+        key_frame_pts = self._get_nearest_keyframe_pts(start / self.time_base)
+
+        if (
+            self._last_packet_accessed is None or
+            self._last_packet_accessed.pts != key_frame_pts
+        ):
+            self._seek(key_frame_pts)
+
+        for packet in self.container.demux(self.stream):
+            if packet.pts is None:
+                continue
+
+            self._last_packet_accessed = packet
+
+            for frame in packet.decode():
+                if frame.pts is None:
+                    continue
+
+                # We store all the frames in cache
+                self._store_frame_in_cache(frame)
+
+                current_frame_time = frame.pts * self.time_base
+                # End is not included, its the start of the
+                # next frame actually
+                frame_end = current_frame_time + (frame.samples / self.stream.sample_rate)
+
+                # For the next comments imagine we are looking
+                # for the [1.0, 2.0) audio time range
+                # Previous frame and nothing is inside
+                if frame_end <= start:
+                    # From 0.25 to 1.0
+                    continue
+                
+                # We finished, nothing is inside and its after
+                if current_frame_time >= end:
+                    # From 2.0 to 2.75
+                    return
+
+                """
+                If we need audio from 1 to 2, audio is:
+                - from 0 to 0.75    (Not included, omit)
+                - from 0.5 to 1.5   (Included, take 1.0 to 1.5)
+                - from 0.5 to 2.5   (Included, take 1.0 to 2.0)
+                - from 1.25 to 1.5  (Included, take 1.25 to 1.5)
+                - from 1.25 to 2.5  (Included, take 1.25 to 2.0)
+                - from 2.5 to 3.5   (Not included, omit)
+                """
+                
+                # Here below, at least a part is inside
+                if (
+                    current_frame_time < start and
+                    frame_end > start
+                ):
+                    # A part at the end is included
+                    end_time = (
+                        # From 0.5 to 1.5 0> take 1.0 to 1.5
+                        frame_end
+                        if frame_end <= end else
+                        # From 0.5 to 2.5 => take 1.0 to 2.0
+                        end
+                    )
+                    #print('A part at the end is included.')
+                    frame = trim_audio_frame(
+                        frame = frame,
+                        start = start,
+                        end = end_time,
+                        time_base = self.time_base
+                    )
+                elif (
+                    current_frame_time >= start and
+                    current_frame_time < end
+                ):
+                    end_time = (
+                        # From 1.25 to 1.5 => take 1.25 to 1.5
+                        frame_end
+                        if frame_end <= end else
+                        # From 1.25 to 2.5 => take 1.25 to 2.0
+                        end
+                    )
+                    # A part at the begining is included
+                    #print('A part at the begining is included.')
+                    frame = trim_audio_frame(
+                        frame = frame,
+                        start = current_frame_time,
+                        end = end_time,
+                        time_base = self.time_base
+                    )
+
+                # If the whole frame is in, past as it is
+                yield frame

yta_video_opengl/reader/cache/utils.py

@@ -0,0 +1,48 @@
+from av.audio.frame import AudioFrame
+from quicktions import Fraction
+from typing import Union
+
+
+def trim_audio_frame(
+    frame: AudioFrame,
+    start: Union[int, float, Fraction],
+    end: Union[int, float, Fraction],
+    time_base: Fraction
+) -> AudioFrame:
+    """
+    Trim an audio frame to obtain the part between
+    [start, end), that is provided in seconds.
+    """
+    # (channels, n_samples)
+    samples = frame.to_ndarray()  
+    n_samples = samples.shape[1]
+
+    # In seconds
+    frame_start = frame.pts * float(time_base)
+    frame_end = frame_start + (n_samples / frame.sample_rate)
+
+    # Overlapping 
+    cut_start = max(frame_start, float(start))
+    cut_end = min(frame_end, float(end))
+
+    if cut_start >= cut_end:
+        # No overlapping
+        return None  
+
+    # To sample indexes
+    start_index = int(round((cut_start - frame_start) * frame.sample_rate))
+    end_index = int(round((cut_end - frame_start) * frame.sample_rate))
+
+    new_frame = AudioFrame.from_ndarray(
+        # end_index is not included: so [start, end)
+        array = samples[:, start_index:end_index],
+        format = frame.format,
+        layout = frame.layout
+    )
+
+    # Set attributes
+    new_frame.sample_rate = frame.sample_rate
+    new_frame.time_base = time_base
+    new_frame.pts = int(round(cut_start / float(time_base)))
+
+    return new_frame