shaderflow 0.4.0.dev1__py2.py3-none-any.whl

ShaderFlow/Modules/Video.py

@@ -0,0 +1,234 @@
+import contextlib
+import io
+import sys
+import time
+from collections import deque
+from pathlib import Path
+from typing import Any, Callable, Dict, Tuple
+
+import cv2
+import numpy
+import PIL
+from attr import Factory, define
+
+from Broken import BrokenAttrs, BrokenRelay, BrokenThread, SameTracker, log
+from Broken.Externals.FFmpeg import BrokenFFmpeg
+from Broken.Types import Hertz, Seconds
+from ShaderFlow.Module import ShaderModule
+from ShaderFlow.Texture import ShaderTexture
+
+
+@define(slots=False)
+class BrokenSmartVideoFrames(BrokenAttrs):
+    path:     Path    = None
+    buffer:   Seconds = 60
+    threads:  int     = 6
+    quality:  int     = 95
+    time:     Seconds = 0
+    lossless: bool    = True
+    _width:   int     = None
+    _height:  int     = None
+    _fps:     Hertz   = None
+    _turbo:   Any     = None
+    _raw:     deque   = Factory(deque)
+    _frames:  Dict    = Factory(dict)
+
+    # Dynamically set
+    encode:  Callable = None
+    decode:  Callable = None
+
+    @property
+    def max_raw(self) -> int:
+        return self.threads*4
+
+    @property
+    def width(self) -> int:
+        return self._width
+
+    @property
+    def height(self) -> int:
+        return self._height
+
+    # # Initialization
+
+    LOSSLESS_MAX_BUFFER_LENGTH = 4
+
+    def __post__(self):
+        self._fps = BrokenFFmpeg.get_video_framerate(self.path)
+        self._width, self._height = BrokenFFmpeg.get_video_resolution(self.path)
+
+        if not all((self._fps, self._width, self._height)):
+            raise ValueError("Could not get video metadata")
+
+        # TurboJPEG will raise if shared lib is not found
+        with contextlib.suppress(RuntimeError, ModuleNotFoundError):
+            import turbojpeg
+            self._turbo = turbojpeg.TurboJPEG()
+
+        if self.lossless:
+            log.warning("Using lossless frames. Limiting buffer length for Out of Memory safety")
+            self.buffer = min(self.buffer, BrokenSmartVideoFrames.LOSSLESS_MAX_BUFFER_LENGTH)
+            self.encode = lambda frame: frame
+            self.decode = lambda frame: frame
+
+        elif (self._turbo is not None):
+            log.success("Using TurboJPEG for compression. Best speeds available")
+            self.encode = lambda frame: self._turbo.encode(frame, quality=self.quality)
+            self.decode = lambda frame: self._turbo.decode(frame)
+
+        elif ("cv2" in sys.modules):
+            log.success("Using OpenCV for compression. Slower than TurboJPEG but enough")
+            self.encode = lambda frame: cv2.imencode(".jpeg", frame)[1]
+            self.decode = lambda frame: cv2.imdecode(frame, cv2.IMREAD_COLOR)
+
+        else:
+            log.warning("Using PIL for compression. Performance killer GIL fallback")
+            self.decode = lambda frame: PIL.Image.open(io.BytesIO(frame))
+            self.encode = lambda frame: PIL.Image.fromarray(frame).save(
+                io.BytesIO(), format="jpeg", quality=self.quality
+            )
+
+        # Create worker threads. The good, the bad and the ugly
+        BrokenThread(target=self.extractor, daemon=True)
+        BrokenThread(target=self.deleter,   daemon=True)
+        for _ in range(self.threads):
+            BrokenThread(target=self.worker, daemon=True)
+
+    # # Utilities
+
+    def time2index(self, time: Seconds) -> int:
+        return int(time*self._fps)
+
+    def index2time(self, index: int) -> Seconds:
+        return (index/self._fps)
+
+    # # Check if we can decode and encode with the libraries
+
+    def get_frame(self, time: Seconds) -> Tuple[int, numpy.ndarray]:
+        want = self.time2index(time)
+        self.time = time
+        import time
+
+        # Wait until the frame exists
+        while (jpeg := self._frames.get(want)) is None:
+            time.sleep(0.01)
+
+        return (want, lambda: self.decode(jpeg))
+
+    @property
+    def buffer_frames(self) -> int:
+        return int(self.buffer*self._fps)
+
+    @property
+    def time_index(self) -> int:
+        return self.time2index(self.time)
+
+    @property
+    def _future_index(self) -> int:
+        return self.time_index + self.buffer_frames
+
+    def _future_window(self, index: int) -> bool:
+        return index < self._future_index
+
+    @property
+    def _past_index(self) -> int:
+        return self.time_index - self.buffer_frames
+
+    def _past_window(self, index: int) -> bool:
+        return self._past_index < index
+
+    def _time_window(self, index: int) -> bool:
+        return self._past_window(index) and self._future_window(index)
+
+    def _should_rewind(self, index: int) -> bool:
+        """Point must be older than the past cutoff to trigger a rewind"""
+        return (self.time_index + self.buffer_frames) < index
+
+    # # Workers
+
+    _oldest: int = 0
+    _newest: int = 0
+
+    def extractor(self):
+        def forward():
+            for index, frame in enumerate(BrokenFFmpeg.iter_video_frames(self.path)):
+
+                # Skip already processed frames
+                if self._frames.get(index) is not None:
+                    continue
+
+                # Skip frames outside of the past time window
+                if not self._past_window(index):
+                    continue
+
+                while not self._future_window(index):
+                    if self._should_rewind(index):
+                        return
+                    time.sleep(0.01)
+
+                # Limit how much raw frames there can be
+                while len(self._raw) > self.max_raw:
+                    time.sleep(0.01)
+
+                self._raw.append((index, frame))
+                self._newest = max(self._newest, index)
+
+        while True:
+            forward()
+
+    def worker(self):
+        """Blindly get new frames from the deque, compress and store them"""
+        while True:
+            try:
+                index, frame = self._raw.popleft()
+                frame = numpy.array(numpy.flip(frame, axis=0))
+                self._frames[index] = self.encode(frame)
+            except IndexError:
+                time.sleep(0.01)
+
+    def deleter(self):
+        """Delete old frames that are not in the time window"""
+        while True:
+            for index in range(self._oldest, self._past_index):
+                self._frames[index] = None
+                self._oldest = index
+            for index in range(self._newest, self._future_index, -1):
+                self._frames[index] = None
+                self._newest = index
+            time.sleep(0.5)
+
+# -------------------------------------------------------------------------------------------------|
+
+@define
+class ShaderVideo(BrokenSmartVideoFrames, ShaderModule):
+    name: str = "iVideo"
+    texture: ShaderTexture = None
+
+    temporal: int = 10
+    """How many """
+
+    on_frame: BrokenRelay = Factory(BrokenRelay)
+    """Whenever a new video frame is decoded, this attribute is called. Preferably subscribe to
+    it with `video.on_frame.subscribe(callable)` or `video.on_frame @ (A, B, C)`, see BokenRelay"""
+
+    def __post__(self):
+        self.texture = ShaderTexture(
+            scene=self.scene,
+            name=self.name,
+            width=self.width,
+            height=self.height,
+            temporal=self.temporal,
+            components=3,
+            dtype="f1"
+        )
+
+    __same__: SameTracker = Factory(SameTracker)
+
+    def update(self):
+        index, decode = self.get_frame(self.scene.time)
+
+        if not self.__same__(index):
+            image = decode()
+            self.texture.roll()
+            self.texture.write(image)
+            self.on_frame(image)

ShaderFlow/Modules/Waveform.py

@@ -0,0 +1,100 @@
+import math
+from typing import Iterable
+
+import numpy
+from attr import Factory, define
+
+from Broken import BrokenEnum, SameTracker, nearest
+from Broken.Types import Hertz, Samples, Seconds
+from ShaderFlow.Module import ShaderModule
+from ShaderFlow.Modules.Audio import BrokenAudio
+from ShaderFlow.Texture import ShaderTexture, TextureType
+from ShaderFlow.Variable import ShaderVariable
+
+
+class WaveformReducer(BrokenEnum):
+    def Average(x: numpy.ndarray) -> numpy.ndarray:
+        return numpy.sqrt(numpy.mean(numpy.abs(x), axis=2))
+
+    def RMS(x: numpy.ndarray) -> numpy.ndarray:
+        return numpy.sqrt(numpy.sqrt(numpy.mean(x**2, axis=2))*(2**0.5))
+
+    def STD(x: numpy.ndarray) -> numpy.ndarray:
+        return numpy.sqrt(numpy.std(x, axis=2))
+
+@define
+class ShaderWaveform(ShaderModule):
+
+    name: str = "iWaveform"
+    """Prefix name and Texture name of the Shader Variables"""
+
+    audio: BrokenAudio = None
+    """Audio class to read the data from"""
+
+    length: Seconds = 3
+    """Horizontal length of the Waveform content"""
+
+    samplerate: Hertz = 180
+    """Number of bars per second"""
+
+    reducer: WaveformReducer = WaveformReducer.Average
+    """How to convert a (channels, length, samples) chunks into (channels, length)"""
+
+    smooth: bool = True
+    """Enables Linear interpolation on the Texture, not much useful for Bars mode"""
+
+    texture: ShaderTexture = None
+    """Internal managed Texture"""
+
+    @property
+    def length_samples(self) -> Samples:
+        return int(max(1, self.length*self.scene.fps))
+
+    def __post__(self):
+        self.texture = ShaderTexture(
+            scene=self.scene,
+            name=self.name,
+            height=1,
+            mipmaps=False,
+            dtype=TextureType.f4,
+        )
+
+    @property
+    def chunk_size(self) -> Samples:
+        return max(1, int(self.length*self.audio.samplerate/self._points))
+
+    @property
+    def _points(self) -> int:
+        return self.length*self.samplerate
+
+    @property
+    def _offset(self) -> int:
+        return self.audio.tell % self.chunk_size
+
+    @property
+    def _cutoff(self) -> Samples:
+        return nearest(
+            number=self.audio.buffer_size,
+            multiple=self.chunk_size,
+            operator=math.floor,
+            type=int,
+        )
+
+    _same: SameTracker = Factory(SameTracker)
+
+    def update(self):
+        if self._same(self.audio.tell):
+            return
+        self.texture.filter     = ("linear" if self.smooth else "nearest")
+        self.texture.components = self.audio.channels
+        self.texture.width      = self._points
+        start  = -int(self.chunk_size*self._points + self._offset + 1)
+        end    = -int(self._offset + 1)
+        chunks = self.audio.data[:, start:end]
+        chunks = chunks.reshape(self.audio.channels, -1, self.chunk_size)
+        chunks = self.reducer(chunks)
+        chunks = numpy.ascontiguousarray(chunks.T)
+        self.texture.write(chunks)
+
+    def pipeline(self) -> Iterable[ShaderVariable]:
+        yield ShaderVariable("uniform", "int", f"{self.name}Length", self.length_samples)

ShaderFlow/Resources/Images/ShaderFlow.svg

@@ -0,0 +1 @@
+<svg width="32" height="32" version="1" xmlns="http://www.w3.org/2000/svg"><g transform="matrix(.91242 0 0 .91242 1.4013026 1.6537216)"><rect style="opacity:.2" width="28" height="29" x="2" y="2" rx="4.1999998" ry="4.1999998"/><rect style="fill:#0286e3;fill-opacity:1" width="28" height="29" x="2" y="1" rx="4.1999998" ry="4.1999998"/><path style="opacity:.1;fill:#fff" d="M6.1992188 1C3.8724189 1 2 2.8724189 2 5.1992188v1C2 3.8724189 3.8724189 2 6.1992188 2H25.800781C28.127581 2 30 3.8724189 30 6.1992188v-1C30 2.8724189 28.127581 1 25.800781 1Z"/><text xml:space="preserve" style="font-size:26.9514px;font-family:'Roboto Slab';-inkscape-font-specification:'Roboto Slab';opacity:.2;fill:#000;fill-opacity:1;stroke:none;stroke-width:42.808;stroke-linejoin:round;stroke-dasharray:none" x="7.6854787" y="26.417761" transform="translate(.21518099 -.33798856)"><tspan x="7.6854787" y="26.417761" style="font-weight:400;fill:#000;fill-opacity:1;stroke-width:42.808;stroke-dasharray:none">S</tspan></text><text xml:space="preserve" style="font-size:26.9514px;font-family:'Roboto Slab';-inkscape-font-specification:'Roboto Slab';fill:#fff;fill-opacity:1;stroke:none;stroke-width:42.8078;stroke-linejoin:round;stroke-dasharray:none" x="7.6843672" y="25.418978" transform="translate(.21518099 -.33798856)"><tspan x="7.6843672" y="25.418978" style="font-style:normal;font-weight:400;fill:#fff;fill-opacity:1;stroke-width:42.8078">S</tspan></text></g></svg>

ShaderFlow/Resources/Scenes/Examples/Demo.py

@@ -0,0 +1,284 @@
+import math
+from pathlib import Path
+
+import numpy
+from ShaderFlow import SHADERFLOW
+from ShaderFlow.Common.Notes import BrokenPianoNote
+from ShaderFlow.Modules.Audio import ShaderAudio
+from ShaderFlow.Modules.Bouncing import ShaderBouncing
+from ShaderFlow.Modules.Dynamics import ShaderDynamics
+from ShaderFlow.Modules.Noise import ShaderNoise
+from ShaderFlow.Modules.Spectrogram import ShaderSpectrogram
+from ShaderFlow.Modules.Video import ShaderVideo
+from ShaderFlow.Modules.Waveform import ShaderWaveform
+from ShaderFlow.Scene import ShaderScene
+from ShaderFlow.Shader import ShaderObject
+from ShaderFlow.Texture import ShaderTexture, TextureFilter
+from ShaderFlow.Variable import ShaderVariable
+
+from Broken import BrokenPath
+
+BACKGROUND = "https://w.wallhaven.cc/full/e7/wallhaven-e778vr.jpg"
+
+# -------------------------------------------------------------------------------------------------|
+
+class Default(ShaderScene):
+    """The most basic ShaderScene, the default shader"""
+    ...
+
+# -------------------------------------------------------------------------------------------------|
+
+class ShaderToy(ShaderScene):
+    """ShaderToy Default Shader"""
+
+    def build(self):
+        ShaderScene.build(self)
+        self.shader.fragment = (self.directory/"GLSL"/"ShaderToy.frag")
+
+# -------------------------------------------------------------------------------------------------|
+
+class Nested(ShaderScene):
+    """Basic scene with two shaders acting together, main shader referencing the child"""
+    __name__ = "Nested Shaders"
+
+    def build(self):
+        ShaderScene.build(self)
+        self.child = ShaderObject(scene=self, name="child")
+
+        # - Left screen is black, right screen is red
+        # - Adds content of child shader to final image
+        self.shader.fragment = ("""
+            void main() {
+                fragColor.rgb = vec3(stuv.x, 0, 0);
+                fragColor.rgb += texture(child, astuv).rgb;
+                fragColor.a = 1;
+            }
+        """)
+
+        # Left screen is green, right screen is black
+        self.child.fragment = ("""
+            void main() {
+                fragColor.rgb = vec3(0, 1 - stuv.x, 0);
+                fragColor.a = 1;
+            }
+        """)
+
+# -------------------------------------------------------------------------------------------------|
+
+class Multipass(ShaderScene):
+    """Many Layers ('Buffers') done on a single shader"""
+    __name__ = "Multipass"
+
+    def build(self):
+        ShaderScene.build(self)
+        ShaderTexture(scene=self, name="background").from_image(BACKGROUND)
+        self.shader.texture.layers = 2
+        self.shader.fragment = (self.directory/"GLSL"/"Multipass.frag")
+
+# -------------------------------------------------------------------------------------------------|
+
+class MotionBlur(ShaderScene):
+    """Poor's man Motion Blur. If you dislike the effect, definitely don't run this"""
+    __name__ = "MotionBlur"
+
+    def build(self):
+        ShaderScene.build(self)
+        ShaderTexture(scene=self, name="background").from_image(BACKGROUND)
+        self.shader.texture.temporal = 10
+        self.shader.texture.layers = 2
+        self.shader.fragment = (self.directory/"GLSL"/"MotionBlur.frag")
+
+# -------------------------------------------------------------------------------------------------|
+
+class Dynamics(ShaderScene):
+    """Second order system"""
+    __name__ = "Dynamics"
+
+    def build(self):
+        ShaderScene.build(self)
+        ShaderTexture(scene=self, name="background").from_image(BACKGROUND)
+        self.dynamics = ShaderDynamics(scene=self, name="iShaderDynamics", frequency=4)
+        self.shader.fragment = ("""
+            void main() {
+                vec2 uv = zoom(stuv, 0.85 + 0.1*iShaderDynamics, vec2(0.5));
+                fragColor = stexture(background, uv);
+            }
+        """)
+
+    def update(self):
+        self.dynamics.target = 0.5 * (1 + numpy.sign(numpy.sin(2*math.pi*self.time * 0.5)))
+
+# -------------------------------------------------------------------------------------------------|
+
+class Noise(ShaderScene):
+    """Basics of Simplex noise"""
+    __name__ = "Procedural Noise"
+
+    def build(self):
+        ShaderScene.build(self)
+        ShaderTexture(scene=self, name="background").from_image(BACKGROUND)
+        self.shake_noise = ShaderNoise(scene=self, name="Shake", dimensions=2)
+        self.zoom_noise  = ShaderNoise(scene=self, name="Zoom")
+        self.shader.fragment = ("""
+            void main() {
+                vec2 uv = zoom(stuv, 0.95 + 0.02*iZoom, vec2(0.5));
+                uv += 0.02 * iShake;
+                fragColor = stexture(background, uv);
+            }
+        """)
+
+# -------------------------------------------------------------------------------------------------|
+
+class Bouncing(ShaderScene):
+    """Bouncing Logo animation"""
+    __name__ = "Bouncing Logo"
+
+    def build(self):
+        ShaderScene.build(self)
+        LOGO = SHADERFLOW.RESOURCES.ICON
+        self.dvd = ShaderTexture(scene=self, name="logo").from_image(LOGO)
+        self.shader.fragment = (self.directory/"GLSL"/"Bouncing.frag")
+        self.bounce = ShaderBouncing(scene=self)
+        self.bounce.advanced_ratios(LOGO)
+
+    def update(self):
+        self.bounce.aspect_ratio = self.aspect_ratio
+
+    def pipeline(self):
+        yield from ShaderScene.pipeline(self)
+        yield ShaderVariable("uniform", "float", "iLogoSize", 0.3)
+
+# -------------------------------------------------------------------------------------------------|
+
+class Video(ShaderScene):
+    """Video as a Texture demo"""
+    __name__ = "Video"
+
+    def build(self):
+        ShaderScene.build(self)
+        BUNNY = "https://download.blender.org/demo/movies/BBB/bbb_sunflower_1080p_60fps_normal.mp4.zip"
+        self.video = next(BrokenPath.get_external(BUNNY).rglob("*.mp4"))
+        ShaderVideo(scene=self, path=self.video)
+        self.shader.fragment = (self.directory/"GLSL"/"Video.frag")
+
+# -------------------------------------------------------------------------------------------------|
+
+class Audio(ShaderScene):
+    """Basic audio processing"""
+    __name__ = "Audio"
+
+    def build(self):
+        ShaderScene.build(self)
+        self.audio = ShaderAudio(scene=self, name="iAudio")
+        self.audio.open_recorder()
+        self.shader.fragment = ("""
+            void main() {
+                fragColor = vec4(vec3(iAudioVolume), 1.0);
+            }
+        """)
+
+# -------------------------------------------------------------------------------------------------|
+
+class Waveform(ShaderScene):
+    """Audio Waveform Oscilloscope demo"""
+    __name__ = "Waveform"
+
+    def build(self):
+        ShaderScene.build(self)
+        self.audio = ShaderAudio(scene=self, name="iAudio", file="/path/to/audio.ogg")
+        self.waveform = ShaderWaveform(scene=self, audio=self.audio)
+        self.shader.fragment = (self.directory/"GLSL"/"Waveform.frag")
+
+# -------------------------------------------------------------------------------------------------|
+
+class Bars(ShaderScene):
+    """Basic music bars"""
+    __name__ = "Music Bars"
+
+    def build(self):
+        ShaderScene.build(self)
+        self.audio = ShaderAudio(scene=self, name="iAudio", file="/path/to/audio.ogg")
+        self.spectrogram = ShaderSpectrogram(scene=self, audio=self.audio, length=0)
+        self.spectrogram.from_notes(
+            start=BrokenPianoNote.from_frequency(20),
+            end=BrokenPianoNote.from_frequency(18000),
+            piano=True
+        )
+        self.shader.fragment = (self.directory/"GLSL"/"Bars.frag")
+
+# -------------------------------------------------------------------------------------------------|
+
+class Visualizer(ShaderScene):
+    """Radial Bars Music Visualizer Scene"""
+    __name__ = "Visualizer"
+
+    def build(self):
+        ShaderScene.build(self)
+        self.audio = ShaderAudio(scene=self, name="iAudio", file="/path/to/audio.ogg")
+        self.waveform = ShaderWaveform(scene=self, audio=self.audio)
+        self.spectrogram = ShaderSpectrogram(scene=self, length=0, audio=self.audio, smooth=False)
+        self.spectrogram.from_notes(
+            start=BrokenPianoNote.from_frequency(20),
+            end=BrokenPianoNote.from_frequency(14000),
+            piano=True
+        )
+        ShaderTexture(scene=self, name="background").from_image("https://w.wallhaven.cc/full/ex/wallhaven-ex6kmr.jpg")
+        ShaderTexture(scene=self, name="logo").from_image(SHADERFLOW.RESOURCES.ICON)
+        self.shader.fragment = (self.directory/"GLSL"/"Visualizer.frag")
+
+# -------------------------------------------------------------------------------------------------|
+
+class RayMarch(ShaderScene):
+    """Ray Marching demo"""
+    __name__ = "Ray Marching"
+
+    def build(self):
+        ShaderScene.build(self)
+        self.shader.fragment = (self.directory/"GLSL"/"RayMarch.frag")
+
+# -------------------------------------------------------------------------------------------------|
+
+class Batch(ShaderScene):
+    """Batch exporting demo. Run with `shaderflow batch -b 1-3 --base /path/to/folder"""
+    __name__ = "Batch"
+
+    def export_name(self, path: Path) -> Path:
+        return path.with_stem({
+            1: "SubScene A",
+            2: "SubScene B",
+            3: "SubScene C",
+        }[self.export_index])
+
+    def build(self):
+        ShaderScene.build(self)
+
+# -------------------------------------------------------------------------------------------------|
+
+class Life(ShaderScene):
+    """Conway's Game of Life in GLSL"""
+    __name__ = "Game of Life"
+
+    life_period: int = 6
+    """Number of frames between each life update"""
+
+    def setup(self):
+        width, height = 192, 108
+        random = numpy.random.randint(0, 2, (width, height), dtype=bool)
+        self.simulation.texture.size = (width, height)
+        self.simulation.texture.write(random.astype(numpy.float32), temporal=1)
+
+    def build(self):
+        ShaderScene.build(self)
+        self.simulation = ShaderObject(scene=self, name="iLife")
+        self.simulation.texture.temporal = 10
+        self.simulation.texture.filter = TextureFilter.Nearest
+        self.simulation.texture.components = 1
+        self.simulation.texture.track = False
+        self.simulation.fragment = (self.directory/"GLSL"/"Life/Simulation.glsl")
+        self.shader.fragment = (self.directory/"GLSL"/"Life/Visuals.glsl")
+
+    def pipeline(self):
+        yield from ShaderScene.pipeline(self)
+        yield ShaderVariable("uniform", "int", "iLifePeriod", self.life_period)
+
+# -------------------------------------------------------------------------------------------------|

ShaderFlow/Resources/Scenes/Examples/GLSL/Bars.frag

@@ -0,0 +1,20 @@
+/*
+// (c) MIT, Tremeschin
+*/
+
+void main() {
+    vec2 intensity = sqrt(texture(iSpectrogram, astuv.yx).xy)/70;
+
+    if (astuv.y < intensity.x) {
+        fragColor.rgb += vec3(1.0, 0.0, 0.0);
+    }
+    if (astuv.y < intensity.y) {
+        fragColor.rgb += vec3(0.0, 1.0, 0.0);
+    }
+    if (astuv.y < (intensity.y + intensity.x)/2.0) {
+        fragColor.rgb += vec3(0.0, 0.0, 1.0);
+    }
+
+    fragColor.rgb += vec3(0.0, 0.0, 0.4*(intensity.x + intensity.y)*(1.0 - astuv.y));
+    fragColor.a = 1;
+}