react-native-skia 0.0.1 → 3.0.0

package/android/src/main/java/com/reactnative/skia/RNSkVideo.java

@@ -0,0 +1,320 @@
+package com.reactnative.skia;
+
+import android.content.Context;
+import android.graphics.ImageFormat;
+import android.hardware.HardwareBuffer;
+import android.media.AudioAttributes;
+import android.media.AudioManager;
+import android.media.MediaCodec;
+import android.media.MediaExtractor;
+import android.media.MediaFormat;
+import android.media.MediaPlayer;
+import android.media.MediaSync;
+import android.media.Image;
+import android.media.ImageReader;
+import android.net.Uri;
+import android.os.Build;
+import android.view.Surface;
+import android.graphics.Point;
+
+import androidx.annotation.RequiresApi;
+
+import com.facebook.jni.annotations.DoNotStrip;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+public class RNSkVideo {
+    private final Uri uri;
+    private final Context context;
+
+    private MediaExtractor extractor;
+    private MediaCodec decoder;
+    private ImageReader imageReader;
+    private Surface outputSurface;
+    private MediaPlayer mediaPlayer;
+    private MediaSync mediaSync;
+    private double durationMs;
+    private double frameRate;
+    private int rotationDegrees = 0;
+    private int width = 0;
+    private int height = 0;
+
+    private boolean isPlaying = false;
+    private boolean isLooping = false;
+    private boolean isPrepared = false;
+    private boolean playWhenReady = false;
+    private HardwareBuffer lastBuffer = null;
+    private boolean pendingSeek = false;
+    private double lastFrameTimeMs = 0;
+
+    RNSkVideo(Context context, String localUri) {
+        this.uri = Uri.parse(localUri);
+        this.context = context;
+        this.initializeReader();
+    }
+
+    private void initializeReader() {
+        extractor = new MediaExtractor();
+        try {
+            extractor.setDataSource(context, this.uri, null);
+            int trackIndex = selectVideoTrack(extractor);
+            if (trackIndex < 0) {
+                throw new RuntimeException("No video track found in " + this.uri);
+            }
+            extractor.selectTrack(trackIndex);
+            MediaFormat format = extractor.getTrackFormat(trackIndex);
+
+            // Initialize MediaPlayer
+            mediaPlayer = new MediaPlayer();
+            mediaPlayer.setDataSource(context, uri);
+            mediaPlayer.setAudioStreamType(AudioManager.STREAM_MUSIC);
+            mediaPlayer.setOnPreparedListener(mp -> {
+                durationMs = mp.getDuration();
+                isPrepared = true;
+                // Apply looping setting
+                mp.setLooping(isLooping);
+                // Start playing if play() was called before preparation completed
+                if (playWhenReady) {
+                    mp.start();
+                    isPlaying = true;
+                    playWhenReady = false;
+                }
+            });
+            // Note: When setLooping(true) is used, this listener won't fire as
+            // MediaPlayer handles looping automatically. This only handles the
+            // non-looping case to update playback state.
+            mediaPlayer.setOnCompletionListener(mp -> {
+                isPlaying = false;
+            });
+            mediaPlayer.prepareAsync();
+
+            // Retrieve and store video properties
+            if (format.containsKey(MediaFormat.KEY_DURATION)) {
+                durationMs = format.getLong(MediaFormat.KEY_DURATION) / 1000;  // Convert microseconds to milliseconds
+            }
+            if (format.containsKey(MediaFormat.KEY_FRAME_RATE)) {
+                frameRate = format.getInteger(MediaFormat.KEY_FRAME_RATE);
+            }
+            if (format.containsKey(MediaFormat.KEY_ROTATION)) {
+                rotationDegrees = format.getInteger(MediaFormat.KEY_ROTATION);
+            }
+            width = format.getInteger(MediaFormat.KEY_WIDTH);
+            height = format.getInteger(MediaFormat.KEY_HEIGHT);
+            if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
+                imageReader = ImageReader.newInstance(
+                        width,
+                        height,
+                        ImageFormat.PRIVATE,
+                        2,
+                        HardwareBuffer.USAGE_GPU_SAMPLED_IMAGE
+                );
+            } else {
+                imageReader = ImageReader.newInstance(width, height,
+                        ImageFormat.PRIVATE, 2);
+            }
+            outputSurface = imageReader.getSurface();
+
+            // Create a decoder for the format
+            String mime = format.getString(MediaFormat.KEY_MIME);
+            decoder = MediaCodec.createDecoderByType(mime);
+
+            // Note: Use an output Surface for rendering if necessary, otherwise handle buffers
+            decoder.configure(format, outputSurface, null, 0);
+            decoder.start();
+        } catch (IOException e) {
+            throw new RuntimeException("Failed to initialize extractor or decoder", e);
+        }
+    }
+
+    @DoNotStrip
+    public double getDuration() {
+        return durationMs;
+    }
+
+    @DoNotStrip
+    public double getFrameRate() {
+        return frameRate;
+    }
+
+    @DoNotStrip
+    public int getRotationDegrees() {
+        return rotationDegrees;
+    }
+
+    @DoNotStrip
+    public HardwareBuffer nextImage() {
+        // If paused and not seeking, return cached buffer
+        if (!isPlaying && !pendingSeek && lastBuffer != null) {
+            return lastBuffer;
+        }
+
+        // Decode new frame if needed
+        if (!decoderOutputAvailable()) {
+            decodeFrame();
+        }
+
+        Image image = imageReader.acquireLatestImage();
+        if (image != null) {
+            HardwareBuffer hardwareBuffer = image.getHardwareBuffer();
+            image.close();  // Make sure to close the Image to free up the buffer
+            // Cache the buffer
+            lastBuffer = hardwareBuffer;
+            // Clear pending seek flag after getting frame
+            if (pendingSeek) {
+                pendingSeek = false;
+            }
+            return hardwareBuffer;
+        }
+        // Return cached buffer if no new frame available
+        return lastBuffer;
+    }
+
+    @DoNotStrip
+    public void seek(double timestamp) {
+        long timestampUs = (long)(timestamp * 1000); // Convert milliseconds to microseconds
+
+        extractor.seekTo(timestampUs, MediaExtractor.SEEK_TO_CLOSEST_SYNC);
+        if (mediaPlayer != null && isPrepared) {
+            int timestampMs = (int) timestamp; // Convert to milliseconds
+            mediaPlayer.seekTo(timestampMs, MediaPlayer.SEEK_CLOSEST);
+        }
+
+        // Flush the codec to reset internal state and prepare for new position
+        if (decoder != null) {
+            decoder.flush();
+        }
+
+        // Signal that we need to decode a new frame (for seek while paused)
+        pendingSeek = true;
+        // Decode a frame immediately so it's ready
+        decodeFrame();
+    }
+
+    @DoNotStrip
+    public Point getSize() {
+        return new Point(width, height);
+    }
+
+    private int selectVideoTrack(MediaExtractor extractor) {
+        int numTracks = extractor.getTrackCount();
+        for (int i = 0; i < numTracks; i++) {
+            MediaFormat format = extractor.getTrackFormat(i);
+            String mime = format.getString(MediaFormat.KEY_MIME);
+            if (mime.startsWith("video/")) {
+                return i;
+            }
+        }
+        return -1;
+    }
+
+    private boolean decoderOutputAvailable() {
+        MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
+        int outputBufferId = decoder.dequeueOutputBuffer(info, 0);
+        if (outputBufferId >= 0) {
+            // If a buffer is available, release it immediately back since we are just checking
+            decoder.releaseOutputBuffer(outputBufferId, true);
+            return true;
+        }
+        return false;
+    }
+
+    private void decodeFrame() {
+        MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
+        long timeoutUs = 10000;
+        boolean isEOS = false;
+
+        int inputBufferId = decoder.dequeueInputBuffer(timeoutUs);
+        if (inputBufferId >= 0) {
+            ByteBuffer inputBuffer = decoder.getInputBuffer(inputBufferId);
+            int sampleSize = extractor.readSampleData(inputBuffer, 0);
+            if (sampleSize < 0) {
+                // End of stream, make sure to send this information to the decoder
+                decoder.queueInputBuffer(inputBufferId, 0, 0, 0L, MediaCodec.BUFFER_FLAG_END_OF_STREAM);
+                isEOS = true;
+            } else {
+                long presentationTimeUs = extractor.getSampleTime();
+                decoder.queueInputBuffer(inputBufferId, 0, sampleSize, presentationTimeUs, 0);
+                extractor.advance();
+            }
+        }
+
+        int outputBufferId = decoder.dequeueOutputBuffer(info, timeoutUs);
+        if (outputBufferId >= 0) {
+            // Store the frame's presentation timestamp for accurate time reporting
+            lastFrameTimeMs = info.presentationTimeUs / 1000.0;
+            // If we have a valid buffer, release it to make it available to the ImageReader's surface
+            decoder.releaseOutputBuffer(outputBufferId, true);
+
+            if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
+                isEOS = true;
+            }
+        }
+    }
+
+    @DoNotStrip
+    public void play() {
+        if (mediaPlayer == null || isPlaying) {
+            return;
+        }
+        if (isPrepared) {
+            mediaPlayer.start();
+            isPlaying = true;
+        } else {
+            // Queue the play request for when preparation completes
+            playWhenReady = true;
+        }
+    }
+
+    @DoNotStrip
+    public void pause() {
+        playWhenReady = false;
+        if (mediaPlayer != null && isPlaying) {
+            mediaPlayer.pause();
+            isPlaying = false;
+        }
+    }
+
+    @DoNotStrip
+    public void setVolume(float volume) {
+        if (mediaPlayer != null) {
+            mediaPlayer.setVolume(volume, volume);
+        }
+    }
+
+    @DoNotStrip
+    public double getCurrentTime() {
+        // Return the timestamp of the last decoded frame for accurate synchronization
+        return lastFrameTimeMs;
+    }
+
+    @DoNotStrip
+    public void setLooping(boolean looping) {
+        isLooping = looping;
+        if (mediaPlayer != null && isPrepared) {
+            mediaPlayer.setLooping(looping);
+        }
+    }
+
+    @DoNotStrip
+    public boolean getIsPlaying() {
+        return isPlaying;
+    }
+
+    public void release() {
+        lastBuffer = null;
+        if (mediaPlayer != null) {
+            mediaPlayer.release();
+            mediaPlayer = null;
+        }
+        if (decoder != null) {
+            decoder.stop();
+            decoder.release();
+            decoder = null;
+        }
+        if (extractor != null) {
+            extractor.release();
+            extractor = null;
+        }
+    }
+}

package/android/src/main/java/com/reactnative/skia/RNSkiaModule.java

@@ -0,0 +1,67 @@
+// RnskiaModule.java
+
+package com.reactnative.skia;
+
+import android.util.Log;
+
+import com.facebook.react.bridge.LifecycleEventListener;
+import com.facebook.react.bridge.ReactApplicationContext;
+import com.facebook.react.bridge.ReactMethod;
+import com.facebook.react.module.annotations.ReactModule;
+
+import java.lang.ref.WeakReference;
+
+@ReactModule(name="RNSkiaModule")
+public class RNSkiaModule extends NativeSkiaModuleSpec {
+    public static final String NAME = "RNSkiaModule";
+
+    private final WeakReference<ReactApplicationContext> weakReactContext;
+    private SkiaManager skiaManager;
+
+    public RNSkiaModule(ReactApplicationContext reactContext) {
+        super(reactContext);
+        this.weakReactContext = new WeakReference<>(reactContext);
+    }
+
+    @Override
+    public void invalidate() {
+        super.invalidate();
+
+        if (this.skiaManager != null) {
+            this.skiaManager.invalidate();
+            this.skiaManager.destroy();
+            this.skiaManager = null;
+        }
+    }
+
+    @Override
+    public String getName() {
+        return NAME;
+    }
+
+    public SkiaManager getSkiaManager() {
+        return skiaManager;
+    }
+
+    @ReactMethod(isBlockingSynchronousMethod = true)
+    public boolean install() {
+        if (skiaManager != null) {
+            // Already initialized, ignore call.
+            return true;
+        }
+
+        try {
+            System.loadLibrary("rnskia");
+            ReactApplicationContext context = weakReactContext.get();
+            if (context == null) {
+                Log.e(NAME, "React Application Context was null!");
+                return false;
+            }
+            skiaManager = new SkiaManager(context);
+            return true;
+        } catch (Exception exception) {
+            Log.e(NAME, "Failed to initialize Skia Manager!", exception);
+            return false;
+        }
+    }
+}

package/android/src/main/java/com/reactnative/skia/RNSkiaPackage.java

@@ -0,0 +1,58 @@
+// RnskiaPackage.java
+
+package com.reactnative.skia;
+
+import androidx.annotation.Nullable;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import com.facebook.react.BaseReactPackage;
+import com.facebook.react.ReactPackage;
+import com.facebook.react.bridge.NativeModule;
+import com.facebook.react.bridge.ReactApplicationContext;
+import com.facebook.react.module.model.ReactModuleInfo;
+import com.facebook.react.module.model.ReactModuleInfoProvider;
+import com.facebook.react.uimanager.ViewManager;
+
+// `implements ReactPackage` is redundant (BaseReactPackage already implements it)
+// but required: older @react-native-community/cli versions only autolink
+// classes matching `implements ReactPackage` or `extends TurboReactPackage`.
+public class RNSkiaPackage extends BaseReactPackage implements ReactPackage {
+    @Nullable
+    @Override
+    public NativeModule getModule(String name, ReactApplicationContext reactApplicationContext) {
+        if (RNSkiaModule.NAME.equals(name)) {
+            return new RNSkiaModule(reactApplicationContext);
+        }
+        return null;
+    }
+
+    @Override
+    public List<ViewManager> createViewManagers(ReactApplicationContext reactContext) {
+        return Arrays.<ViewManager>asList(
+            new SkiaPictureViewManager(),
+            new SkiaGraphiteViewManager()
+        );
+    }
+
+    @Override
+    public ReactModuleInfoProvider getReactModuleInfoProvider() {
+        return () -> {
+            final Map<String, ReactModuleInfo> reactModuleInfoMap = new HashMap<>();
+            reactModuleInfoMap.put(
+                    RNSkiaModule.NAME,
+                    new ReactModuleInfo(
+                            RNSkiaModule.NAME,
+                            RNSkiaModule.class.getName(),
+                            false, // canOverrideExistingModule
+                            false, // needsEagerInit
+                            false, // isCxxModule
+                            true   // isTurboModule
+                    ));
+            return reactModuleInfoMap;
+        };
+    }
+}

package/android/src/main/java/com/reactnative/skia/SkiaBaseView.java

@@ -0,0 +1,185 @@
+package com.reactnative.skia;
+
+import android.content.Context;
+import android.graphics.SurfaceTexture;
+import android.util.Log;
+import android.view.MotionEvent;
+import android.view.Surface;
+import android.view.View;
+
+import com.facebook.react.uimanager.PointerEvents;
+import com.facebook.react.views.view.ReactViewGroup;
+
+public abstract class SkiaBaseView extends ReactViewGroup implements SkiaViewAPI {
+    // Backing view kinds, see updateView().
+    private static final int KIND_SURFACE_VIEW = 0;
+    private static final int KIND_TEXTURE_VIEW = 1;
+
+    private View mView;
+
+    // Props, applied together in updateView().
+    private boolean mOpaque = false;
+    private String mSurfaceType = "auto";
+    private boolean mZOrderOnTop = false;
+    private boolean mHighBitDepth = false;
+
+    // What the current backing view was created with.
+    private int mAppliedKind = -1;
+    private boolean mAppliedZOrderOnTop;
+    private boolean mAppliedHighBitDepth;
+
+    private final boolean debug = false;
+    private final String tag = "SkiaView";
+
+    public SkiaBaseView(Context context) {
+        super(context);
+    }
+
+    @Override
+    public boolean dispatchTouchEvent(MotionEvent ev) {
+        // When pointerEvents is "none" or "box-none", make this view completely
+        // transparent to touch dispatch so events pass through to views behind it
+        if (!PointerEvents.canBeTouchTarget(getPointerEvents())) {
+            return false;
+        }
+        return super.dispatchTouchEvent(ev);
+    }
+
+    public void setOpaque(boolean value) {
+        mOpaque = value;
+    }
+
+    public void setSurfaceType(String value) {
+        mSurfaceType = value == null ? "auto" : value;
+    }
+
+    public void setZOrderOnTop(boolean value) {
+        mZOrderOnTop = value;
+    }
+
+    public void setHighBitDepth(boolean value) {
+        mHighBitDepth = value;
+    }
+
+    // Resolve the backing view from the props. "auto" picks SurfaceView for an
+    // opaque canvas and TextureView for a non-opaque one.
+    private int resolveKind() {
+        if ("SurfaceView".equals(mSurfaceType)) {
+            return KIND_SURFACE_VIEW;
+        }
+        if ("TextureView".equals(mSurfaceType)) {
+            return KIND_TEXTURE_VIEW;
+        }
+        return mOpaque ? KIND_SURFACE_VIEW : KIND_TEXTURE_VIEW;
+    }
+
+    // The 10-bit buffer format only has 2 bits of alpha, which would visibly
+    // break translucency, and the extra precision would be lost in the 8-bit
+    // composition pass of a TextureView anyway. So the flag only applies to an
+    // opaque SurfaceView.
+    private boolean resolveHighBitDepth(int kind) {
+        if (!mHighBitDepth) {
+            return false;
+        }
+        if (kind != KIND_SURFACE_VIEW || !mOpaque) {
+            Log.w(tag, "highBitDepth requires an opaque SurfaceView on Android, falling back to the 8-bit format");
+            return false;
+        }
+        return true;
+    }
+
+    // Apply the complete prop transaction once, after every prop has arrived.
+    // Only a change of backing view, or of a setting the view must know before
+    // it attaches (zOrderOnTop, the buffer format), replaces the child; opacity
+    // is applied in place.
+    void updateView() {
+        int kind = resolveKind();
+        boolean zOrderOnTop = kind == KIND_SURFACE_VIEW && mZOrderOnTop;
+        boolean highBitDepth = resolveHighBitDepth(kind);
+        if (mView == null
+                || kind != mAppliedKind
+                || zOrderOnTop != mAppliedZOrderOnTop
+                || highBitDepth != mAppliedHighBitDepth) {
+            if (mView != null) {
+                removeView(mView);
+            }
+            mAppliedKind = kind;
+            mAppliedZOrderOnTop = zOrderOnTop;
+            mAppliedHighBitDepth = highBitDepth;
+            mView = kind == KIND_SURFACE_VIEW
+                    ? new SkiaSurfaceView(getContext(), this, debug, zOrderOnTop, mOpaque)
+                    : new SkiaTextureView(getContext(), this, debug, mOpaque);
+            addView(mView);
+            // React Native sizes native children explicitly through onLayout, so
+            // the requestLayout triggered by addView is ignored; size the new
+            // child ourselves or it stays 0x0 and never gets a surface.
+            if (getWidth() > 0 || getHeight() > 0) {
+                mView.layout(0, 0, getWidth(), getHeight());
+            }
+        } else if (kind == KIND_SURFACE_VIEW) {
+            ((SkiaSurfaceView) mView).setOpaque(mOpaque);
+        } else {
+            ((SkiaTextureView) mView).setOpaque(mOpaque);
+        }
+    }
+
+    void dropInstance() {
+        unregisterView();
+    }
+
+    @Override
+    protected void onLayout(boolean changed, int left, int top, int right, int bottom) {
+        super.onLayout(changed, left, top, right, bottom);
+        if (mView != null) {
+            mView.layout(0, 0, right - left, bottom - top);
+        }
+    }
+
+    // SurfaceView callbacks: the native side receives an android.view.Surface.
+
+    @Override
+    public void onSurfaceCreated(Surface surface, int width, int height) {
+        surfaceAvailable(surface, width, height, true, mAppliedHighBitDepth);
+    }
+
+    @Override
+    public void onSurfaceChanged(Surface surface, int width, int height) {
+        Log.i(tag, "onSurfaceChanged " + width + "/" + height);
+        surfaceSizeChanged(surface, width, height, true, mAppliedHighBitDepth);
+    }
+
+    // TextureView callbacks: the native side receives the SurfaceTexture and
+    // creates the Surface it draws into. The 10-bit format is never used here.
+
+    @Override
+    public void onSurfaceTextureCreated(SurfaceTexture surface, int width, int height) {
+        surfaceAvailable(surface, width, height, false, false);
+    }
+
+    @Override
+    public void onSurfaceTextureChanged(SurfaceTexture surface, int width, int height) {
+        Log.i(tag, "onSurfaceTextureSizeChanged " + width + "/" + height);
+        surfaceSizeChanged(surface, width, height, false, false);
+    }
+
+    @Override
+    public void onSurfaceDestroyed() {
+        surfaceDestroyed();
+    }
+
+    // isSurface tells the native side whether `surface` is an
+    // android.view.Surface (SurfaceView) or a SurfaceTexture (TextureView).
+    protected abstract void surfaceAvailable(Object surface, int width, int height, boolean isSurface, boolean highBitDepth);
+
+    protected abstract void surfaceSizeChanged(Object surface, int width, int height, boolean isSurface, boolean highBitDepth);
+
+    protected abstract void surfaceDestroyed();
+
+    protected abstract void setDebugMode(boolean show);
+
+    protected abstract void registerView(int nativeId);
+
+    protected abstract void unregisterView();
+
+    protected abstract int[] getBitmap(int width, int height);
+}

package/android/src/main/java/com/reactnative/skia/SkiaBaseViewManager.java

@@ -0,0 +1,65 @@
+package com.reactnative.skia;
+
+import com.facebook.react.uimanager.PointerEvents;
+import com.facebook.react.uimanager.ViewProps;
+import com.facebook.react.uimanager.annotations.ReactProp;
+import com.facebook.react.views.view.ReactViewGroup;
+import com.facebook.react.views.view.ReactViewManager;
+
+import androidx.annotation.NonNull;
+import androidx.annotation.Nullable;
+
+public abstract class SkiaBaseViewManager<T extends SkiaBaseView> extends ReactViewManager {
+
+    @Override
+    public void setNativeId(@NonNull ReactViewGroup view, @Nullable String nativeId) {
+        super.setNativeId(view, nativeId);
+        int nativeIdResolved = Integer.parseInt(nativeId);
+        ((SkiaBaseView)view).registerView(nativeIdResolved);
+    }
+
+    @ReactProp(name = "debug")
+    public void setDebug(T view, boolean show) {
+        ((SkiaBaseView)view).setDebugMode(show);
+    }
+
+    @ReactProp(name = "opaque")
+    public void setOpaque(T view, boolean value) {
+        ((SkiaBaseView)view).setOpaque(value);
+    }
+
+    @ReactProp(name = "highBitDepth")
+    public void setHighBitDepth(T view, boolean value) {
+        ((SkiaBaseView)view).setHighBitDepth(value);
+    }
+
+    @ReactProp(name = "androidSurfaceType")
+    public void setAndroidSurfaceType(T view, @Nullable String value) {
+        ((SkiaBaseView)view).setSurfaceType(value);
+    }
+
+    @ReactProp(name = "androidZOrderOnTop")
+    public void setAndroidZOrderOnTop(T view, boolean value) {
+        ((SkiaBaseView)view).setZOrderOnTop(value);
+    }
+
+    // The backing view depends on several props (opaque, androidSurfaceType,
+    // androidZOrderOnTop, highBitDepth), so it is resolved once per transaction
+    // rather than in each setter.
+    @Override
+    protected void onAfterUpdateTransaction(@NonNull ReactViewGroup view) {
+        super.onAfterUpdateTransaction(view);
+        ((SkiaBaseView)view).updateView();
+    }
+
+    @ReactProp(name = ViewProps.POINTER_EVENTS)
+    public void setPointerEvents(T view, @Nullable String pointerEventsStr) {
+        view.setPointerEvents(PointerEvents.parsePointerEvents(pointerEventsStr));
+    }
+
+    @Override
+    public void onDropViewInstance(@NonNull ReactViewGroup view) {
+        super.onDropViewInstance(view);
+        ((SkiaBaseView)view).dropInstance();
+    }
+}