vision-camera-face-detection 1.2.0

package/src/Camera.tsx

@@ -0,0 +1,192 @@
+import React from 'react';
+import {
+  Camera as VisionCamera,
+  // runAsync,
+  useFrameProcessor,
+} from 'react-native-vision-camera';
+import {
+  Worklets,
+  // useRunOnJS,
+  useSharedValue,
+} from 'react-native-worklets-core';
+import { useFaceDetector } from './FaceDetector';
+
+// types
+import type { DependencyList, ForwardedRef } from 'react';
+import type {
+  CameraProps,
+  Frame,
+  FrameInternal,
+} from 'react-native-vision-camera';
+import type { Face, FaceDetectionOptions } from './FaceDetector';
+
+type UseWorkletType = (frame: FrameInternal) => Promise<void>;
+
+type UseRunInJSType = (
+  faces: Face[],
+  frame: Frame
+) => Promise<void | Promise<void>>;
+
+type CallbackType = (faces: Face[], frame: Frame) => void | Promise<void>;
+
+type ComponentType = {
+  faceDetectionOptions?: FaceDetectionOptions;
+  faceDetectionCallback: CallbackType;
+} & CameraProps;
+
+/**
+ * Create a Worklet function that persists between re-renders.
+ * The returned function can be called from both a Worklet context and the JS context, but will execute on a Worklet context.
+ *
+ * @param {function} func The Worklet. Must be marked with the `'worklet'` directive.
+ * @param {DependencyList} dependencyList The React dependencies of this Worklet.
+ * @returns {UseWorkletType} A memoized Worklet
+ */
+function useWorklet(
+  func: (frame: FrameInternal) => void,
+  dependencyList: DependencyList
+): UseWorkletType {
+  const worklet = React.useMemo(() => {
+    const context = Worklets.defaultContext;
+    return context.createRunAsync(func);
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, dependencyList);
+
+  return worklet;
+}
+
+/**
+ * Create a Worklet function that runs the giver function on JS context.
+ * The returned function can be called from a Worklet to hop back to the JS thread.
+ *
+ * @param {function} func The Worklet. Must be marked with the `'worklet'` directive.
+ * @param {DependencyList} dependencyList The React dependencies of this Worklet.
+ * @returns {UseRunInJSType} a memoized Worklet
+ */
+function useRunInJS(
+  func: CallbackType,
+  dependencyList: DependencyList
+): UseRunInJSType {
+  // eslint-disable-next-line react-hooks/exhaustive-deps
+  return React.useMemo(() => Worklets.createRunOnJS(func), dependencyList);
+}
+
+/**
+ * Vision camera wrapper
+ *
+ * @param {ComponentType} props Camera + face detection props
+ * @returns
+ */
+export const Camera = React.forwardRef(
+  (
+    { faceDetectionOptions, faceDetectionCallback, ...props }: ComponentType,
+    ref: ForwardedRef<VisionCamera>
+  ) => {
+    const { detectFaces } = useFaceDetector(faceDetectionOptions);
+    /**
+     * Is there an async task already running?
+     */
+    const isAsyncContextBusy = useSharedValue(false);
+
+    /**
+     * Throws logs/errors back on js thread
+     */
+    const logOnJs = Worklets.createRunOnJS((log: string, error?: Error) => {
+      if (error) {
+        console.error(log, error.message ?? JSON.stringify(error));
+      } else {
+        console.log(log);
+      }
+    });
+
+    /**
+     * Runs on detection callback on js thread
+     */
+    const runOnJs = useRunInJS(faceDetectionCallback, [faceDetectionCallback]);
+
+    /**
+     * Async context that will handle face detection
+     */
+    const runOnAsyncContext = useWorklet(
+      (frame: FrameInternal) => {
+        'worklet';
+        try {
+          const faces = detectFaces(frame);
+          // increment frame count so we can use frame on
+          // js side without frame processor getting stuck
+          frame.incrementRefCount();
+          runOnJs(faces, frame).finally(() => {
+            'worklet';
+            // finally decrement frame count so it can be dropped
+            frame.decrementRefCount();
+          });
+        } catch (error: any) {
+          logOnJs('Execution error:', error);
+        } finally {
+          frame.decrementRefCount();
+          isAsyncContextBusy.value = false;
+        }
+      },
+      [detectFaces, runOnJs]
+    );
+
+    /**
+     * Detect faces on frame on an async context without blocking camera preview
+     *
+     * @param {Frame} frame Current frame
+     */
+    function runAsync(frame: Frame) {
+      'worklet';
+      if (isAsyncContextBusy.value) return;
+      // set async context as busy
+      isAsyncContextBusy.value = true;
+      // cast to internal frame and increment ref count
+      const internal = frame as FrameInternal;
+      internal.incrementRefCount();
+      // detect faces in async context
+      runOnAsyncContext(internal);
+    }
+
+    /**
+     * Camera frame processor
+     */
+    const cameraFrameProcessor = useFrameProcessor(
+      (frame) => {
+        'worklet';
+        runAsync(frame);
+      },
+      [runOnAsyncContext]
+    );
+
+    //
+    // use bellow when vision-camera's
+    // context creation issue is solved
+    //
+    // /**
+    //  * Runs on detection callback on js thread
+    //  */
+    // const runOnJs = useRunOnJS( faceDetectionCallback, [
+    //   faceDetectionCallback
+    // ] )
+
+    // const cameraFrameProcessor = useFrameProcessor( ( frame ) => {
+    //   'worklet'
+    //   runAsync( frame, () => {
+    //     'worklet'
+    //     runOnJs(
+    //       detectFaces( frame ),
+    //       frame
+    //     )
+    //   } )
+    // }, [ runOnJs ] )
+
+    return (
+      <VisionCamera
+        {...props}
+        ref={ref}
+        frameProcessor={cameraFrameProcessor}
+        pixelFormat="yuv"
+      />
+    );
+  }
+);

package/src/FaceDetector.ts

@@ -0,0 +1,161 @@
+import { useMemo } from 'react';
+import { VisionCameraProxy, type Frame } from 'react-native-vision-camera';
+
+type FaceDetectorPlugin = {
+  /**
+   * Detect faces on frame
+   *
+   * @param {Frame} frame Frame to detect faces
+   */
+  detectFaces: (frame: Frame) => Face[];
+};
+
+type Point = {
+  x: number;
+  y: number;
+};
+
+export interface Face {
+  pitchAngle: number;
+  rollAngle: number;
+  yawAngle: number;
+  bounds: Bounds;
+  leftEyeOpenProbability: number;
+  rightEyeOpenProbability: number;
+  smilingProbability: number;
+  contours: Contours;
+  landmarks: Landmarks;
+  data: number[];
+}
+
+export interface Bounds {
+  width: number;
+  height: number;
+  x: number;
+  y: number;
+}
+
+export interface Contours {
+  FACE: Point[];
+  LEFT_EYEBROW_TOP: Point[];
+  LEFT_EYEBROW_BOTTOM: Point[];
+  RIGHT_EYEBROW_TOP: Point[];
+  RIGHT_EYEBROW_BOTTOM: Point[];
+  LEFT_EYE: Point[];
+  RIGHT_EYE: Point[];
+  UPPER_LIP_TOP: Point[];
+  UPPER_LIP_BOTTOM: Point[];
+  LOWER_LIP_TOP: Point[];
+  LOWER_LIP_BOTTOM: Point[];
+  NOSE_BRIDGE: Point[];
+  NOSE_BOTTOM: Point[];
+  LEFT_CHEEK: Point[];
+  RIGHT_CHEEK: Point[];
+}
+
+export interface Landmarks {
+  LEFT_CHEEK: Point;
+  LEFT_EAR: Point;
+  LEFT_EYE: Point;
+  MOUTH_BOTTOM: Point;
+  MOUTH_LEFT: Point;
+  MOUTH_RIGHT: Point;
+  NOSE_BASE: Point;
+  RIGHT_CHEEK: Point;
+  RIGHT_EAR: Point;
+  RIGHT_EYE: Point;
+}
+
+export interface FaceDetectionOptions {
+  /**
+   * Favor speed or accuracy when detecting faces.
+   *
+   * @default 'fast'
+   */
+  performanceMode?: 'fast' | 'accurate';
+
+  /**
+   * Whether to attempt to identify facial 'landmarks': eyes, ears, nose, cheeks, mouth, and so on.
+   *
+   * @default 'none'
+   */
+  landmarkMode?: 'none' | 'all';
+
+  /**
+   * Whether to detect the contours of facial features. Contours are detected for only the most prominent face in an image.
+   *
+   * @default 'none'
+   */
+  contourMode?: 'none' | 'all';
+
+  /**
+   * Whether or not to classify faces into categories such as 'smiling', and 'eyes open'.
+   *
+   * @default 'none'
+   */
+  classificationMode?: 'none' | 'all';
+
+  /**
+   * Sets the smallest desired face size, expressed as the ratio of the width of the head to width of the image.
+   *
+   * @default 0.15
+   */
+  minFaceSize?: number;
+
+  /**
+   * Whether or not to assign faces an ID, which can be used to track faces across images.
+   *
+   * Note that when contour detection is enabled, only one face is detected, so face tracking doesn't produce useful results. For this reason, and to improve detection speed, don't enable both contour detection and face tracking.
+   *
+   * @default false
+   */
+  trackingEnabled?: boolean;
+
+  /**
+   * Should auto scale face bounds, contour and landmarks on native side?
+   * This option should be disabled if you want to draw on frame using `Skia Frame Processor`.
+   * See [this](https://github.com/luicfrr/react-native-vision-camera-face-detector/issues/30#issuecomment-2058805546) and [this](https://github.com/luicfrr/react-native-vision-camera-face-detector/issues/35) for more details.
+   *
+   * @default false
+   */
+  autoScale?: boolean;
+}
+
+/**
+ * Create a new instance of face detector plugin
+ *
+ * @param {FaceDetectionOptions | undefined} options Detection options
+ * @returns {FaceDetectorPlugin} Plugin instance
+ */
+function createFaceDetectorPlugin(
+  options?: FaceDetectionOptions
+): FaceDetectorPlugin {
+  const plugin = VisionCameraProxy.initFrameProcessorPlugin('detectFaces', {
+    ...options,
+  });
+
+  if (!plugin) {
+    throw new Error('Failed to load Frame Processor Plugin "detectFaces"!');
+  }
+
+  return {
+    detectFaces: (frame: Frame): Face[] => {
+      'worklet';
+      // @ts-ignore
+      return plugin.call(frame) as Face[];
+    },
+  };
+}
+
+/**
+ * Use an instance of face detector plugin.
+ *
+ * @param {FaceDetectionOptions | undefined} options Detection options
+ * @returns {FaceDetectorPlugin} Memoized plugin instance that will be
+ * destroyed once the component using `useFaceDetector()` unmounts.
+ */
+export function useFaceDetector(
+  options?: FaceDetectionOptions
+): FaceDetectorPlugin {
+  return useMemo(() => createFaceDetectorPlugin(options), [options]);
+}

package/src/Tensor.ts

@@ -0,0 +1,27 @@
+import { NativeModules, Platform } from 'react-native';
+
+const LINKING_ERROR =
+  `The package 'vision-camera-face-detection' doesn't seem to be linked. Make sure: \n\n` +
+  Platform.select({ ios: "- You have run 'pod install'\n", default: '' }) +
+  '- You rebuilt the app after installing the package\n' +
+  '- You are not using Expo Go\n';
+
+const VisionCameraFaceDetectionModule =
+  NativeModules.VisionCameraFaceDetectionModule
+    ? NativeModules.VisionCameraFaceDetectionModule
+    : new Proxy(
+        {},
+        {
+          get() {
+            throw new Error(LINKING_ERROR);
+          },
+        }
+      );
+
+export function initTensor(modelPath: string, count?: number): Promise<string> {
+  return VisionCameraFaceDetectionModule.initTensor(modelPath, count);
+}
+
+export function detectFromBase64(imageString: string): Promise<any> {
+  return VisionCameraFaceDetectionModule.detectFromBase64(imageString);
+}

package/src/index.tsx

@@ -0,0 +1,3 @@
+export * from './Camera';
+export * from './FaceDetector';
+export * from './Tensor';