react-native-tgfx 0.0.0 → 0.0.2-alpha.0

package/src/hooks/useCameraFrame.test.tsx

@@ -0,0 +1,286 @@
+import { afterAll, beforeEach, expect, mock, test } from 'bun:test'
+import React, { useState } from 'react'
+import TestRenderer, { act } from 'react-test-renderer'
+import { compileScene } from '../compiler/compile'
+import { SceneWriter } from '../compiler/writer'
+import { NodeType, type Node } from '../reconciler/types'
+import { nativeMocks } from '../testing/nativeMocks'
+import type {
+  CameraFrameOptions,
+  CameraFrameSource,
+  FrameLike,
+} from './useCameraFrame'
+
+// A real SharedValue is stable per hook instance; a fresh object would hide the identity bugs.
+mock.module('react-native-reanimated', () => ({
+  useSharedValue: <T,>(initial: T) => useState(() => ({ value: initial }))[0],
+}))
+mock.module('react-native-worklets', () => ({
+  scheduleOnRN: <A extends unknown[]>(fn: (...a: A) => void, ...a: A) => {
+    fn(...a)
+  },
+}))
+
+class FakeImage {
+  handle = 7
+  width = 720
+  height = 1280
+  buffers: [bigint, string][] = []
+  disposed = false
+  setNativeBuffer(pointer: bigint, format: string): void {
+    this.buffers.push([pointer, format])
+  }
+  dispose(): void {
+    this.disposed = true
+  }
+}
+
+let made: FakeImage[] = []
+let imported: [bigint, string][] = []
+let released = 0
+let disposed = 0
+let importThrows = false
+
+const previousHybrid = nativeMocks.hybridObject
+afterAll(() => {
+  nativeMocks.hybridObject = previousHybrid
+})
+
+beforeEach(() => {
+  made = []
+  imported = []
+  released = 0
+  disposed = 0
+  importThrows = false
+  nativeMocks.hybridObject = () => ({
+    fromNativeBuffer(pointer: bigint, format: string) {
+      if (importThrows) {
+        throw new Error('import failed')
+      }
+      imported.push([pointer, format])
+      const image = new FakeImage()
+      made.push(image)
+      return image
+    },
+  })
+})
+
+const { useCameraFrame } =
+  require('./useCameraFrame') as typeof import('./useCameraFrame')
+
+function frame(over: Partial<FrameLike> = {}): FrameLike {
+  return {
+    getNativeBuffer: () => ({
+      pointer: 0x1000n as never,
+      release: () => {
+        released++
+      },
+    }),
+    pixelFormat: 'yuv-420-8-bit-video',
+    orientation: 'up',
+    isMirrored: false,
+    width: 720,
+    height: 1280,
+    dispose: () => {
+      disposed++
+    },
+    ...over,
+  }
+}
+
+async function mount(
+  onEachFrame?: (f: FrameLike) => void,
+  options?: CameraFrameOptions
+) {
+  let seen!: CameraFrameSource
+  function Probe(): null {
+    seen = useCameraFrame(onEachFrame, options)
+    return null
+  }
+  let tree!: TestRenderer.ReactTestRenderer
+  await act(async () => {
+    tree = TestRenderer.create(<Probe />)
+  })
+  return {
+    get current() {
+      return seen
+    },
+    async push(over: Partial<FrameLike> = {}) {
+      await act(async () => {
+        seen.onFrame(frame(over))
+      })
+    },
+    async unmount() {
+      await act(async () => {
+        tree.unmount()
+      })
+    },
+  }
+}
+
+function quarterTurns(rotation: number): number {
+  const node = {
+    type: NodeType.CameraFrame,
+    props: { image: { handle: 1 }, rotation },
+    children: [],
+  } as unknown as Node
+  const result = compileScene([node], new SceneWriter())
+  return new Float64Array(result.buffer)[8]!
+}
+
+// A ROTATION map with left and right transposed passes everything else and is 180° out on device.
+test('an orientation reaches the draw program as the turns that cancel it', async () => {
+  const probe = await mount()
+  // 'right' means the pixels are +90 from upright, so it takes 270 to undo.
+  const cases: [string, number, number][] = [
+    ['up', 0, 0],
+    ['right', 270, 3],
+    ['down', 180, 2],
+    ['left', 90, 1],
+  ]
+  for (const [orientation, rotation, turns] of cases) {
+    await probe.push({ orientation })
+    expect(probe.current.posture.rotation).toBe(rotation)
+    expect(quarterTurns(probe.current.posture.rotation)).toBe(turns)
+  }
+  await probe.push({ orientation: 'sideways' })
+  expect(probe.current.posture.rotation).toBe(0)
+})
+
+test('mirror and frame size come off the frame, and only re-render on a change', async () => {
+  let renders = 0
+  function Counter(): null {
+    renders++
+    return null
+  }
+  let seen!: CameraFrameSource
+  function Probe(): null {
+    seen = useCameraFrame()
+    Counter()
+    return null
+  }
+  await act(async () => {
+    TestRenderer.create(<Probe />)
+  })
+  await act(async () => {
+    seen.onFrame(frame({ isMirrored: true }))
+  })
+  expect(seen.posture).toEqual({
+    rotation: 0,
+    mirror: true,
+    width: 720,
+    height: 1280,
+  })
+  const after = renders
+  for (let i = 0; i < 10; i++) {
+    await act(async () => {
+      seen.onFrame(frame({ isMirrored: true }))
+    })
+  }
+  expect(renders).toBe(after)
+})
+
+test('the first frame creates the handle and every later one re-points it', async () => {
+  const probe = await mount()
+  await probe.push()
+  expect(made).toHaveLength(1)
+  expect(probe.current.image).toBe(made[0] as never)
+
+  await probe.push()
+  await probe.push()
+  expect(made).toHaveLength(1)
+  expect(made[0]!.buffers).toHaveLength(2)
+})
+
+// An external entry is never evicted, so every mount would leak a texture and its buffer.
+test('the handle is disposed on unmount', async () => {
+  const probe = await mount()
+  await probe.push()
+  expect(made[0]!.disposed).toBe(false)
+
+  await probe.unmount()
+  expect(made[0]!.disposed).toBe(true)
+
+  const empty = await mount()
+  await empty.unmount()
+})
+
+test('the range suffix is read off the frame, not hardcoded', async () => {
+  const probe = await mount()
+  await probe.push({ pixelFormat: 'yuv-420-8-bit-full' })
+  expect(imported[0]![1]).toBe('bt601-full')
+  await probe.push({ pixelFormat: 'yuv-420-8-bit-video' })
+  expect(made[0]!.buffers[0]![1]).toBe('bt601-limited')
+})
+
+// A screen recorder or decoder hands over RGBA, and importing that as BT.601 reads it as luma.
+test('a non-YUV producer is not imported as BT.601', async () => {
+  const probe = await mount()
+  await probe.push({ pixelFormat: 'rgba-8-bit' })
+  expect(imported[0]![1]).toBe('rgba')
+})
+
+test('the colour matrix follows the producer, not a default', async () => {
+  const p709 = await mount()
+  await p709.push({ pixelFormat: 'yuv-420-10-bit-bt709-full' })
+  expect(imported[0]![1]).toBe('bt709-full')
+
+  imported.length = 0
+  made.length = 0
+  const p2020 = await mount()
+  await p2020.push({ pixelFormat: 'yuv-420-10-bit-bt2020-video' })
+  expect(imported[0]![1]).toBe('bt2020-limited')
+})
+
+test('the caller can override the format mapping entirely', async () => {
+  const probe = await mount(undefined, { format: () => 'jpeg-full' })
+  await probe.push({ pixelFormat: 'yuv-420-8-bit-video' })
+  expect(imported[0]![1]).toBe('jpeg-full')
+})
+
+test('the caller can override the rotation mapping', async () => {
+  const probe = await mount(undefined, { rotation: () => 90 })
+  await probe.push({ orientation: 'up' })
+  expect(probe.current.posture.rotation).toBe(90)
+})
+
+test('the caller callback runs while the frame is still alive', async () => {
+  const aliveAt: number[] = []
+  const probe = await mount(() => {
+    aliveAt.push(disposed)
+  })
+  await probe.push()
+  expect(aliveAt).toEqual([0])
+  expect(disposed).toBe(1)
+})
+
+test('a throw inside still releases the buffer and disposes the frame', async () => {
+  const probe = await mount()
+
+  importThrows = true
+  await expect(probe.push()).rejects.toThrow('import failed')
+  expect(released).toBe(1)
+  expect(disposed).toBe(1)
+
+  // A leaked frame stalls the producer's bounded pool, which is what kills the stream.
+  importThrows = false
+  const boom = await mount(() => {
+    throw new Error('detector failed')
+  })
+  await expect(boom.push()).rejects.toThrow('detector failed')
+  expect(released).toBe(2)
+  expect(disposed).toBe(2)
+})
+
+test('getNativeBuffer throwing still disposes the frame', async () => {
+  const probe = await mount()
+  await expect(
+    probe.push({
+      getNativeBuffer: () => {
+        throw new Error('no buffer')
+      },
+    })
+  ).rejects.toThrow('no buffer')
+  expect(released).toBe(0)
+  expect(disposed).toBe(1)
+})

package/src/hooks/useCameraFrame.ts

@@ -0,0 +1,222 @@
+import { useCallback, useEffect, useState } from 'react'
+import type { UInt64 } from 'react-native-nitro-modules'
+import { requireReanimated, worklets } from '../reanimated/peer'
+import type { PixelFormat, TgfxImage } from '../specs/TgfxImage.nitro'
+import { imageFromNativeBuffer } from './useImage'
+
+/** The buffer a {@link FrameLike} hands over, valid only inside the callback. */
+export interface NativeBufferLike {
+  pointer: UInt64
+  release(): void
+}
+
+/**
+ * What {@link useCameraFrame} needs of a frame, structurally — so it works for
+ * a VisionCamera `Frame` and for any other producer, and costs this package no
+ * dependency on either.
+ */
+export interface FrameLike {
+  getNativeBuffer(): NativeBufferLike
+  pixelFormat: string
+  orientation: string
+  isMirrored: boolean
+  width: number
+  height: number
+  dispose(): void
+}
+
+/** How the buffer is oriented, read off the frame rather than assumed. */
+export interface CameraFramePosture {
+  rotation: number
+  mirror: boolean
+  width: number
+  height: number
+}
+
+export interface CameraFrameSource<F extends FrameLike = FrameLike> {
+  image: TgfxImage | null
+  posture: CameraFramePosture
+  onFrame: (frame: F) => void
+}
+
+export interface CameraFrameOptions<F extends FrameLike = FrameLike> {
+  /**
+   * Map the producer's `pixelFormat` to the one tgfx imports with. A worklet —
+   * it runs on the frame thread.
+   *
+   * The default reads the string: `rgb`/`rgba`/`bgra` import as `'rgba'`,
+   * `709`/`2020` pick that matrix, anything else is BT.601, and a `-full`
+   * suffix selects the full-range variant. Override it for a producer that
+   * names its formats differently, or to pin one.
+   */
+  format?: (frame: F) => PixelFormat
+  /**
+   * Map the producer's `orientation` to the clockwise degrees that cancel it.
+   * A worklet. The default understands `'up' | 'right' | 'down' | 'left'`;
+   * anything else is 0, which draws the frame unrotated.
+   */
+  rotation?: (frame: F) => number
+}
+
+const ROTATION: Record<string, 0 | 90 | 180 | 270> = {
+  up: 0,
+  right: 270,
+  down: 180,
+  left: 90,
+}
+
+function defaultFormat(pixelFormat: string): PixelFormat {
+  'worklet'
+  const f = pixelFormat.toLowerCase()
+  if (f.startsWith('rgb') || f.startsWith('bgra') || f.startsWith('argb')) {
+    return 'rgba'
+  }
+  const full = f.endsWith('-full') || f.includes('full-range')
+  if (f.includes('2020')) {
+    return full ? 'bt2020-full' : 'bt2020-limited'
+  }
+  if (f.includes('709')) {
+    return full ? 'bt709-full' : 'bt709-limited'
+  }
+  return full ? 'bt601-full' : 'bt601-limited'
+}
+
+const UPRIGHT: CameraFramePosture = Object.freeze({
+  rotation: 0,
+  mirror: false,
+  width: 0,
+  height: 0,
+})
+
+/**
+ * Turn a stream of camera frames into a `<CameraFrame>` handle.
+ *
+ * ```tsx
+ * const { image, posture, onFrame } = useCameraFrame()
+ * const output = useFrameOutput({ pixelFormat: 'yuv', onFrame })
+ * // ...
+ * <CameraFrame image={image} {...posture} />
+ * ```
+ *
+ * The callback is *returned* rather than registered, so this package needs no
+ * camera dependency at all and any producer of a {@link FrameLike} works.
+ *
+ * It owns the buffer's lifetime, which is the reason to prefer it over calling
+ * {@link imageFromNativeBuffer} yourself: the raw pointer never leaves this
+ * scope, so it cannot be stored, and a `try`/`finally` releases the buffer and
+ * disposes the frame on every path including a throw. A leaked frame stalls a
+ * bounded producer pool; a stored pointer is a use-after-free that does not
+ * throw.
+ *
+ * The first frame creates the handle and every later one re-points it, so the
+ * scene is committed once and a new frame costs one pointer swap — no React
+ * render, no reconciler commit, no program recompile. `posture` only reaches
+ * React state when it actually changes. The handle is disposed on unmount: an
+ * external registry entry is never evicted, so nothing else would ever free it.
+ *
+ * `onEachFrame` is your own per-frame worklet — a detector, say. It runs inside
+ * the same `try`, so work that throws still cannot leak the frame. Memoise it,
+ * or the output re-registers every render.
+ *
+ * @remarks
+ * **The frame `onEachFrame` receives dies when it returns.** Read from it and
+ * let it go: do not retain it, do not call `getNativeBuffer()` on it again —
+ * that is a second `+1` retain the hook will not release, and one unreleased
+ * buffer stalls the producer's bounded pool — and never store
+ * `getNativeBuffer().pointer` anywhere. A stale pointer does not throw; it
+ * imports whatever memory is now at that address. Everything the hook itself
+ * needs from the buffer is already done before your callback runs.
+ *
+ * Needs react-native-reanimated: the handle has to outlive one callback on the
+ * producer's worklet runtime.
+ */
+export function useCameraFrame<F extends FrameLike = FrameLike>(
+  onEachFrame?: (frame: F) => void,
+  options?: CameraFrameOptions<F>
+): CameraFrameSource<F> {
+  const mod = requireReanimated('useCameraFrame', ['useSharedValue'])
+  const scheduleOnRN = worklets().scheduleOnRN
+  if (scheduleOnRN == null) {
+    throw new Error(
+      '[tgfx] react-native-worklets does not export scheduleOnRN, which useCameraFrame needs.'
+    )
+  }
+  // Destructured, or an inline `{}` at the call site would re-register every render.
+  const formatOf = options?.format
+  const rotationOf = options?.rotation
+  const [image, setImage] = useState<TgfxImage | null>(null)
+  const [posture, setPosture] = useState<CameraFramePosture>(UPRIGHT)
+  const handle = mod.useSharedValue!<TgfxImage | null>(null)
+  const posted = mod.useSharedValue!('')
+
+  const onImage = useCallback((h: TgfxImage) => {
+    setImage(h)
+  }, [])
+  const onPosture = useCallback(
+    (rotation: number, mirror: boolean, width: number, height: number) => {
+      setPosture({ rotation, mirror, width, height })
+    },
+    []
+  )
+
+  const onFrame = useCallback(
+    (frame: F) => {
+      'worklet'
+      // Acquired inside the try: a throw would skip the dispose() and stall the bounded pool.
+      let buffer: NativeBufferLike | null = null
+      try {
+        buffer = frame.getNativeBuffer()
+        // Read off the frame: `-full` imported as limited crushes the blacks.
+        const format: PixelFormat =
+          formatOf != null ? formatOf(frame) : defaultFormat(frame.pixelFormat)
+        if (handle.value == null) {
+          handle.value = imageFromNativeBuffer(buffer.pointer, format)
+          scheduleOnRN(onImage, handle.value)
+        } else {
+          handle.value.setNativeBuffer(buffer.pointer, format)
+        }
+
+        const rotation =
+          rotationOf != null
+            ? rotationOf(frame)
+            : (ROTATION[frame.orientation] ?? 0)
+        const key = `${rotation}:${frame.isMirrored}:${frame.width}x${frame.height}`
+        if (posted.value !== key) {
+          posted.value = key
+          scheduleOnRN(
+            onPosture,
+            rotation,
+            frame.isMirrored,
+            frame.width,
+            frame.height
+          )
+        }
+
+        onEachFrame?.(frame)
+      } finally {
+        buffer?.release()
+        frame.dispose()
+      }
+    },
+    [
+      handle,
+      posted,
+      onImage,
+      onPosture,
+      onEachFrame,
+      formatOf,
+      rotationOf,
+      scheduleOnRN,
+    ]
+  )
+
+  useEffect(() => {
+    return () => {
+      // An external entry is never evicted, so an undisposed handle leaks for the process's life.
+      handle.value?.dispose()
+      handle.value = null
+    }
+  }, [handle])
+
+  return { image, posture, onFrame }
+}

package/src/hooks/useClock.test.ts

@@ -0,0 +1,50 @@
+import { expect, mock, test } from 'bun:test'
+import type * as TestRenderer from 'react-test-renderer'
+
+// A renamed export must not look like an absent package, and it throws where the hook runs.
+test('Reanimated present but missing an export throws when the hook runs', async () => {
+  mock.module('react-native-reanimated', () => ({ useSharedValue: () => {} }))
+  const { useClock } =
+    (await import('./useClock?broken')) as typeof import('./useClock')
+  expect(() => useClock()).toThrow(/does not export useFrameCallback/)
+})
+
+test('the clock reads the frame time and keeps one callback identity', async () => {
+  const shared = { value: 0, _isReanimatedSharedValue: true as const }
+  let registered: ((frame: { timeSinceFirstFrame: number }) => void) | null =
+    null
+  const seen: unknown[] = []
+  mock.module('react-native-reanimated', () => ({
+    useSharedValue: () => shared,
+    useFrameCallback: (
+      cb: (frame: { timeSinceFirstFrame: number }) => void
+    ) => {
+      registered = cb
+      seen.push(cb)
+    },
+  }))
+  const { useClock } = await import('./useClock?live')
+  const React = await import('react')
+  const TestRenderer = await import('react-test-renderer')
+
+  let got: unknown
+  function Probe(): null {
+    got = useClock()
+    return null
+  }
+  let tree!: TestRenderer.ReactTestRenderer
+  await TestRenderer.act(async () => {
+    tree = TestRenderer.create(React.createElement(Probe))
+  })
+  expect(got).toBe(shared)
+
+  // Seconds, not milliseconds: a ms clock overflows GLSL ES mediump in a minute.
+  registered!({ timeSinceFirstFrame: 250 })
+  expect(shared.value).toBe(0.25)
+
+  await TestRenderer.act(async () => {
+    tree.update(React.createElement(Probe))
+  })
+  expect(seen).toHaveLength(2)
+  expect(seen[1]).toBe(seen[0])
+})

package/src/hooks/useClock.ts

@@ -0,0 +1,36 @@
+import { useCallback } from 'react'
+import type { MaybeSharedValue } from '../compiler/values'
+import { requireReanimated } from '../reanimated/peer'
+
+interface Frame {
+  timeSinceFirstFrame: number
+}
+
+/**
+ * Seconds since the first frame, as a SharedValue.
+ *
+ * Written on the UI thread, so feeding it to a shader uniform animates without
+ * re-encoding the scene from JS.
+ *
+ * Seconds rather than milliseconds because the value is usually bound for a
+ * shader uniform, and GLSL ES mediump floats stop being representable above
+ * 65504: a millisecond clock turns every shader that reads it black after about
+ * a minute of uptime, silently. Seconds buy eighteen hours instead.
+ */
+export function useClock(): MaybeSharedValue<number> {
+  const mod = requireReanimated('useClock', [
+    'useSharedValue',
+    'useFrameCallback',
+  ])
+  const clock = mod.useSharedValue!(0)
+  // Stable identity: Reanimated keys its effect on the callback, so a fresh one resets the clock.
+  const tick = useCallback(
+    (frame: Frame) => {
+      'worklet'
+      clock.value = frame.timeSinceFirstFrame / 1000
+    },
+    [clock]
+  )
+  mod.useFrameCallback!(tick)
+  return clock
+}