react-native-image-stitcher 0.14.2 → 0.15.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (116) hide show
  1. package/CHANGELOG.md +131 -0
  2. package/README.md +35 -0
  3. package/RNImageStitcher.podspec +8 -7
  4. package/android/build.gradle +0 -16
  5. package/android/src/main/cpp/CMakeLists.txt +2 -63
  6. package/android/src/main/cpp/image_stitcher_jni.cpp +14 -0
  7. package/android/src/main/cpp/keyframe_gate_jni.cpp +13 -0
  8. package/android/src/main/java/io/imagestitcher/rn/BatchStitcher.kt +285 -3
  9. package/android/src/main/java/io/imagestitcher/rn/IncrementalStitcher.kt +180 -1162
  10. package/android/src/main/java/io/imagestitcher/rn/KeyframeGate.kt +29 -0
  11. package/android/src/main/java/io/imagestitcher/rn/RNImageStitcherPackage.kt +0 -4
  12. package/android/src/main/java/io/imagestitcher/rn/RNSARCameraView.kt +13 -64
  13. package/cpp/keyframe_gate.cpp +82 -23
  14. package/cpp/keyframe_gate.hpp +31 -2
  15. package/cpp/stitcher.cpp +208 -28
  16. package/cpp/tests/CMakeLists.txt +18 -12
  17. package/cpp/tests/keyframe_timebudget_test.cpp +65 -0
  18. package/cpp/tests/warp_guard_test.cpp +48 -0
  19. package/cpp/warp_guard.hpp +41 -0
  20. package/dist/camera/Camera.d.ts +31 -16
  21. package/dist/camera/Camera.js +10 -2
  22. package/dist/camera/CaptureStitchStatsToast.d.ts +15 -2
  23. package/dist/camera/CaptureStitchStatsToast.js +27 -7
  24. package/dist/camera/PanoramaSettings.d.ts +10 -223
  25. package/dist/camera/PanoramaSettings.js +6 -28
  26. package/dist/camera/PanoramaSettingsBridge.d.ts +1 -24
  27. package/dist/camera/PanoramaSettingsBridge.js +3 -102
  28. package/dist/camera/PanoramaSettingsModal.js +7 -1
  29. package/dist/camera/buildPanoramaInitialSettings.d.ts +11 -0
  30. package/dist/camera/buildPanoramaInitialSettings.js +4 -0
  31. package/dist/camera/cameraErrorMessages.d.ts +32 -0
  32. package/dist/camera/cameraErrorMessages.js +53 -0
  33. package/dist/camera/selectCaptureDevice.d.ts +5 -1
  34. package/dist/camera/selectCaptureDevice.js +22 -2
  35. package/dist/camera/useCapture.js +38 -0
  36. package/dist/index.d.ts +5 -8
  37. package/dist/index.js +11 -34
  38. package/dist/stitching/incremental.d.ts +1 -117
  39. package/dist/stitching/stitchVideo.d.ts +0 -35
  40. package/dist/types.d.ts +0 -87
  41. package/ios/Sources/RNImageStitcher/IncrementalStitcher.swift +96 -674
  42. package/ios/Sources/RNImageStitcher/IncrementalStitcherBridge.swift +9 -12
  43. package/ios/Sources/RNImageStitcher/KeyframeGate.swift +14 -0
  44. package/ios/Sources/RNImageStitcher/KeyframeGateBridge.h +7 -0
  45. package/ios/Sources/RNImageStitcher/KeyframeGateBridge.mm +6 -0
  46. package/ios/Sources/RNImageStitcher/OpenCVKeyframeCollector.h +2 -2
  47. package/ios/Sources/RNImageStitcher/OpenCVKeyframeCollector.mm +3 -3
  48. package/ios/Sources/RNImageStitcher/OpenCVStitcher.h +28 -60
  49. package/ios/Sources/RNImageStitcher/OpenCVStitcher.mm +180 -921
  50. package/ios/Sources/RNImageStitcher/RNSARSession.swift +10 -35
  51. package/ios/Sources/RNImageStitcher/Stitcher.swift +84 -35
  52. package/ios/Sources/RNImageStitcher/StitcherBridge.m +13 -0
  53. package/ios/Sources/RNImageStitcher/StitcherBridge.swift +132 -5
  54. package/package.json +3 -2
  55. package/src/camera/Camera.tsx +43 -22
  56. package/src/camera/CaptureStitchStatsToast.tsx +58 -14
  57. package/src/camera/PanoramaSettings.ts +16 -289
  58. package/src/camera/PanoramaSettingsBridge.ts +3 -114
  59. package/src/camera/PanoramaSettingsModal.tsx +14 -1
  60. package/src/camera/__tests__/PanoramaSettingsBridge.test.ts +3 -188
  61. package/src/camera/__tests__/buildPanoramaInitialSettings.test.ts +41 -0
  62. package/src/camera/__tests__/cameraErrorMessages.test.ts +76 -0
  63. package/src/camera/__tests__/selectCaptureDevice.test.ts +33 -0
  64. package/src/camera/buildPanoramaInitialSettings.ts +17 -0
  65. package/src/camera/cameraErrorMessages.ts +84 -0
  66. package/src/camera/selectCaptureDevice.ts +28 -3
  67. package/src/camera/useCapture.ts +44 -1
  68. package/src/index.ts +11 -40
  69. package/src/stitching/incremental.ts +3 -140
  70. package/src/stitching/stitchVideo.ts +0 -26
  71. package/src/types.ts +0 -95
  72. package/android/src/main/cpp/stitcher_jsi_install_jni.cpp +0 -227
  73. package/android/src/main/java/io/imagestitcher/rn/IncrementalFirstwinsEngine.kt +0 -1081
  74. package/android/src/main/java/io/imagestitcher/rn/StitcherJsiInstallerModule.kt +0 -103
  75. package/android/src/main/java/io/imagestitcher/rn/StitcherWorkletRuntime.kt +0 -256
  76. package/cpp/stitcher_frame_jsi.cpp +0 -214
  77. package/cpp/stitcher_frame_jsi.hpp +0 -108
  78. package/cpp/stitcher_proxy_jsi.cpp +0 -109
  79. package/cpp/stitcher_proxy_jsi.hpp +0 -46
  80. package/cpp/stitcher_worklet_dispatch.cpp +0 -103
  81. package/cpp/stitcher_worklet_dispatch.hpp +0 -71
  82. package/cpp/stitcher_worklet_registry.cpp +0 -91
  83. package/cpp/stitcher_worklet_registry.hpp +0 -146
  84. package/cpp/tests/stitcher_worklet_registry_test.cpp +0 -195
  85. package/dist/stitching/IncrementalStitcherView.d.ts +0 -41
  86. package/dist/stitching/IncrementalStitcherView.js +0 -157
  87. package/dist/stitching/StitcherWorkletRegistry.d.ts +0 -117
  88. package/dist/stitching/StitcherWorkletRegistry.js +0 -78
  89. package/dist/stitching/ensureStitcherProxyInstalled.d.ts +0 -8
  90. package/dist/stitching/ensureStitcherProxyInstalled.js +0 -81
  91. package/dist/stitching/useFrameProcessor.d.ts +0 -119
  92. package/dist/stitching/useFrameProcessor.js +0 -196
  93. package/dist/stitching/useFrameStream.d.ts +0 -34
  94. package/dist/stitching/useFrameStream.js +0 -234
  95. package/dist/stitching/useThrottledFrameProcessor.d.ts +0 -33
  96. package/dist/stitching/useThrottledFrameProcessor.js +0 -132
  97. package/ios/Sources/RNImageStitcher/OpenCVIncrementalStitcher.h +0 -474
  98. package/ios/Sources/RNImageStitcher/OpenCVIncrementalStitcher.mm +0 -1328
  99. package/ios/Sources/RNImageStitcher/OpenCVSlitScanStitcher.h +0 -103
  100. package/ios/Sources/RNImageStitcher/OpenCVSlitScanStitcher.mm +0 -3285
  101. package/ios/Sources/RNImageStitcher/RNSARWorkletRuntime.h +0 -128
  102. package/ios/Sources/RNImageStitcher/RNSARWorkletRuntime.mm +0 -313
  103. package/ios/Sources/RNImageStitcher/SaveFrameAsJpegPlugin.mm +0 -185
  104. package/ios/Sources/RNImageStitcher/StitcherFrameHostObject.h +0 -60
  105. package/ios/Sources/RNImageStitcher/StitcherFrameHostObject.mm +0 -214
  106. package/ios/Sources/RNImageStitcher/StitcherJsiInstaller.h +0 -42
  107. package/ios/Sources/RNImageStitcher/StitcherJsiInstaller.mm +0 -160
  108. package/src/stitching/IncrementalStitcherView.tsx +0 -198
  109. package/src/stitching/StitcherWorkletRegistry.ts +0 -156
  110. package/src/stitching/__tests__/StitcherWorkletRegistry.test.ts +0 -176
  111. package/src/stitching/__tests__/ensureStitcherProxyInstalled.test.ts +0 -94
  112. package/src/stitching/__tests__/useThrottledFrameProcessor.test.ts +0 -178
  113. package/src/stitching/ensureStitcherProxyInstalled.ts +0 -141
  114. package/src/stitching/useFrameProcessor.ts +0 -226
  115. package/src/stitching/useFrameStream.ts +0 -271
  116. package/src/stitching/useThrottledFrameProcessor.ts +0 -145
package/src/index.ts CHANGED
@@ -39,6 +39,12 @@ export type {
39
39
  FramesDroppedInfo,
40
40
  } from './camera/Camera';
41
41
 
42
+ // Recoverable-stitch-failure → friendly Alert copy. Hosts call this in
43
+ // their onError handler to surface actionable guidance ("pan more slowly",
44
+ // "pivot in place") instead of the raw cv::Stitcher diagnostic.
45
+ export { userFacingStitchError } from './camera/cameraErrorMessages';
46
+ export type { UserFacingStitchError } from './camera/cameraErrorMessages';
47
+
42
48
  // ─────────────────────────────────────────────────────────────────────
43
49
  // AR foundation (public since 0.1.0)
44
50
  // ─────────────────────────────────────────────────────────────────────
@@ -128,8 +134,6 @@ export type { PanoramaSettingsModalProps } from './camera/PanoramaSettingsModal'
128
134
  export {
129
135
  DEFAULT_PANORAMA_SETTINGS,
130
136
  DEFAULT_FLOW_GATE_SETTINGS,
131
- DEFAULT_SLITSCAN_SETTINGS,
132
- DEFAULT_HYBRID_SETTINGS,
133
137
  } from './camera/PanoramaSettings';
134
138
  export type {
135
139
  CaptureBaseSettings,
@@ -137,14 +141,6 @@ export type {
137
141
  BatchStitcherSettings,
138
142
  FrameSelectionSettings,
139
143
  FlowGateSettings,
140
- SlitscanSettings,
141
- SlitscanPaintingSettings,
142
- SlitscanRegistrationSettings,
143
- SlitscanAdvancedSettings,
144
- Ncc1dSettings,
145
- Ncc2dSettings,
146
- PlaneProjectionSettings,
147
- HybridSettings,
148
144
  } from './camera/PanoramaSettings';
149
145
 
150
146
  // Settings → native config adapters. Layer 2 hosts building their
@@ -153,8 +149,6 @@ export type {
153
149
  // the single source of truth for the JS↔native wire format.
154
150
  export {
155
151
  panoramaSettingsToNativeConfig,
156
- slitscanSettingsToNativeConfig,
157
- hybridSettingsToNativeConfig,
158
152
  } from './camera/PanoramaSettingsBridge';
159
153
  export type { NativeConfigDict } from './camera/PanoramaSettingsBridge';
160
154
  export { ViewportCropOverlay } from './camera/ViewportCropOverlay';
@@ -206,34 +200,11 @@ export type {
206
200
  StitcherFrameProcessor,
207
201
  ARAnchor,
208
202
  } from './stitching/StitcherFrame';
209
- // v0.8.0 Phase 4a — public host-worklet hook. Hosts that want a
210
- // per-frame callback (OCR overlay, packet detection, ML inference)
211
- // use this to attach a `'worklet'`-prefixed function that fires
212
- // on the camera producer thread. Non-AR mode is fully wired
213
- // today via vision-camera passthrough; AR-mode dispatch is
214
- // API-stable but registration-only until Phase 4b lands the
215
- // cross-runtime handoff (the AR runtime iterating the registry).
216
- // See the hook's docstring + StitcherFrame.ts for the contract.
217
- export { useFrameProcessor } from './stitching/useFrameProcessor';
218
- // v0.9.0 Layer 2 — `useThrottledFrameProcessor`. Throttle gate over
219
- // `useFrameProcessor` for sub-frame-rate worklet-native processing
220
- // (native OCR via Vision.framework / ML Kit, TFLite ML detection,
221
- // LiDAR depth). The worklet runtime has direct access to
222
- // `frame.toArrayBuffer()` / `frame.arDepth`; bridge small payloads
223
- // (bboxes, depth-derived metrics) to JS via `runOnJS`. For JS-thread
224
- // JPEG consumers (file-path OCR libs, cloud upload, thumbnail UI),
225
- // prefer `useFrameStream` (Layer 3, ships in the same release).
226
- export { useThrottledFrameProcessor } from './stitching/useThrottledFrameProcessor';
227
- export type { ThrottledFrameProcessorOptions } from './types';
228
- // v0.9.0 Layer 3 — `useFrameStream`. JS-thread sampled-frame
229
- // stream over Layer 1 (`save_frame_as_jpeg` vc plugin) + Layer 2
230
- // (`useThrottledFrameProcessor`). Use for JS-thread consumers:
231
- // file-path OCR libs (RN modules), cloud upload, thumbnail UI.
232
- // For worklet-native processing (Vision/ML Kit as vc plugins,
233
- // TFLite ML, LiDAR depth), prefer `useThrottledFrameProcessor`
234
- // (Layer 2) — lower latency, no JPEG roundtrip.
235
- export { useFrameStream } from './stitching/useFrameStream';
236
- export type { FrameStreamOptions, SampledFrame } from './types';
203
+ // NOTE: the host-worklet / frame-stream hooks `useFrameProcessor`,
204
+ // `useThrottledFrameProcessor` and `useFrameStream` (v0.8–v0.9) were
205
+ // archived in the batch-keyframe cleanup they drove the third-party
206
+ // `__stitcherProxy` observer API, not batch-keyframe capture. Source is
207
+ // preserved under archive/src/stitching/ to build on later.
237
208
  // vision-camera Frame Processor driver for non-AR captures. As
238
209
  // of v0.6 the only non-AR driver exported (the legacy
239
210
  // `useIncrementalJSDriver` was removed; was deprecated in v0.5).
@@ -422,9 +422,9 @@ export interface IncrementalStartOptions {
422
422
  * 'slitscan' fall back to 'slitscan-both' with a deprecation warning
423
423
  * in the native log.
424
424
  */
425
- engine?: 'hybrid' | 'slitscan-rotate' | 'slitscan-both' | 'batch-keyframe' |
426
- // Deprecated — kept for type-compat during the V14 → V15 transition:
427
- 'firstwins' | 'firstwins-zoomed' | 'firstwins-rectilinear' | 'slitscan';
425
+ // Only 'batch-keyframe' remains; the live engines were archived in the
426
+ // batch-keyframe cleanup (see archive/).
427
+ engine?: 'batch-keyframe';
428
428
  /**
429
429
  * V15 — per-stage correction config overrides. Mode-driven defaults
430
430
  * are applied first (see RLISStitcherConfig +configForMode:); fields
@@ -441,143 +441,6 @@ export interface IncrementalStartOptions {
441
441
  * fields optional (omit to accept the engine-mode default).
442
442
  */
443
443
  export interface StitcherConfig {
444
- // Slit shaping (slit-scan engine only)
445
- /** Fraction of pan-axis the rectilinear slit retains per frame.
446
- * Range 0.10 – 0.70, default 0.30 in V15 slit-scan modes. */
447
- kPanAxisFractionRect: number;
448
- /** Minimum pan-axis advance (px) before a frame is accepted.
449
- * 0 = accept on every consumeFrame (Apple-dense slit-scan, V15
450
- * default). 50 = V13.0g default. */
451
- kMinAcceptDeltaPx: number;
452
-
453
- // Per-stage correction toggles
454
- /** V13.0e+ ORB triangulation + median-Z parallax correction. */
455
- enableTriangulation: boolean;
456
- /** V13.0g per-accept incremental Δt accumulator on top of triangulation. */
457
- enableTriAccumulator: boolean;
458
- /** V15 1D NCC perpendicular-axis wobble correction (slitscan-rotate
459
- * default). Independent of the other correction stages. */
460
- enable1dNcc: boolean;
461
- /** 1D NCC search radius in pixels (5 – 60). */
462
- nccSearchRadius1d: number;
463
- /** V13.0g 2D NCC fine-alignment after triangulation. */
464
- enable2dNcc: boolean;
465
- /** V14.0a RANSAC homography per slit + cv::warpPerspective. When
466
- * enabled and successful, supersedes the rectangular paste path. */
467
- enableRansacHomography: boolean;
468
-
469
- // Paint mode (slit-scan engine only)
470
- /** 'FirstPaintedWins' protects already-painted pixels (V13.0e+
471
- * default). 'FeatherBlend' alpha-blends new content into already-
472
- * painted overlap pixels (V13.0d-style; V15 slitscan-both default). */
473
- paintMode: 'FirstPaintedWins' | 'FeatherBlend';
474
-
475
- // Hybrid engine
476
- /** 'Cylindrical' (V12.x – V14.0a behaviour) or 'Planar' (V15 default;
477
- * cv::detail::PlaneWarper). Planar is well-behaved for pans <60°. */
478
- hybridProjection: 'Cylindrical' | 'Planar';
479
-
480
- /** V15.0c — where on the camera frame the per-accept sliver is taken.
481
- * 'Center' (V13.x default), 'Bottom' (leading edge for top-to-bottom
482
- * pan), or 'Top' (leading edge for bottom-to-top pan). */
483
- sliverPosition: 'Center' | 'Bottom' | 'Top';
484
-
485
- /** V15.0c — when true, the FIRST accepted frame paints the entire
486
- * camera frame at canvas (0, 0); subsequent frames still use the
487
- * configured sliver clip. Default false; set true when sliverPosition
488
- * is Bottom/Top so the canvas is anchored with full-frame content. */
489
- firstFrameFullFrame: boolean;
490
-
491
- /** **DEPRECATED in V15.0d** — use `planeSource` instead.
492
- *
493
- * V15.0b boolean toggle for the plane-projected stitch path.
494
- * Kept for backward compat: when `planeSource` is left at its
495
- * default (Disabled), `useDetectedPlane = true` upgrades it to
496
- * ARKitDetected. New callers should set `planeSource` directly. */
497
- useDetectedPlane: boolean;
498
-
499
- /** V15.0d — source of the plane used by the V15.0b plane-projected
500
- * stitch path.
501
- *
502
- * - 'Disabled' (default): no plane projection; slit-scan path runs.
503
- * - 'ARKitDetected': use ARKit's first vertical plane that aligns
504
- * with the camera's view direction (filter threshold:
505
- * `arkitPlaneAlignmentThreshold`). Falls back to slit-scan
506
- * silently when no aligned plane is found.
507
- * - 'Virtual': synthesize a plane at first frame: origin =
508
- * camera_pos + `virtualPlaneDepthMeters` × camera_forward;
509
- * normal = -camera_forward. Always works; no ARKit dependency.
510
- *
511
- * Field testing showed ARKit plane detection often picks the WRONG
512
- * surface (side wall, doorframe) — Virtual mode is the safer
513
- * default for arbitrary scenes. ARKitDetected wins when ARKit
514
- * finds the correct fixture face. */
515
- planeSource: 'Disabled' | 'ARKitDetected' | 'Virtual';
516
-
517
- /** V15.0d — depth (metres) at which the synthetic plane is placed
518
- * in front of the camera when `planeSource = Virtual`. Set to
519
- * the user's typical scan distance. Range 0.3 – 5.0 m. Default
520
- * 1.5 m. */
521
- virtualPlaneDepthMeters: number;
522
-
523
- /** V15.0d — minimum dot product between an ARKit-detected plane's
524
- * surface normal and the camera's facing direction for the plane
525
- * to be accepted (when `planeSource = ARKitDetected`). 1.0 =
526
- * plane perfectly facing camera; 0.0 = plane edge-on; negative
527
- * = facing away. Range 0.0 – 1.0. Default 0.6 (≈53° max angle
528
- * off-camera). */
529
- arkitPlaneAlignmentThreshold: number;
530
-
531
- /** V15.0g — how the plane-projection helper renders each frame onto
532
- * the canvas. Affects ARKitDetected and Virtual modes; ignored
533
- * when planeSource = Disabled.
534
- *
535
- * - 'Trapezoidal' (V15.0b legacy): geometrically-correct 3D
536
- * raycast. Each camera pixel maps to its plane intersection.
537
- * Result is a trapezoid that grows distorted with tilt
538
- * (cooler-bottom-2.3×-wider-than-top problem).
539
- * - 'Rectified' (V15.0g default): camera frame pasted as a clean
540
- * rectangle around its plane-projected anchor. Eliminates the
541
- * tilt-induced trapezoidal distortion at the cost of strict 3D-
542
- * correctness — the camera's per-pixel perspective stays inside
543
- * the rectangle but doesn't reconcile across tilts. */
544
- planeProjectionStyle: 'Trapezoidal' | 'Rectified';
545
-
546
- /** V15.0d — 2D NCC search half-window in pixels. Was hardcoded
547
- * ±12 in V15.0c.4. Smaller = less wandering on repetitive
548
- * textures (peg holes, slatted panels), but easier to miss the
549
- * true overlap when pose noise is high. Range 4 – 30. Default
550
- * 12. */
551
- nccSearchMargin2d: number;
552
-
553
- /** V15.0d — 2D NCC confidence threshold below which the correction
554
- * is rejected. Was hardcoded 0.75 in V15.0c.4. Higher = stricter,
555
- * fewer false matches on repetitive textures, but more frames
556
- * where NCC silently doesn't fire. Range 0.30 – 0.99. Default
557
- * 0.75. */
558
- nccConfidenceThreshold2d: number;
559
-
560
- /** V15.0d (1B) — exponential-moving-average smoothing on 2D NCC
561
- * corrections. When enabled, the applied correction is
562
- * `α × current + (1−α) × prev` instead of just `current`. Damps
563
- * single-frame snaps to spurious peaks. Default false. */
564
- enableNcc2dEmaSmoothing: boolean;
565
-
566
- /** V15.0d — EMA weight on the CURRENT-frame NCC correction
567
- * (1 − α weight on the previous correction). Range 0.05 – 0.95.
568
- * Default 0.4 (60% prev / 40% current — heavy damping). */
569
- ncc2dEmaAlpha: number;
570
-
571
- /** V15.0d (1C) — pan-axis-aware 2D NCC. When enabled, the cross-
572
- * axis (perpendicular to pan) NCC correction is clamped tighter
573
- * than the pan-axis (since 1D NCC + pose already handle cross-
574
- * axis wobble). Default false. */
575
- enableNcc2dPanAxisLock: boolean;
576
-
577
- /** V15.0d — cross-axis clamp (pixels) for the pan-axis-aware mode.
578
- * Range 0 – 30. Default 5. */
579
- ncc2dCrossAxisLockPx: number;
580
-
581
444
  // Frame selection (V16)
582
445
 
583
446
  /** V16 — how the engine decides which ARFrames to ingest.
@@ -76,32 +76,6 @@ export interface StitchVideoOptions {
76
76
  * Right choice on low-RAM devices or with `feather`.
77
77
  */
78
78
  seamFinderType?: 'graphcut' | 'skip';
79
- /**
80
- * Phase 5: pose-driven stitching. When present and non-empty,
81
- * the native stitcher skips features → matching → BundleAdjuster
82
- * and builds cv::detail::CameraParams directly from each pose's
83
- * intrinsics + quaternion. Each entry has the shape returned
84
- * by `NativeModules.RNSARSession.snapshotPoseLog()`:
85
- *
86
- * { tx, ty, tz, qx, qy, qz, qw,
87
- * fx, fy, cx, cy,
88
- * imageWidth, imageHeight,
89
- * timestampMs, trackingState }
90
- *
91
- * Frames whose closest pose is beyond a 100 ms tolerance are
92
- * dropped before stitching; if fewer than 2 remain the call
93
- * rejects with `opencv-failed-1032` so the host can fall back
94
- * to the feature-matched path (re-call `stitchVideo` without
95
- * `poses`).
96
- */
97
- poses?: Array<{
98
- tx: number; ty: number; tz: number;
99
- qx: number; qy: number; qz: number; qw: number;
100
- fx: number; fy: number; cx: number; cy: number;
101
- imageWidth: number; imageHeight: number;
102
- timestampMs: number;
103
- trackingState: number;
104
- }>;
105
79
  }
106
80
 
107
81
 
package/src/types.ts CHANGED
@@ -56,101 +56,6 @@ export interface DeviceMetadata {
56
56
  // `Camera.tsx` adapts this into the public `CameraCaptureResult` (a
57
57
  // discriminated union of photo + panorama) before emitting `onCapture`.
58
58
 
59
- /**
60
- * v0.9.0 Layer 3 — one sampled frame delivered by `useFrameStream`
61
- * to the JS-thread handler.
62
- *
63
- * The JPEG file at `jpegPath` is the stream's own copy. Hosts that
64
- * need long-term retention MUST copy the file synchronously inside
65
- * the handler — the same path may be overwritten by a subsequent
66
- * sample (slot reuse — see the hook's docstring for the rotation
67
- * policy).
68
- */
69
- export interface SampledFrame {
70
- /** Absolute filesystem path to the JPEG. No `file://` prefix. */
71
- jpegPath: string;
72
-
73
- /**
74
- * Pose at sample time. `translation` is `undefined` in non-AR
75
- * mode (gyro provides rotation only; no spatial anchor).
76
- */
77
- pose: {
78
- rotation: [number, number, number, number];
79
- translation?: [number, number, number];
80
- };
81
-
82
- /** Frame timestamp (ms; per the v0.8.0 StitcherFrame contract). */
83
- timestamp: number;
84
-
85
- /** JPEG width / height in pixels. */
86
- width: number;
87
- height: number;
88
- }
89
-
90
- /**
91
- * v0.9.0 Layer 3 — options for `useFrameStream`.
92
- *
93
- * For worklet-native processing without JPEG roundtrip (OCR via
94
- * Vision/ML Kit, TFLite ML, LiDAR depth), use
95
- * `useThrottledFrameProcessor` (Layer 2) instead.
96
- */
97
- export interface FrameStreamOptions {
98
- /**
99
- * Target sampling rate in Hertz. Clamped to `[0.5, 10]`. The
100
- * Layer 2 throttle gate enforces the rate inside the worklet;
101
- * ticks too close together are dropped silently.
102
- *
103
- * Clamp upper bound (10 Hz) is intentionally lower than Layer 2's
104
- * (30 Hz) — beyond 10 Hz the per-frame JPEG encode + JS-bridge
105
- * cost dominates the wall-clock budget. Hosts that need higher
106
- * rates should be on Layer 2 with their own JPEG encoder call
107
- * (or no JPEG at all).
108
- */
109
- sampleHz: number;
110
-
111
- /**
112
- * JPEG quality (0-100). Default 75. Clamped silently to
113
- * `[1, 100]` by the underlying `save_frame_as_jpeg` native plugin.
114
- */
115
- quality?: number;
116
-
117
- /**
118
- * Directory to write JPEG files into. Defaults to a per-app
119
- * `<cache>/rnis-frame-stream/` subdirectory. The directory is
120
- * `mkdir -p`'d on first use; hosts that supply an existing
121
- * absolute path are responsible for its lifecycle.
122
- */
123
- outputDir?: string;
124
- }
125
-
126
- /**
127
- * v0.9.0 Layer 2 — options for `useThrottledFrameProcessor`.
128
- *
129
- * Wraps v0.8.0's `useFrameProcessor` with a monotonic-time throttle
130
- * gate so the supplied worklet fires at most `sampleHz` times per
131
- * second. Use for sub-frame-rate worklet-native processing — native
132
- * OCR (Vision.framework / ML Kit), TFLite ML detection, LiDAR depth
133
- * processing — where the bbox / depth payloads are small enough to
134
- * bridge to JS via `runOnJS`.
135
- *
136
- * For JS-thread JPEG consumers (file-path OCR libraries, cloud
137
- * upload, thumbnail UI), use `useFrameStream` (Layer 3) instead.
138
- */
139
- export interface ThrottledFrameProcessorOptions {
140
- /**
141
- * Target sampling rate in Hertz. Clamped to `[0.5, 30]`. Inside
142
- * the worklet a monotonic-time gate enforces the rate; ticks too
143
- * close together are silently dropped.
144
- *
145
- * The clamp upper bound (30 Hz) sits at typical AR rates on
146
- * mid-range Android devices — beyond that, the host should just
147
- * use `useFrameProcessor` directly (no throttle). The clamp
148
- * lower bound (0.5 Hz) prevents accidentally-zero-divide values
149
- * + matches `useFrameStream`'s convention.
150
- */
151
- sampleHz: number;
152
- }
153
-
154
59
  export interface CaptureResult {
155
60
  /** Unique device-generated UUID */
156
61
  deviceUuid: string;
@@ -1,227 +0,0 @@
1
- // SPDX-License-Identifier: Apache-2.0
2
- //
3
- // stitcher_jsi_install_jni.cpp — JNI binding for the Android-side
4
- // JSI install (v0.8.0 Phase 4b.ii).
5
- //
6
- // Kotlin's `StitcherJsiInstallerModule.nativeInstall(jsiRuntimeRef)`
7
- // calls into this file. We unbox the `jsi::Runtime*` from the
8
- // Java `long` and hand it to the shared
9
- // `retailens::installStitcherProxy(runtime)` function which sets
10
- // `globalThis.__stitcherProxy`. Same destination as iOS — the
11
- // host object class lives in `cpp/stitcher_proxy_jsi.{hpp,cpp}`.
12
- //
13
- // ## Why a `long` ref, not a JSI handle wrapper class
14
- //
15
- // `ReactApplicationContext.getJavaScriptContextHolder()` returns a
16
- // `JavaScriptContextHolder` whose `.get()` returns a Java `long`
17
- // that's the raw pointer to the C++ `jsi::Runtime*`. Same
18
- // contract as worklets-core's `WorkletsModule.nativeInstall`
19
- // (verified at the same call site). Caller is responsible for
20
- // ensuring the runtime outlives this call — in practice, the
21
- // runtime IS the JS thread's runtime which lives the whole
22
- // process lifetime, so this is structurally always safe in our
23
- // usage.
24
- //
25
- // ## Threading
26
- //
27
- // Kotlin invokes this from a `@ReactMethod(isBlockingSynchronousMethod
28
- // = true)` so we're already on the JS thread. Synchronous JSI
29
- // access is safe.
30
-
31
- #include "stitcher_proxy_jsi.hpp"
32
-
33
- // v0.8.0 Phase 4b.iii — per-frame fan-out support. The shared
34
- // `dispatchToHostWorklets` posts to worklets-core's default context;
35
- // this JNI file's `nativeDispatchToHostWorklets` constructs the
36
- // `StitcherFrameData` from raw bytes + pose + dims and forwards it.
37
- #include "stitcher_frame_data.hpp"
38
- #include "stitcher_worklet_dispatch.hpp"
39
- #include "stitcher_worklet_registry.hpp"
40
-
41
- #include <react-native-worklets-core/WKTJsiWorkletContext.h>
42
-
43
- #include <jni.h>
44
- #include <jsi/jsi.h>
45
-
46
- #include <android/log.h>
47
-
48
- #include <cstdint>
49
- #include <cstring>
50
- #include <memory>
51
- #include <utility>
52
- #include <vector>
53
-
54
- #define LOG_TAG "StitcherJsiInstaller"
55
- #define LOGI(...) __android_log_print(ANDROID_LOG_INFO, LOG_TAG, __VA_ARGS__)
56
- #define LOGE(...) __android_log_print(ANDROID_LOG_ERROR, LOG_TAG, __VA_ARGS__)
57
-
58
- extern "C" JNIEXPORT jboolean JNICALL
59
- Java_io_imagestitcher_rn_StitcherJsiInstallerModule_nativeInstall(
60
- JNIEnv* /*env*/, jobject /*thiz*/, jlong jsiRuntimeRef) {
61
- if (jsiRuntimeRef == 0) {
62
- // ReactApplicationContext.getJavaScriptContextHolder().get()
63
- // returns 0 when the runtime isn't ready (rare — JS would have
64
- // had to call us before its own runtime was up; impossible in
65
- // practice). Defensive.
66
- return JNI_FALSE;
67
- }
68
- auto* runtime = reinterpret_cast<facebook::jsi::Runtime*>(jsiRuntimeRef);
69
- retailens::installStitcherProxy(*runtime);
70
- LOGI("installed globalThis.__stitcherProxy on main JS runtime.");
71
- return JNI_TRUE;
72
- }
73
-
74
- // ─── v0.8.0 Phase 4b.iii — Android NV21 PixelBufferReader ──────────
75
- //
76
- // Owns a heap-allocated `std::vector<uint8_t>` of pre-copied NV21
77
- // bytes. Constructed by `nativeDispatchToHostWorklets` after one
78
- // JNI byte-array copy from Kotlin; outlives the AR render thread
79
- // scope via `StitcherFrameData::pixelReader`'s `shared_ptr` —
80
- // dropped when the host object is invalidated.
81
-
82
- namespace {
83
-
84
- class AndroidNV21BufferReader : public retailens::PixelBufferReader {
85
- public:
86
- explicit AndroidNV21BufferReader(std::vector<uint8_t>&& bytes)
87
- : _bytes(std::move(bytes)) {}
88
-
89
- std::size_t byteSize() const override { return _bytes.size(); }
90
-
91
- std::size_t copyTo(uint8_t* dst, std::size_t maxBytes) override {
92
- if (dst == nullptr) return 0;
93
- std::size_t n = std::min(maxBytes, _bytes.size());
94
- if (n > 0) {
95
- std::memcpy(dst, _bytes.data(), n);
96
- }
97
- return n;
98
- }
99
-
100
- private:
101
- std::vector<uint8_t> _bytes;
102
- };
103
-
104
- } // namespace
105
-
106
- // ─── v0.8.0 Phase 4b.iii — registry count accessor ─────────────────
107
- //
108
- // Cheap (microsecond) accessor for the per-frame gate in
109
- // `RNSARCameraView.onDrawFrame`. Avoids the NV21 byte-pack cost
110
- // when no host worklets are registered AND no capture is active.
111
- // Same atomic-read the JSI host object's `count()` host function
112
- // goes through.
113
- extern "C" JNIEXPORT jint JNICALL
114
- Java_io_imagestitcher_rn_StitcherWorkletRuntime_nativeRegistryCount(
115
- JNIEnv* /*env*/, jobject /*thiz*/) {
116
- return static_cast<jint>(
117
- retailens::StitcherWorkletRegistry::shared().count());
118
- }
119
-
120
- // ─── v0.8.0 Phase 4b.iii — per-frame dispatch JNI binding ──────────
121
- //
122
- // Called from Kotlin's `StitcherWorkletRuntime.dispatchToHostWorklets`
123
- // after the first-party stitching block has returned (the AR-frame
124
- // data is still in scope on the Kotlin side because
125
- // `RNSARCameraView.onDrawFrame` reads the ARCore Frame, builds the
126
- // NV21 byte[], invokes first-party via `runFirstParty { ... }`,
127
- // THEN calls into here).
128
- //
129
- // The byte[] is COPIED into our owned vector — ARCore's pixel data
130
- // becomes inaccessible shortly after `onDrawFrame` returns, and our
131
- // async dispatch must outlive that scope. Cost: one ~3MB memcpy
132
- // per frame at 1080p NV21 (~90 MB/s at 30 fps; <5 ms on a mid-range
133
- // Android device). Fast-path early-exit when the registry is empty
134
- // skips the copy entirely.
135
- //
136
- // trackingState: Kotlin passes one of "" / "notAvailable" / "limited"
137
- // / "normal" (empty string = field unset → JS sees undefined).
138
- extern "C" JNIEXPORT void JNICALL
139
- Java_io_imagestitcher_rn_StitcherWorkletRuntime_nativeDispatchToHostWorklets(
140
- JNIEnv* env, jobject /*thiz*/,
141
- jbyteArray nv21Bytes,
142
- jint width, jint height,
143
- jdouble qx, jdouble qy, jdouble qz, jdouble qw,
144
- jdouble tx, jdouble ty, jdouble tz,
145
- jdouble timestampNs,
146
- jstring trackingState) {
147
- // Fast-path early-exit BEFORE the JNI byte-array copy. Saves the
148
- // ~3MB memcpy + JSI host object alloc on every frame in the
149
- // common first-party-only case.
150
- if (retailens::StitcherWorkletRegistry::shared().count() == 0) {
151
- return;
152
- }
153
-
154
- if (nv21Bytes == nullptr) {
155
- LOGE("nativeDispatchToHostWorklets: nv21Bytes is null");
156
- return;
157
- }
158
-
159
- const jsize byteLen = env->GetArrayLength(nv21Bytes);
160
- if (byteLen <= 0) {
161
- LOGE("nativeDispatchToHostWorklets: nv21Bytes is empty");
162
- return;
163
- }
164
-
165
- // Copy into our owned vector. `GetByteArrayRegion` is the
166
- // canonical "copy" path — `GetByteArrayElements + Release` MAY
167
- // pin the JVM array (zero-copy) but the contract isn't
168
- // guaranteed; we need our own buffer for the async dispatch
169
- // anyway, so the explicit copy is cleaner.
170
- std::vector<uint8_t> bytes(static_cast<std::size_t>(byteLen));
171
- env->GetByteArrayRegion(
172
- nv21Bytes, 0, byteLen,
173
- reinterpret_cast<jbyte*>(bytes.data()));
174
-
175
- // Extract trackingState string (may be null on the Kotlin side
176
- // for non-AR or pre-tracking frames — guard accordingly).
177
- std::string trackingStateStr;
178
- if (trackingState != nullptr) {
179
- const char* cs = env->GetStringUTFChars(trackingState, nullptr);
180
- if (cs != nullptr) {
181
- trackingStateStr = cs;
182
- env->ReleaseStringUTFChars(trackingState, cs);
183
- }
184
- }
185
-
186
- // Build StitcherFrameData. Field semantics match the iOS
187
- // `StitcherFrameHostObject::fromARFrame:pose:` factory; this is
188
- // the Android equivalent path.
189
- retailens::StitcherFrameData data;
190
- data.source = "ar";
191
- data.width = static_cast<int32_t>(width);
192
- data.height = static_cast<int32_t>(height);
193
- // ARCore's camera image is YUV_420_888 on Android, mapped to NV21
194
- // by the existing `YuvImageConverter.packNV21` path — the byte[]
195
- // we receive is interleaved Y then VU. Worklets gate on this
196
- // string identifier (`'yuv'` vs `'unknown'`); v0.8.0 always
197
- // emits `'yuv'` for AR mode on Android (NV21).
198
- data.pixelFormat = "yuv";
199
- // Android AR-mode camera image is always landscape-natural; the
200
- // mapping matches iOS' coarse two-value set. Hosts that need
201
- // exact display orientation read it from the device-orientation
202
- // sensors (see `useDeviceOrientation` hook).
203
- data.orientation = (width >= height) ? "landscape-right" : "portrait";
204
- data.timestampNs = timestampNs;
205
- data.qx = qx;
206
- data.qy = qy;
207
- data.qz = qz;
208
- data.qw = qw;
209
- data.tx = tx;
210
- data.ty = ty;
211
- data.tz = tz;
212
- data.hasTranslation = true; // AR mode always has translation
213
- data.arTrackingState = trackingStateStr;
214
- data.pixelReader =
215
- std::make_shared<AndroidNV21BufferReader>(std::move(bytes));
216
-
217
- // Dispatch on worklets-core's default context. That context is
218
- // initialised by JS' `Worklets.install()` (which runs at lib
219
- // bootstrap when worklets-core's module is imported); by the
220
- // time host worklets are registered, the default context is up.
221
- // The shared dispatch helper handles the registry snapshot,
222
- // host-object construction (inside the worklet thread), per-
223
- // worklet failure isolation, and invalidation.
224
- retailens::dispatchToHostWorklets(
225
- RNWorklet::JsiWorkletContext::getDefaultInstance(),
226
- std::move(data));
227
- }