react-native-image-stitcher 0.1.0

package/src/sensors/useIMUTranslationGate.ts

@@ -0,0 +1,347 @@
+// SPDX-License-Identifier: Apache-2.0
+//
+// useIMUTranslationGate.ts — JS-side IMU translation tracker for the
+// non-AR translation-warning banner + (optional) gate force-accept.
+//
+// 2026-05-17 (Issue #4-A v3): rewritten on top of `expo-sensors`
+// `DeviceMotion` (which returns gravity-subtracted linear
+// acceleration via Apple's CoreMotion fusion on iOS and Android's
+// `TYPE_LINEAR_ACCELERATION` sensor on Android — both significantly
+// less noisy than raw accel + JS-side IIR gravity subtraction).
+// Tracks a SINGLE device-frame axis (device-X — the phone's lateral /
+// short side) rather than the 3D translation magnitude.
+//
+// Why exists
+// ──────────
+//
+// In non-AR mode the SDK has no ARSession pose stream, so the shared
+// C++ `KeyframeGate`'s translation-budget feature stays at zero and
+// never trips.  This hook fills the gap on the JS side and emits a
+// budget-crossed callback the host can wire to either:
+//
+//   (a) `markNextFrameAsLastKeyframe()` — tell the gate "force-accept
+//       the next frame regardless of overlap", so the trailing-edge
+//       frame still lands when the operator translates instead of
+//       rotates.
+//
+//   (b) A user-facing warning banner ("Rotate the camera instead of
+//       moving it sideways" — see AuditCaptureScreen).
+//
+// AuditCaptureScreen wires it to both.
+//
+// Why device-X (the shorter side)
+// ───────────────────────────────
+//
+// We track motion ALONG the pan axis (the direction the operator is
+// supposed to be rotating-through but might be translating-through
+// instead) because translation orthogonal to the pan axis is
+// acceptable — vertical translation while panning horizontally in
+// portrait, for example, doesn't cause horizontal parallax.
+//
+// The pan axis maps to device-X in BOTH supported orientations
+// (per memory/ar-stitching-two-modes.md):
+//
+//   Portrait  + horizontal pan: device-X = user-left/right = pan axis.
+//   Landscape + vertical   pan: device-X has rotated 90° into the
+//                                user's up/down direction = pan axis.
+//
+// The lateral axis of the phone (its short side) always aligns with
+// the pan direction in either supported mode, so a single-axis
+// tracker works without needing to know which orientation we're in.
+//
+// Drift mitigation
+// ────────────────
+//
+// `DeviceMotion.acceleration` (gravity removed in native code via
+// IMU fusion) has a noise floor roughly 30-50 % lower than what the
+// previous raw-accel + JS IIR pipeline produced.  Single-axis math
+// further reduces apparent drift by ≈√3 vs the prior 3D magnitude.
+// Together they should keep the typical "stationary phone" reading
+// below ~5-10 cm even after several seconds.
+//
+// Anchor resets happen at (a) recording start (via the host calling
+// `resetAnchor()` from handleHoldStart) and (b) every accepted
+// keyframe — these bound the per-interval drift window to typically
+// 0.3-2 s.
+//
+// What we no longer do
+// ────────────────────
+//
+//   - JS-side 1-pole IIR for gravity subtraction (native API gives
+//     gravity-subtracted accel directly).
+//   - 3D vector magnitude (now single device-X axis).
+//   - Velocity damping (kept as a safety net at 5%/sample so a
+//     persistent noise-floor offset doesn't slowly drift the axis —
+//     low cost, high robustness).
+
+import { useCallback, useEffect, useRef } from 'react';
+import { DeviceMotion } from 'expo-sensors';
+import type { DeviceMotionMeasurement } from 'expo-sensors';
+
+// expo-sensors doesn't re-export Subscription from its index, but
+// `addListener` returns one — use the inferred return type so we
+// don't have to chase the right deep-import path.
+type DeviceMotionSubscription = ReturnType<typeof DeviceMotion.addListener>;
+
+
+export interface UseIMUTranslationGateOptions {
+  /**
+   * Whether the gate is engaged.  Pass `false` to skip the subscription
+   * entirely — useful when the host is in AR mode (where the gate
+   * gets pose-derived translation natively).  Hot-toggleable;
+   * subscribing/unsubscribing is cheap.
+   */
+  enabled: boolean;
+
+  /**
+   * Translation budget in METRES along the device-X (pan) axis.
+   * When the integrated displacement magnitude exceeds this since
+   * the last accept, the hook fires `onBudgetExceeded`.  Default
+   * 0.40 m / 40 cm (80 % of the 50 cm default
+   * `flowMaxTranslationCm`).  Caller typically passes
+   * `panoramaSettings.flowMaxTranslationCm * 0.8 / 100`.
+   */
+  budgetMeters?: number;
+
+  /**
+   * Update interval in MILLISECONDS for the DeviceMotion sensor.
+   * Default 20 ms ≈ 50 Hz.  Lower (faster sampling) = more accurate
+   * integration; higher = lower CPU + battery.  Matches the previous
+   * raw-accel cadence so reset/integrate behaviour stays comparable.
+   */
+  sampleIntervalMs?: number;
+
+  /**
+   * Fired exactly once per "budget crossing" — i.e., when the
+   * running translation along device-X crosses `budgetMeters` from
+   * below.  The host is responsible for both (a) calling
+   * `IncrementalStitcher.markNextFrameAsLastKeyframe()` and
+   * (b) invoking the returned `resetAnchor()` once the next
+   * keyframe actually accepts, so the integrator restarts from zero.
+   */
+  onBudgetExceeded: () => void;
+
+  /**
+   * 2026-05-18 (Issue #4 investigation) — when true, log every Nth
+   * accelerometer sample (default N=20 ≈ 400 ms at 50 Hz) showing
+   * the current `acceleration.x`, accumulated `posX`, and time
+   * since anchor reset.  Helps diagnose drift behaviour vs real
+   * translation magnitude in field testing.  Defaults to false —
+   * production captures stay quiet.
+   */
+  debug?: boolean;
+}
+
+
+export interface UseIMUTranslationGateReturn {
+  /**
+   * Reset the running translation to zero.  Call this at recording
+   * start AND after each confirmed keyframe accept — the typical
+   * wiring is to subscribe to `IncrementalStateUpdate` and
+   * call `resetAnchor()` from inside the listener AND from the host's
+   * `handleHoldStart`.
+   */
+  resetAnchor: () => void;
+
+  /**
+   * Read the current running displacement along device-X in METRES.
+   * Returns the absolute value (sign is uninteresting — either left
+   * or right counts the same toward the budget).
+   * Useful for the on-screen debug HUD ("translation since last
+   * accept: 0.07 m").  Not exposed via state — host polls if needed.
+   */
+  getCurrentTranslationM: () => number;
+}
+
+
+/**
+ * IMU-based translation tracker — single-axis (device-X / pan axis),
+ * fused IMU via `expo-sensors` `DeviceMotion`.  See file header for
+ * algorithm + rationale.  No platform-specific code; the underlying
+ * native fusion is platform-aware (CoreMotion on iOS, fused
+ * `TYPE_LINEAR_ACCELERATION` on Android).
+ */
+export function useIMUTranslationGate(
+  options: UseIMUTranslationGateOptions,
+): UseIMUTranslationGateReturn {
+  const {
+    enabled,
+    budgetMeters = 0.40,
+    sampleIntervalMs = 20,
+    onBudgetExceeded,
+    debug = false,
+  } = options;
+
+  // Integrator state, kept in refs so the listener can write without
+  // re-creating its closure on every render.
+  // ─ velX  : velocity along device-X (m/s)
+  // ─ posX  : position along device-X (m)
+  // ─ lastMs: epoch ms of the previous sample (for dt)
+  // ─ budgetCrossed: debounce flag — clears on resetAnchor
+  // ─ sampleCount: rolling counter for debug log throttle
+  // ─ anchorMs: timestamp of the most recent resetAnchor (or first
+  //             sample) — gives "time since anchor" in debug output
+  const velX = useRef<number>(0);
+  const posX = useRef<number>(0);
+  const lastMs = useRef<number>(0);
+  const budgetCrossed = useRef<boolean>(false);
+  const sampleCount = useRef<number>(0);
+  const anchorMs = useRef<number>(0);
+
+  // Keep the callback in a ref so we don't tear down + re-subscribe
+  // on every prop change.  React idiom for stable callback identity.
+  const onBudgetExceededRef = useRef(onBudgetExceeded);
+  useEffect(() => { onBudgetExceededRef.current = onBudgetExceeded; },
+    [onBudgetExceeded]);
+
+  useEffect(() => {
+    if (debug) {
+      // eslint-disable-next-line no-console
+      console.log(
+        `[IMUTransGate] effect re-run: enabled=${enabled} `
+        + `budget=${budgetMeters.toFixed(2)}m sampleIntervalMs=${sampleIntervalMs}`,
+      );
+    }
+    if (!enabled) return;
+
+    // Lock in the DeviceMotion update rate.  Other expo-sensors
+    // consumers in the SDK can override later; the LAST setter wins
+    // per Expo's docs, which is fine because our budget logic
+    // tolerates a wide range of cadences.
+    DeviceMotion.setUpdateInterval(sampleIntervalMs);
+
+    // Reset state on (re-)engage so the first measurement after
+    // enabled-toggles-true doesn't carry stale velocity from a
+    // previous capture session.
+    velX.current = 0;
+    posX.current = 0;
+    lastMs.current = 0;
+    budgetCrossed.current = false;
+    sampleCount.current = 0;
+    anchorMs.current = Date.now();
+
+    // 2026-05-18 (Issue #3 diagnostics) — track whether we've ever
+    // received a non-null `acceleration` for this subscription.  If
+    // the user reports "no logs" we can correlate with these
+    // start-of-subscription and first-real-data markers.
+    let everGotData = false;
+    let nullSampleCount = 0;
+    if (debug) {
+      // eslint-disable-next-line no-console
+      console.log('[IMUTransGate] subscribing to DeviceMotion');
+    }
+
+    const sub: DeviceMotionSubscription = DeviceMotion.addListener((m: DeviceMotionMeasurement) => {
+      const a = m.acceleration;       // gravity-subtracted (m/s²)
+      if (!a) {
+        nullSampleCount += 1;
+        if (debug && nullSampleCount === 1) {
+          // eslint-disable-next-line no-console
+          console.log(
+            '[IMUTransGate] first sample: acceleration=null '
+            + '(CoreMotion warming up; will retry on next sample)',
+          );
+        }
+        if (debug && nullSampleCount > 0 && nullSampleCount % 100 === 0) {
+          // eslint-disable-next-line no-console
+          console.log(
+            `[IMUTransGate] STILL receiving null acceleration after `
+            + `${nullSampleCount} samples — sensor source may be broken`,
+          );
+        }
+        return;                 // can be null briefly on cold start
+      }
+      if (debug && !everGotData) {
+        everGotData = true;
+        // eslint-disable-next-line no-console
+        console.log(
+          `[IMUTransGate] first real sample: ax=${a.x.toFixed(3)} `
+          + `ay=${a.y.toFixed(3)} az=${a.z.toFixed(3)} m/s² `
+          + `(after ${nullSampleCount} null sample(s))`,
+        );
+      }
+      const now = Date.now();
+      if (lastMs.current === 0) {
+        lastMs.current = now;
+        return;
+      }
+      const dt = Math.max(0, Math.min(0.1, (now - lastMs.current) / 1000.0));
+      lastMs.current = now;
+      if (dt === 0) return;
+
+      // Single-axis integration along device-X (lateral / pan axis).
+      // See file header for why device-X is the right axis in both
+      // portrait and landscape captures.
+      velX.current += a.x * dt;
+      velX.current *= 0.95;            // 5%/sample damping — see file header
+      posX.current += velX.current * dt;
+
+      const mag = Math.abs(posX.current);
+
+      // 2026-05-18 (Issue #4 investigation) — debug-gated diagnostic
+      // log.  Throttled to every 20th sample (~400 ms at 50 Hz) so
+      // the log isn't a firehose.  When this runs and we still see
+      // posX hovering at < 5 cm during a real translation, the
+      // sensor source isn't capturing what we think it is.
+      sampleCount.current += 1;
+      if (debug && sampleCount.current % 20 === 0) {
+        const secs = (now - anchorMs.current) / 1000.0;
+        // eslint-disable-next-line no-console
+        console.log(
+          `[IMUTransGate] t+${secs.toFixed(2)}s `
+          + `ax=${a.x.toFixed(3)}m/s² `
+          + `velX=${velX.current.toFixed(4)}m/s `
+          + `posX=${posX.current.toFixed(4)}m `
+          + `(|mag|=${mag.toFixed(4)}m, budget=${budgetMeters.toFixed(2)}m, crossed=${budgetCrossed.current})`,
+        );
+      }
+
+      // Budget crossing — fire exactly once per crossing (the
+      // `budgetCrossed` flag clears on `resetAnchor`).
+      if (!budgetCrossed.current && mag >= budgetMeters) {
+        budgetCrossed.current = true;
+        if (debug) {
+          // eslint-disable-next-line no-console
+          console.log(
+            `[IMUTransGate] BUDGET CROSSED at posX=${posX.current.toFixed(4)}m `
+            + `(budget=${budgetMeters.toFixed(2)}m)`,
+          );
+        }
+        onBudgetExceededRef.current();
+      }
+    });
+
+    return () => {
+      if (debug) {
+        // eslint-disable-next-line no-console
+        console.log(
+          `[IMUTransGate] unsubscribing (everGotData=${everGotData}, `
+          + `nullSamples=${nullSampleCount}, realSamples=${sampleCount.current})`,
+        );
+      }
+      sub.remove();
+    };
+  }, [enabled, budgetMeters, sampleIntervalMs, debug]);
+
+  // 2026-05-18 (Issue B meta-bug fix): wrap the returned functions in
+  // useCallback so the hook's return value is REFERENTIALLY STABLE
+  // across renders.  Consumer code (AuditCaptureScreen) puts `imuGate`
+  // in a useEffect dep array; without stability that effect re-runs
+  // every render, wiping out the prevAcceptedCount delta-tracker
+  // (which is what caused the resetAnchor-too-often bug we just
+  // diagnosed).  These functions only touch refs, so empty-deps
+  // useCallback is safe — no stale-closure risk.
+  const resetAnchor = useCallback(() => {
+    velX.current = 0;
+    posX.current = 0;
+    lastMs.current = 0;
+    budgetCrossed.current = false;
+    sampleCount.current = 0;
+    anchorMs.current = Date.now();
+  }, []);
+  const getCurrentTranslationM = useCallback(
+    () => Math.abs(posX.current),
+    [],
+  );
+  return { resetAnchor, getCurrentTranslationM };
+}

package/src/stitching/IncrementalStitcherView.tsx

@@ -0,0 +1,198 @@
+// SPDX-License-Identifier: Apache-2.0
+/**
+ * IncrementalStitcherView — live preview component for the panorama
+ * engine.  Renders the latest snapshot JPEG written by the native
+ * side, with confidence + hint overlays.
+ *
+ * Why <Image> + cache-bust query string instead of a custom native
+ * view: per the design doc's open question, the JPEG-write approach
+ * is V1; if perf measurements show we're hitting RN's image cache
+ * too hard, swap in an `Animated.Image` or a native UIView with
+ * an in-memory bitmap.  Until then, the simple path keeps the
+ * cross-platform surface tiny.
+ */
+
+import React, { useMemo } from 'react';
+import {
+  ActivityIndicator,
+  Image,
+  StyleSheet,
+  Text,
+  View,
+  type ViewStyle,
+} from 'react-native';
+import type { IncrementalState } from './incremental';
+import type { IncrementalHint } from './useIncrementalStitcher';
+
+
+export interface IncrementalStitcherViewProps {
+  /** Latest engine state — typically `useIncrementalStitcher().state`. */
+  state: IncrementalState | null;
+  /**
+   * Active hint to surface as a banner overlay.  Pass
+   * `useIncrementalStitcher().hint` directly; the view picks the
+   * right wording.
+   */
+  hint: IncrementalHint;
+  /**
+   * Confidence ring colour driver — typically
+   * `useIncrementalStitcher().confidenceLevel`.
+   */
+  confidenceLevel?: 'high' | 'medium' | null;
+  /** Outer container style (size, position).  Required: the view
+   *  has no intrinsic size since the panorama dimensions vary. */
+  style?: ViewStyle;
+  /**
+   * Optional override for the spinner shown before the first frame
+   * is accepted.  Default is a subtle "Pan to begin" caption.
+   */
+  emptyText?: string;
+}
+
+
+function hintMessage(hint: IncrementalHint): string | null {
+  switch (hint) {
+    case 'slow-down':         return 'Slow down — alignment lost';
+    case 'scene-uniform':     return 'Pan to a textured area';
+    case 'alignment-lost':    return 'Slow down — re-acquiring alignment';
+    case 'tracking-poor':     return 'Hold steady — AR re-acquiring';
+    default:                  return null;
+  }
+}
+
+
+export function IncrementalStitcherView({
+  state,
+  hint,
+  confidenceLevel,
+  style,
+  emptyText = 'Pan to begin capturing',
+}: IncrementalStitcherViewProps): React.JSX.Element {
+  // Cache-bust the panorama URI.  The native side rotates through
+  // 4 filenames so the path itself changes between snapshots, plus
+  // we tag with acceptedCount as belt-and-suspenders since RN's
+  // image cache on iOS sometimes ignores file:// query strings.
+  const imageUri = useMemo(() => {
+    if (!state?.panoramaPath) return null;
+    return `file://${state.panoramaPath}?v=${state.acceptedCount}`;
+  }, [state?.panoramaPath, state?.acceptedCount]);
+
+  // Use the panorama's NATURAL aspect ratio so the strip widens as
+  // the user pans across.  Falls back to 4:3 (a single frame's
+  // shape) before any snapshot has been written.  Without this the
+  // PiP was forced into a 3:1 letterbox, cropping the actual
+  // panorama to a thin slice across the middle.
+  const naturalAspect = state?.width && state?.height && state.height > 0
+    ? state.width / state.height
+    : 4 / 3;
+
+  const ringColor = confidenceLevel === 'high'
+    ? '#1aaf5d'
+    : confidenceLevel === 'medium'
+      ? '#e6b800'
+      : 'rgba(255,255,255,0.35)';
+
+  const message = hintMessage(hint);
+
+  return (
+    <View style={[styles.container, { aspectRatio: naturalAspect }, style]}>
+      {imageUri ? (
+        // `contain` so the FULL panorama is visible inside the
+        // strip, not cropped to a slice.  Background fills the
+        // letterbox edges.  Key={acceptedCount} forces RN to
+        // remount the Image component each accept — the surest
+        // way to defeat the native image cache on file:// URIs.
+        <Image
+          key={state?.acceptedCount ?? 0}
+          source={{ uri: imageUri }}
+          style={StyleSheet.absoluteFill}
+          resizeMode="contain"
+          fadeDuration={0}
+        />
+      ) : (
+        <View style={styles.empty}>
+          <ActivityIndicator color="#fff" />
+          <Text style={styles.emptyText}>{emptyText}</Text>
+        </View>
+      )}
+
+      {/* Confidence ring — subtle border that picks up colour for
+          medium-confidence accepts.  Always visible (white-translucent
+          when no confidence signal) so the operator can see exactly
+          where the live preview is on screen. */}
+      <View
+        pointerEvents="none"
+        style={[styles.ring, { borderColor: ringColor }]}
+      />
+
+      {message ? (
+        <View pointerEvents="none" style={styles.hintBanner}>
+          <Text style={styles.hintText}>{message}</Text>
+        </View>
+      ) : null}
+
+      {state ? (
+        <View pointerEvents="none" style={styles.counterPill}>
+          <Text style={styles.counterText}>
+            {state.acceptedCount} frame{state.acceptedCount === 1 ? '' : 's'}
+          </Text>
+        </View>
+      ) : null}
+    </View>
+  );
+}
+
+
+const styles = StyleSheet.create({
+  container: {
+    backgroundColor: 'rgba(0,0,0,0.7)',
+    overflow: 'hidden',
+    borderRadius: 8,
+  },
+  empty: {
+    ...StyleSheet.absoluteFillObject,
+    alignItems: 'center',
+    justifyContent: 'center',
+    gap: 6,
+  },
+  emptyText: {
+    color: '#fff',
+    fontSize: 12,
+    opacity: 0.85,
+  },
+  ring: {
+    ...StyleSheet.absoluteFillObject,
+    borderRadius: 8,
+    borderWidth: 2,
+  },
+  hintBanner: {
+    position: 'absolute',
+    left: 8,
+    right: 8,
+    bottom: 8,
+    paddingVertical: 4,
+    paddingHorizontal: 8,
+    backgroundColor: 'rgba(220, 53, 69, 0.92)',
+    borderRadius: 6,
+  },
+  hintText: {
+    color: '#fff',
+    fontSize: 11,
+    textAlign: 'center',
+    fontWeight: '500',
+  },
+  counterPill: {
+    position: 'absolute',
+    top: 6,
+    right: 6,
+    paddingVertical: 2,
+    paddingHorizontal: 8,
+    backgroundColor: 'rgba(0, 0, 0, 0.55)',
+    borderRadius: 10,
+  },
+  counterText: {
+    color: '#fff',
+    fontSize: 10,
+    fontWeight: '500',
+  },
+});