react-native-image-stitcher 0.15.2 → 0.16.0

package/src/camera/captureWarnings.ts

@@ -0,0 +1,190 @@
+// SPDX-License-Identifier: Apache-2.0
+/**
+ * captureWarnings — non-fatal quality / behaviour signals attached to a
+ * SUCCESSFUL capture result.  A stitch that *failed* surfaces a
+ * `CameraError` (via the `ok:false` result + `onError`); these warnings
+ * cover the "it succeeded but the host/user should know something" cases:
+ *
+ *   • LOW_FRAME_UTILIZATION — fewer than `threshold` (default 70 %) of the
+ *     captured frames survived the confidence filter, so the panorama may
+ *     be patchy / shorter than intended.
+ *   • LATERAL_DRIFT_FINALIZE — the capture was auto-finalized early because
+ *     the phone drifted sideways (item 6); only the pre-drift portion was
+ *     stitched.
+ *   • HIGH_PAN_SPEED — the pan exceeded the recommended pace at some point
+ *     during the capture (the live "too fast" cue fired), so motion blur /
+ *     thin overlap may have hurt the result.
+ *
+ * `<Camera>` builds these at finalize and threads them into BOTH the
+ * `onCapture` result payload (so any host — not just the example app —
+ * learns of degraded output programmatically) AND the crop editor's banner
+ * (so the user sees it before accepting the crop).
+ *
+ * Pure + dependency-free so it's unit-testable in isolation (the lib's jest
+ * config is pure-TS and can't mount `<Camera>`), mirroring
+ * `classifyStitchError` / `buildPanoramaInitialSettings`.
+ */
+
+/** Stable codes a host can branch on (in addition to the message). */
+export type CaptureWarningCode =
+  | 'LOW_FRAME_UTILIZATION'
+  | 'LATERAL_DRIFT_FINALIZE'
+  | 'HIGH_PAN_SPEED';
+
+export interface CaptureWarning {
+  /** Stable, host-switchable code. */
+  code: CaptureWarningCode;
+  /** Plain-language default message (shown in the crop banner). */
+  message: string;
+  /** Frames the engine tried to use (LOW_FRAME_UTILIZATION only). */
+  framesRequested?: number;
+  /** Frames that survived the confidence filter (LOW_FRAME_UTILIZATION). */
+  framesIncluded?: number;
+  /** included / requested in [0, 1] (LOW_FRAME_UTILIZATION only). */
+  utilization?: number;
+}
+
+/**
+ * Default trip point for LOW_FRAME_UTILIZATION: warn when fewer than 70 %
+ * of captured frames survived.  Matches the threshold the user specified.
+ */
+export const LOW_FRAME_UTILIZATION_THRESHOLD = 0.7;
+
+/**
+ * The overridable message strings for the three capture warnings.  This is
+ * the SINGLE SOURCE OF TRUTH for the default English warning copy — the
+ * `GuidanceCopy` surface re-uses these defaults (see `cameraGuidanceCopy`),
+ * so a host that localises via the `guidanceCopy` `<Camera>` prop re-words
+ * these too.
+ *
+ * `lowFrameUtilization` is a TEMPLATE: the placeholders `{included}`,
+ * `{requested}` and `{percent}` are substituted at build time with the
+ * actual frame counts.  A translation must keep the placeholders (any it
+ * omits is simply not interpolated; an unknown placeholder is left as-is).
+ */
+export interface CaptureWarningCopy {
+  /** LOW_FRAME_UTILIZATION — template; `{included}`/`{requested}`/`{percent}`. */
+  lowFrameUtilization: string;
+  /** LATERAL_DRIFT_FINALIZE. */
+  lateralDriftFinalize: string;
+  /** HIGH_PAN_SPEED. */
+  highPanSpeed: string;
+}
+
+export const DEFAULT_CAPTURE_WARNING_COPY: CaptureWarningCopy = {
+  lowFrameUtilization:
+    'Only {included} of {requested} captured frames ({percent}%) could be '
+    + 'used — the panorama may be incomplete. Pan more slowly and steadily '
+    + 'next time.',
+  lateralDriftFinalize:
+    'Capture stopped early because the phone drifted sideways — only the '
+    + 'part captured before the drift was stitched.',
+  highPanSpeed:
+    'The capture was taken faster than the recommended pace — the result '
+    + 'may not be the best. Pan more slowly next time.',
+};
+
+/**
+ * Substitute `{name}` placeholders in a template with `vars[name]`.  An
+ * unknown placeholder is left verbatim (so a malformed translation degrades
+ * to showing `{percent}` rather than throwing).
+ */
+function fillTemplate(
+  tpl: string,
+  vars: Record<string, string | number>,
+): string {
+  return tpl.replace(/\{(\w+)\}/g, (m, k: string) =>
+    k in vars ? String(vars[k]) : m,
+  );
+}
+
+export interface BuildCaptureWarningsInput {
+  /** `framesRequested` from the native finalize result. */
+  framesRequested?: number;
+  /** `framesIncluded` from the native finalize result. */
+  framesIncluded?: number;
+  /** True when this finalize was triggered by lateral-drift auto-stop. */
+  lateralFinalize?: boolean;
+  /** True when the pan exceeded the recommended pace during the capture. */
+  highPanSpeed?: boolean;
+  /** Override the LOW_FRAME_UTILIZATION trip point (fraction in (0, 1]). */
+  lowFrameUtilizationThreshold?: number;
+  /**
+   * Localised / re-worded warning messages.  Missing keys fall back to
+   * {@link DEFAULT_CAPTURE_WARNING_COPY}.  `<Camera>` threads the resolved
+   * `guidanceCopy` here so the crop-banner warnings honour the host's i18n.
+   */
+  copy?: Partial<CaptureWarningCopy>;
+}
+
+/**
+ * Build the warning list for a successful capture.  Order is by cause →
+ * symptom: a lateral-drift stop (the reason a capture is short) is listed
+ * before the low-utilization symptom it usually produces.
+ */
+export function buildCaptureWarnings(
+  input: BuildCaptureWarningsInput,
+): CaptureWarning[] {
+  const {
+    framesRequested,
+    framesIncluded,
+    lateralFinalize = false,
+    highPanSpeed = false,
+    lowFrameUtilizationThreshold = LOW_FRAME_UTILIZATION_THRESHOLD,
+  } = input;
+  const copy: CaptureWarningCopy = {
+    ...DEFAULT_CAPTURE_WARNING_COPY,
+    ...stripUndefinedCopy(input.copy),
+  };
+
+  const warnings: CaptureWarning[] = [];
+
+  if (lateralFinalize) {
+    warnings.push({
+      code: 'LATERAL_DRIFT_FINALIZE',
+      message: copy.lateralDriftFinalize,
+    });
+  }
+
+  if (highPanSpeed) {
+    warnings.push({
+      code: 'HIGH_PAN_SPEED',
+      message: copy.highPanSpeed,
+    });
+  }
+
+  if (
+    typeof framesRequested === 'number'
+    && typeof framesIncluded === 'number'
+    && framesRequested > 0
+    && framesIncluded >= 0
+    && framesIncluded < framesRequested * lowFrameUtilizationThreshold
+  ) {
+    const utilization = framesIncluded / framesRequested;
+    warnings.push({
+      code: 'LOW_FRAME_UTILIZATION',
+      message: fillTemplate(copy.lowFrameUtilization, {
+        included: framesIncluded,
+        requested: framesRequested,
+        percent: Math.round(utilization * 100),
+      }),
+      framesRequested,
+      framesIncluded,
+      utilization,
+    });
+  }
+
+  return warnings;
+}
+
+/** Drop `undefined` values so a partial override never clobbers a default. */
+function stripUndefinedCopy(
+  o?: Partial<CaptureWarningCopy>,
+): Partial<CaptureWarningCopy> {
+  if (!o) return {};
+  const out: Partial<CaptureWarningCopy> = {};
+  (Object.keys(o) as (keyof CaptureWarningCopy)[]).forEach((k) => {
+    if (o[k] !== undefined) out[k] = o[k];
+  });
+  return out;
+}

package/src/camera/classifyStitchError.ts

@@ -0,0 +1,68 @@
+// SPDX-License-Identifier: Apache-2.0
+/**
+ * classifyStitchError — map a raw native stitch-failure message to a
+ * `CameraErrorCode`.
+ *
+ * This is the load-bearing C++↔JS contract: the native pipeline reports
+ * failures only as exception strings (see cpp/stitcher.cpp — the warp /
+ * cumulative-canvas guards throw "warpRoi too large … degenerate camera
+ * params", cv::Stitcher reports "need more images", etc.), and this
+ * function is the single place that turns those strings into the typed
+ * codes `<Camera onError>` surfaces.  The code then drives the friendly
+ * copy in {@link userFacingStitchError} (cameraErrorMessages.ts) — e.g.
+ * STITCH_CAMERA_PARAMS_FAIL → "Please pan more slowly".
+ *
+ * Extracted from the inline chain in Camera.tsx so the contract is
+ * unit-testable against the actual native strings (the lib's jest config
+ * is pure-TS and can't mount <Camera>).
+ *
+ * Ordering matters — the branches are checked top-to-bottom and the first
+ * match wins:
+ *   1. need-more-images (insufficient overlap) — most specific.
+ *   2. homography estimation.
+ *   3. degenerate camera params / warp-canvas guard (the divergent-warp
+ *      OOM path, now converted to a clean throw by the canvas guard).
+ *   4. low-quality / disjoint output (the post-stitch validator, v0.16).
+ *   5. out-of-memory (incl. the pre-stitch memory abort).
+ *   6. fallback — an unclassified finalize failure.
+ */
+import type { CameraErrorCode } from './Camera';
+
+export function classifyStitchError(message: string): CameraErrorCode {
+  // Insufficient overlap surfaces two ways: cv::Stitcher's
+  // ERR_NEED_MORE_IMGS ("need more images") and the manual pipeline's
+  // "0 valid pairwise matches / frames may not overlap enough" — both are
+  // the same recoverable "pan more slowly" case.
+  if (/need more images|pairwise match|overlap enough/i.test(message)) {
+    return 'STITCH_NEED_MORE_IMGS';
+  }
+  if (/homography/i.test(message)) {
+    return 'STITCH_HOMOGRAPHY_FAIL';
+  }
+  // Degenerate camera params — the rapid/wide-pan divergent-warp path.
+  // Broadened beyond the original "camera params" so a future reword of
+  // the native throw can't silently drop the "pan more slowly" copy: the
+  // per-frame warp guard and the cumulative-canvas guard carry "warpRoi"
+  // / "canvas too large" / "degenerate" respectively (see cpp/stitcher.cpp
+  // degenerateFrameException / degenerateCanvasException).  Kept AFTER the
+  // homography branch and BEFORE the OOM branch so a true OOM string still
+  // routes to STITCH_OOM.
+  if (/camera params|warpRoi|degenerate|canvas too large/i.test(message)) {
+    return 'STITCH_CAMERA_PARAMS_FAIL';
+  }
+  // v0.16 — the native post-stitch validator rejected the output as
+  // disjoint / fragmented / wildly mis-proportioned (frames didn't connect
+  // into one coherent panorama).  The cpp throw carries "stitch validation"
+  // / "disjoint" / "fragmented" (see cpp/stitcher.cpp validateStitchOutput).
+  // Kept BEFORE the OOM branch so its distinct message isn't swallowed.
+  if (/stitch validation|disjoint|fragmented|low-quality stitch/i.test(message)) {
+    return 'STITCH_LOW_QUALITY';
+  }
+  // OOM, including the pre-stitch headroom abort ("pre-stitch memory abort"
+  // / "memory abort") that fires when even the minimal streaming config
+  // won't fit — same user remedy (shorter sweep), so same code.
+  if (/out of memory|oom|memory abort/i.test(message)) {
+    return 'STITCH_OOM';
+  }
+  return 'PANORAMA_FINALIZE_FAILED';
+}

package/src/camera/cropGeometry.ts

@@ -0,0 +1,268 @@
+// SPDX-License-Identifier: Apache-2.0
+/**
+ * cropGeometry — pure coordinate + quad helpers behind the item-7
+ * draggable-corner crop editor (`RectCropPreview`).
+ *
+ * The editor shows the full result image with a `resizeMode="contain"`
+ * letterbox: the image is centred and uniformly scaled to fit the layout
+ * box, leaving symmetric bars on one axis.  The 4 draggable corners live in
+ * ON-SCREEN coordinates (the touch space PanResponder reports), but the
+ * native crop needs IMAGE-PIXEL coordinates.  These helpers are the
+ * letterbox transform and its inverse — extracted verbatim from
+ * `example/InscribedRectDebug.tsx` (~lines 178-204), which mapped an
+ * inscribed-rect from image px → screen the same way.
+ *
+ * Everything here is pure (no React, no native) so it's unit-testable
+ * without booting a render — same posture as `contentRotationDeg` and
+ * `buildPanoramaInitialSettings`.
+ *
+ * Coordinate conventions:
+ *   - A `Point` is `{ x, y }`.  Screen points are in the layout box's
+ *     local space (origin = box top-left); image points are in pixel
+ *     space (origin = image top-left, range [0..imageW] × [0..imageH]).
+ *   - A `Quad` is exactly 4 points.  `orderQuadCorners` canonicalises
+ *     winding to [TL, TR, BR, BL] so downstream native perspective
+ *     rectify gets corners in the order it expects.
+ */
+
+/** A 2-D point in either screen-local or image-pixel space. */
+export interface Point {
+  x: number;
+  y: number;
+}
+
+/** The contain-fit letterbox layout of the image inside its box. */
+export interface ContainLayout {
+  /** Layout box width (on-screen px). */
+  width: number;
+  /** Layout box height (on-screen px). */
+  height: number;
+}
+
+/** Exactly four points (corners of a crop quad). */
+export type Quad = [Point, Point, Point, Point];
+
+
+/**
+ * The contain-fit transform: uniform scale + centring offsets that map
+ * image-pixel space into the on-screen layout box.  Returns `null` when
+ * any dimension is non-positive (nothing to lay out).
+ *
+ * `scale` is `min(box.w / imageW, box.h / imageH)` — the same
+ * `resizeMode="contain"` math RN's <Image> applies — and `offX`/`offY`
+ * centre the scaled image, producing the letterbox bars.
+ */
+export function containFit(
+  layout: ContainLayout,
+  imageW: number,
+  imageH: number,
+): { scale: number; offX: number; offY: number } | null {
+  if (
+    layout.width <= 0
+    || layout.height <= 0
+    || imageW <= 0
+    || imageH <= 0
+  ) {
+    return null;
+  }
+  const scale = Math.min(layout.width / imageW, layout.height / imageH);
+  const dispW = imageW * scale;
+  const dispH = imageH * scale;
+  const offX = (layout.width - dispW) / 2;
+  const offY = (layout.height - dispH) / 2;
+  return { scale, offX, offY };
+}
+
+
+/**
+ * Map an on-screen point (layout-box local coords) → image-pixel coords.
+ * Inverse of {@link imageToScreen}.  The result is clamped to
+ * `[0..imageW] × [0..imageH]` so a corner dragged onto / past the
+ * letterbox bar still yields a valid in-bounds pixel for the native crop
+ * (the user can't pick pixels that don't exist).
+ *
+ * Returns the un-mapped point unchanged when the layout is degenerate
+ * (see {@link containFit}) — the caller has no valid letterbox yet.
+ */
+export function screenToImage(
+  point: Point,
+  layout: ContainLayout,
+  imageW: number,
+  imageH: number,
+): Point {
+  const fit = containFit(layout, imageW, imageH);
+  if (!fit) return point;
+  const x = (point.x - fit.offX) / fit.scale;
+  const y = (point.y - fit.offY) / fit.scale;
+  return {
+    x: clamp(x, 0, imageW),
+    y: clamp(y, 0, imageH),
+  };
+}
+
+
+/**
+ * Map an image-pixel point → on-screen point (layout-box local coords).
+ * Inverse of {@link screenToImage}.  Used to seed the draggable corners
+ * from an image-space initial rect and to keep the overlay aligned to the
+ * letterboxed image.
+ *
+ * Returns the un-mapped point unchanged when the layout is degenerate.
+ */
+export function imageToScreen(
+  point: Point,
+  layout: ContainLayout,
+  imageW: number,
+  imageH: number,
+): Point {
+  const fit = containFit(layout, imageW, imageH);
+  if (!fit) return point;
+  return {
+    x: fit.offX + point.x * fit.scale,
+    y: fit.offY + point.y * fit.scale,
+  };
+}
+
+
+/**
+ * Re-order 4 arbitrary corner points into canonical
+ * [TL, TR, BR, BL] (clockwise from top-left) winding.
+ *
+ * Strategy (robust to slight perspective skew, no trig):
+ *   - Top two = the two points with the smallest `y`; bottom two = the
+ *     largest `y`.  Within each pair, the smaller `x` is left.
+ * Ties on `y` (a perfectly axis-aligned rect) resolve deterministically
+ * because the sort is stable and we then split by `x`.
+ *
+ * This matches the corner order the native `cropToQuad` perspective
+ * rectify expects (dst rect: TL→TR→BR→BL).
+ */
+export function orderQuadCorners(pts: Quad): Quad {
+  // Sort a copy by y ascending so [0,1] are the top pair, [2,3] bottom.
+  const byY = [...pts].sort((a, b) => a.y - b.y);
+  const [t0, t1, b0, b1] = byY;
+  // Within each horizontal pair, smaller x is the left corner.
+  const [tl, tr] = t0.x <= t1.x ? [t0, t1] : [t1, t0];
+  const [bl, br] = b0.x <= b1.x ? [b0, b1] : [b1, b0];
+  return [tl, tr, br, bl];
+}
+
+
+/**
+ * 2× the signed area of a polygon via the shoelace formula.  Positive for
+ * counter-clockwise winding, negative for clockwise, ~0 for degenerate.
+ * Exported for tests + reused by {@link isQuadValid}.
+ */
+export function signedArea2(pts: Quad): number {
+  let sum = 0;
+  for (let i = 0; i < pts.length; i++) {
+    const a = pts[i];
+    const b = pts[(i + 1) % pts.length];
+    sum += a.x * b.y - b.x * a.y;
+  }
+  return sum;
+}
+
+
+/**
+ * True when the 4 points form a usable crop quad:
+ *   1. **Non-degenerate area** — `|signed area|` ≥ `minArea` (default
+ *      `1`, i.e. at least 1 px²).  Rejects all-collinear / zero-size.
+ *   2. **Convex** — every cross-product of consecutive edges shares one
+ *      sign (allowing zero for a straight, axis-aligned corner).  Rejects
+ *      self-intersecting / "bowtie" quads, which the native perspective
+ *      warp can't rectify.
+ *
+ * Operates on the points in their given winding (call `orderQuadCorners`
+ * first if you need canonical order); convexity is winding-agnostic.
+ */
+export function isQuadValid(pts: Quad, minArea = 1): boolean {
+  if (Math.abs(signedArea2(pts)) < minArea * 2) return false;
+  return isConvex(pts);
+}
+
+
+/**
+ * Target rectangle size for the perspective `dst` quad, derived from the
+ * 4 ORDERED ([TL, TR, BR, BL]) image-pixel corners:
+ *   - `w` = average of the top edge (TL→TR) and bottom edge (BL→BR)
+ *     lengths.
+ *   - `h` = average of the left edge (TL→BL) and right edge (TR→BR)
+ *     lengths.
+ * Averaging opposite edges gives a stable output size for a skewed quad
+ * (each pair of opposite edges differs under perspective; the mean is the
+ * least-distorting target).  Rounds to whole pixels — the native crop
+ * allocates an integer-sized bitmap.
+ *
+ * Caller must pass corners already in [TL, TR, BR, BL] order (use
+ * {@link orderQuadCorners}); the math assumes that winding.
+ */
+export function rectSizeForQuad(orderedImagePts: Quad): {
+  w: number;
+  h: number;
+} {
+  const [tl, tr, br, bl] = orderedImagePts;
+  const top = dist(tl, tr);
+  const bottom = dist(bl, br);
+  const left = dist(tl, bl);
+  const right = dist(tr, br);
+  return {
+    w: Math.round((top + bottom) / 2),
+    h: Math.round((left + right) / 2),
+  };
+}
+
+
+/**
+ * True when an ORDERED ([TL, TR, BR, BL]) image-pixel quad is, within
+ * `tolerancePx`, an axis-aligned rectangle — i.e. the cheap axis-aligned
+ * `cropToRect` path applies and no perspective warp is needed.  The parent
+ * uses this to choose `cropToRect` vs `cropToQuad`.
+ *
+ * Checks the two top/bottom corners share a `y` and the two left/right
+ * corners share an `x`, all within tolerance.
+ */
+export function isAxisAlignedRect(
+  orderedImagePts: Quad,
+  tolerancePx = 1,
+): boolean {
+  const [tl, tr, br, bl] = orderedImagePts;
+  return (
+    Math.abs(tl.y - tr.y) <= tolerancePx
+    && Math.abs(bl.y - br.y) <= tolerancePx
+    && Math.abs(tl.x - bl.x) <= tolerancePx
+    && Math.abs(tr.x - br.x) <= tolerancePx
+  );
+}
+
+
+/** Convexity test: all consecutive edge cross-products share a sign. */
+function isConvex(pts: Quad): boolean {
+  let sign = 0;
+  for (let i = 0; i < pts.length; i++) {
+    const a = pts[i];
+    const b = pts[(i + 1) % pts.length];
+    const c = pts[(i + 2) % pts.length];
+    const cross = (b.x - a.x) * (c.y - b.y) - (b.y - a.y) * (c.x - b.x);
+    if (cross !== 0) {
+      const s = cross > 0 ? 1 : -1;
+      if (sign === 0) sign = s;
+      else if (s !== sign) return false;
+    }
+  }
+  return true;
+}
+
+
+/** Euclidean distance between two points. */
+function dist(a: Point, b: Point): number {
+  return Math.hypot(a.x - b.x, a.y - b.y);
+}
+
+
+/** Clamp `v` into the inclusive `[lo, hi]` range. */
+function clamp(v: number, lo: number, hi: number): number {
+  if (v < lo) return lo;
+  if (v > hi) return hi;
+  return v;
+}

package/src/camera/displayDecodeImageProps.ts

@@ -0,0 +1,25 @@
+// SPDX-License-Identifier: Apache-2.0
+/**
+ * DISPLAY_DECODE_IMAGE_PROPS — props every <Image> that displays a
+ * FULL-RES capture (a stitched panorama or a photo file) must spread.
+ *
+ * Why this exists (the accumulation half of the OOM crash):
+ *   On Android 8+, decoded bitmap pixels live in the NATIVE heap, and the
+ *   source here is the full-resolution capture file — a wide panorama can
+ *   be tens of megapixels.  Without `resizeMethod="resize"`, Android/Fresco
+ *   decodes the source at FULL resolution into a native bitmap that the
+ *   mounted <Image> pins (not LRU-evictable), and Fresco's URI-keyed cache
+ *   keeps it even after the view unmounts.  Each capture (especially wide
+ *   panoramas) then accumulates tens of MB of native heap until lmkd
+ *   OOM-kills the app.  'resize' decodes at the on-screen (~device-width)
+ *   size instead, making per-image memory tiny and panorama-size-
+ *   independent.  No-op on iOS (harmless).
+ *
+ * Centralised (rather than a bare `resizeMethod="resize"` at each call
+ * site) so the decode strategy + its rationale have one home, and so the
+ * contract is unit-testable without mounting a component.  Spread it:
+ *   <Image source={...} resizeMode="cover" {...DISPLAY_DECODE_IMAGE_PROPS} />
+ */
+export const DISPLAY_DECODE_IMAGE_PROPS = {
+  resizeMethod: 'resize',
+} as const;