react-native-mask-segment-canvas 0.1.0

package/src/components/MaskSegmentCanvas.types.ts

@@ -0,0 +1,216 @@
+import type { ReactNode } from 'react';
+import type { StyleProp, TextStyle, ViewStyle } from 'react-native';
+import type { SegmentRegion } from '../utils/maskSegmentation';
+import type { MaskSemanticColor } from '../utils/maskSemanticPalette';
+
+export type BgrColor = { b: number; g: number; r: number };
+
+export type MaskSegmentWatchState =
+  | 'init'
+  | 'images_loaded'
+  | 'mask_aligned'
+  | 'mask_sampled'
+  | 'regions_ready'
+  | 'layers_ready'
+  | 'interactive'
+  | 'mask_paths_ready'
+  | 'error';
+
+export type MaskSegmentWatchDetail = {
+  regionCount?: number;
+  maskPathsReady?: boolean;
+  freqLayersReady?: boolean;
+  errorMessage?: string;
+};
+
+export type PipelinePreset = 'high' | 'medium' | 'low';
+
+export type PipelineConfig = {
+  maxImageLongSide?: number;
+  /** 高低频 LAB 处理最长边（可低于 maxImageLongSide，Shader 拉伸采样） */
+  paintFreqMaxLongSide?: number;
+  originPreviewMaxLongSide?: number;
+  maskPathMaxLongSide?: number;
+  minContourArea?: number;
+  contourApproxEpsilon?: number;
+  maxRegions?: number;
+};
+
+export type MaskSegmentConfig = {
+  semanticColors?: MaskSemanticColor[];
+  baseboardMaxColorDist?: number;
+  blackThreshold?: number;
+  quantStep?: number;
+  baseboardStripQuantKeys?: string[];
+  wallQuantKeys?: string[];
+  cabinetQuantKeys?: string[];
+  maxRegionColors?: number;
+  secondarySemanticNames?: string[];
+  secondaryMinPixelRatio?: number;
+  junctionHRadiusPx?: number;
+  junctionVRadiusPx?: number;
+  kickBridgeHalfWPx?: number;
+  baseboardJunctionRowMarginPx?: number;
+  baseboardJunctionVReachPx?: number;
+  baseboardMinRunPx?: number;
+};
+
+export type PaintConfig = {
+  palette?: BgrColor[];
+  colorBaseOpacity?: number;
+  lLightOpacity?: number;
+  textureOpacity?: number;
+  lLowBlurKernel?: number;
+  lLowContrast?: number;
+  lLowBrightness?: number;
+  lHighGain?: number;
+  maskFeatherColor?: number;
+  maskFeatherTexture?: number;
+  regionOverlayFill?: string;
+  regionOutlineStrokeWidth?: number;
+};
+
+export type InteractionConfig = {
+  kickMaskPickRadiusPx?: number;
+  pickMapSearchRadiusPx?: number;
+  thinStripPadding?: number;
+  regionPadding?: number;
+  initRegionFlashMs?: number;
+  enableInitRegionFlash?: boolean;
+};
+
+export type PaintedRegionRecord = {
+  regionId: number;
+  regionName: string;
+  color: BgrColor;
+  configJson?: Record<string, unknown>;
+};
+
+export type MaskSegmentSession = {
+  version: 1;
+  originUrl: string;
+  maskUrl: string;
+  painted: PaintedRegionRecord[];
+  paintHistory: number[];
+  currentColor?: BgrColor;
+  currentColorConfigJson?: Record<string, unknown>;
+  savedAt: number;
+};
+
+export type SavePaintResult = {
+  filePath: string;
+  width: number;
+  height: number;
+  paintedCount: number;
+  previewPath?: string;
+};
+
+export type SavePaintOptions = {
+  destDir?: string;
+};
+
+export type PaintSuccessPayload = {
+  kind: 'painted';
+  regionId: number;
+  regionName: string;
+  color: BgrColor;
+  configJson?: Record<string, unknown>;
+};
+
+export type PaintBrushRequiredPayload = {
+  kind: 'brush_required';
+  /** 未选笔刷时的提示文案 */
+  hint: string;
+  regionId: number;
+  regionName: string;
+};
+
+export type PaintCallbackPayload = PaintSuccessPayload | PaintBrushRequiredPayload;
+
+export type OverlayButtonRenderProps = {
+  onPress: () => void;
+  disabled?: boolean;
+  text: string;
+};
+
+export type MaskSegmentCanvasRef = {
+  reset: () => void;
+  swap: (showOrigin?: boolean) => void;
+  save: (options?: SavePaintOptions) => Promise<SavePaintResult>;
+  session: () => MaskSegmentSession;
+  loadSession: (session: MaskSegmentSession) => void;
+  setPaintColor: (color: BgrColor, configJson?: Record<string, unknown>) => void;
+  setMaskConfig: (config: MaskSegmentConfig) => void;
+  clearAllPaint: () => void;
+  /** Undo the most recent single coloring (paint) step. Distinct from clearAllPaint (full reset). */
+  undoSelection?: () => void;
+  resegment: () => Promise<void>;
+  getRegions: () => SegmentRegion[];
+  getPaintedRegions: () => PaintedRegionRecord[];
+  /** Returns the most recent auto-export or save() result, if any. */
+  getLastExport?: () => SavePaintResult | null;
+};
+
+export type MaskSegmentCanvasProps = {
+  originUrl?: string;
+  maskUrl?: string;
+  /** @deprecated 使用 originUrl */
+  originImgPath?: string;
+  /** @deprecated 使用 maskUrl */
+  maskImgPath?: string;
+  /** 掩码语义识别色，初始化配置；等同 maskConfig.semanticColors */
+  semanticColors?: MaskSemanticColor[];
+  /** 分区虚线高亮色，初始化配置；等同 paintConfig.regionOverlayFill */
+  regionOutlineColor?: string;
+  maskConfig?: MaskSegmentConfig;
+  /** Performance preset (high / medium / low). Merged with pipelineConfig overrides. */
+  pipelinePreset?: PipelinePreset;
+  pipelineConfig?: PipelineConfig;
+  paintConfig?: PaintConfig;
+  interactionConfig?: InteractionConfig;
+  initialSession?: MaskSegmentSession;
+  initialPaintColor?: BgrColor;
+  initialPaintConfigJson?: Record<string, unknown>;
+  showDebugPickers?: boolean;
+  showToolbar?: boolean;
+  showColorBar?: boolean;
+  showStatusRow?: boolean;
+  showOverlayButtons?: boolean;
+  disabled?: boolean;
+  style?: StyleProp<ViewStyle>;
+  canvasStyle?: StyleProp<ViewStyle>;
+  /**
+   * Max container height available for this component (px). When set, the SDK
+   * computes canvas dimensions as a fit-contain within (screenWidth - 20, maxHeight)
+   * instead of using the full image aspect at full screen width.  This prevents
+   * internal ScrollView scrolling for tall images.
+   */
+  maxHeight?: number;
+  undoButtonStyle?: StyleProp<ViewStyle>;
+  compareButtonStyle?: StyleProp<ViewStyle>;
+  undoButtonTextStyle?: StyleProp<TextStyle>;
+  compareButtonTextStyle?: StyleProp<TextStyle>;
+  undoButtonText?: string;
+  compareButtonText?: string;
+  compareExitButtonText?: string;
+  renderUndoButton?: (props: OverlayButtonRenderProps) => ReactNode;
+  renderCompareButton?: (props: OverlayButtonRenderProps) => ReactNode;
+  onWatch?: (
+    state: MaskSegmentWatchState,
+    durationMs: number,
+    detail?: MaskSegmentWatchDetail,
+  ) => void;
+  onPaintCallback?: (payload: PaintCallbackPayload) => void;
+  onError?: (message: string, error?: unknown) => void;
+  /**
+   * When true, once the canvas reaches a ready interactive state (segmentation complete
+   * + any initialSession / painted colors applied), the SDK will automatically call its
+   * internal save pipeline to produce the recolored result image and fire onExported.
+   * This moves "auto-generate After preview" capability inside the SDK.
+   */
+  autoExportOnReady?: boolean;
+  /** Fired by SDK when autoExportOnReady produced a result (the recolored file). */
+  onExported?: (result: SavePaintResult) => void;
+};
+
+export type { SegmentRegion, MaskSemanticColor };

package/src/globals.d.ts

@@ -0,0 +1,19 @@
+// 补充 React Native 运行时可用的全局 API 类型声明
+
+declare var performance: {
+  now(): number;
+};
+
+declare function btoa(data: string): string;
+declare function atob(data: string): string;
+
+declare var TextEncoder: {
+  prototype: TextEncoder;
+  new (): TextEncoder;
+};
+
+declare interface TextEncoder {
+  encode(input?: string): Uint8Array;
+  encodeInto(input: string, dest: Uint8Array): { read: number; written: number };
+  readonly encoding: string;
+}

package/src/index.ts

@@ -0,0 +1,45 @@
+export { default } from './components/MaskSegmentCanvas';
+export type {
+  BgrColor,
+  InteractionConfig,
+  MaskSegmentCanvasProps,
+  MaskSegmentCanvasRef,
+  MaskSegmentConfig,
+  MaskSegmentSession,
+  MaskSegmentWatchDetail,
+  MaskSegmentWatchState,
+  MaskSemanticColor,
+  OverlayButtonRenderProps,
+  PaintBrushRequiredPayload,
+  PaintCallbackPayload,
+  PaintSuccessPayload,
+  PaintConfig,
+  PaintedRegionRecord,
+  PipelineConfig,
+  PipelinePreset,
+  SavePaintOptions,
+  SavePaintResult,
+  SegmentRegion,
+} from './components/MaskSegmentCanvas.types';
+export {
+  BASEBOARD_SEMANTIC_NAME,
+  MASK_SEMANTIC_COLORS,
+} from './utils/maskSemanticPalette';
+export {
+  createRuntimeConfig,
+  DEFAULT_INTERACTION_CONFIG,
+  DEFAULT_MASK_CONFIG,
+  DEFAULT_PAINT_CONFIG,
+  DEFAULT_PIPELINE_CONFIG,
+  PIPELINE_HIGH,
+  PIPELINE_LOW,
+  PIPELINE_MEDIUM,
+  PIPELINE_PRESETS,
+  getMaskSegmentRuntimeConfig,
+  resolvePipelineConfig,
+  setMaskSegmentRuntimeConfig,
+} from './utils/maskSegmentRuntime';
+export {
+  prewarmPngBgrCache,
+  prewarmPngBgrCacheAsync,
+} from './utils/pngImage';

package/src/shaders/regionPaint.sksl.ts

@@ -0,0 +1,71 @@
+/** SkSL：分区上色（保留原图明暗与纹理，按 paintColorMap 叠色） */
+export const REGION_PAINT_SKSL = `
+uniform shader originTex;
+uniform shader paintColorTex;
+uniform shader lowFreqTex;
+uniform shader highFreqTex;
+
+uniform float colorBaseOpacity;
+uniform float lLightOpacity;
+uniform float textureOpacity;
+uniform float showOrigin;
+
+float luminance(half3 c) {
+  return dot(c, half3(0.2126, 0.7152, 0.0722));
+}
+
+half3 setLuminance(float lum, half3 base) {
+  float diff = lum - luminance(base);
+  return base + diff;
+}
+
+half3 luminosityBlend(half3 base, half3 blend) {
+  return setLuminance(luminance(blend), base);
+}
+
+half3 overlayBlend(half3 base, half3 blend) {
+  half3 low = 2.0 * base * blend;
+  half3 high = 1.0 - 2.0 * (1.0 - base) * (1.0 - blend);
+  return mix(low, high, step(half3(0.5), base));
+}
+
+half4 main(float2 coord) {
+  half4 origin = originTex.eval(coord);
+  if (showOrigin > 0.5) {
+    return origin;
+  }
+
+  half4 paintEntry = paintColorTex.eval(coord);
+  // The paintColorMap uses Unpremul alpha: painted pixels → (R,G,B,255),
+  // transparent pixels → (0,0,0,0). GPU bilinear sampling interpolates
+  // straight-alpha values, so at boundaries rgb = trueColor * sampled.a
+  // (contaminated with black from the transparent neighbour).
+  //
+  // Unpremultiply to recover the true paint color:
+  //   trueColor = sampled.rgb / sampled.a
+  // This eliminates dark fringing at region boundaries.
+  float pa = paintEntry.a + 0.0001;
+  paintEntry.rgb /= pa;
+
+  // Gate sub-pixel alpha to kill residual sampling noise.
+  // Thresholds are deliberately low (≈1.3–3.8 in byte space) because
+  // post-unpremul the RGB is correct at any alpha ≥ 0 — we only need
+  // to suppress samples that contribute negligibly to the final blend.
+  // Using *= preserves the smooth edge; higher-alpha samples pass through.
+  paintEntry.a *= smoothstep(0.005, 0.015, paintEntry.a);
+
+  half3 paintRgb = paintEntry.rgb;
+  half lowL = lowFreqTex.eval(coord).r;
+  half highL = highFreqTex.eval(coord).r;
+
+  half3 base = paintRgb * colorBaseOpacity;
+  half3 lit = luminosityBlend(base, half3(lowL));
+  half3 withLight = mix(base, lit, lLightOpacity);
+  half3 tex = overlayBlend(withLight, half3(highL));
+  half3 finalRgb = mix(withLight, tex, textureOpacity);
+
+  // Soft edge blend using the (feathered) alpha from the paint color map as coverage.
+  half3 blended = mix(origin.rgb, finalRgb, paintEntry.a);
+  return half4(blended, 1.0);
+}
+`;

package/src/upng-js.d.ts

@@ -0,0 +1,33 @@
+declare module 'upng-js' {
+  export interface UPNGImage {
+    width: number;
+    height: number;
+    depth: number;
+    ctype: number;
+    frames: number;
+    tabs: Record<string, string>;
+    data: ArrayBuffer;
+  }
+
+  export function decode(buffer: ArrayBuffer): UPNGImage;
+
+  export function toRGBA8(img: UPNGImage): ArrayBuffer[];
+
+  export function encode(
+    imgs: ArrayBuffer[],
+    w: number,
+    h: number,
+    cnum: number,
+    dels?: number[],
+  ): ArrayBuffer;
+
+  export function encodeLL(
+    imgs: ArrayBuffer[],
+    w: number,
+    h: number,
+    cc: number,
+    ac: number,
+    depth: number,
+    dels?: number[],
+  ): ArrayBuffer;
+}

package/src/utils/compositePaintedImage.ts

@@ -0,0 +1,201 @@
+import { Buffer } from 'buffer';
+import RNFS from 'react-native-fs';
+import type { BgrColor, SavePaintResult } from '../components/MaskSegmentCanvas.types';
+import type { SkImage } from '@shopify/react-native-skia';
+// upng-js: used for PNG encode of CPU recolor.
+import UPNG from 'upng-js';
+import { renderPaintedImageOffscreen } from './paintShaderRuntime';
+import { writePngBase64ToFile, writePngBytesToFile } from './exportUtils';
+
+export type CompositePaintInput = {
+  originBuffer: Uint8Array;
+  cols: number;
+  rows: number;
+  pickBuffer: Uint8Array;
+  paintedRegions: Map<number, BgrColor>;
+  destDir?: string;
+  /**
+   * Preferred path for rich export: PNG base64 from makeImageSnapshot() — written
+   * directly to disk without an extra decode/re-encode round trip.
+   */
+  exportPngBase64?: string;
+  /**
+   * Preferred path for rich export: if the caller (MaskSegmentCanvas) provides bytes
+   * that were produced by makeImageSnapshot() on a high-resolution Canvas rendering the
+   * exact same PaintShaderLayer + regionPaint SkSL at work resolution, we write them
+   * directly. This captures the live editor 质感 (lighting + high/low-freq texture)
+   * without CPU pixel math and without a second declarative drawAsImage.
+   */
+  exportPngBytes?: Uint8Array;
+  /**
+   * Fallback rich path (when no pre-captured snapshot bytes): pass the live textures
+   * so we can try renderPaintedImageOffscreen (drawAsImage with the shader tree).
+   */
+  shaderTextures?: {
+    originImage: SkImage;
+    paintColorMap: SkImage;
+    lowFreqImage: SkImage;
+    highFreqImage: SkImage;
+  };
+  /** The logical size at which to render the shader tree for export (typically the work image res). */
+  renderWidth?: number;
+  renderHeight?: number;
+};
+
+/**
+ * CPU recolor: directly map pick codes to painted BGR colors (or copy origin).
+ * Produces RGBA PNG bytes via upng-js. This is the *fallback* path when rich shader offscreen
+ * is not available or fails. It produces flat colors without the editor's lighting + freq texture.
+ */
+function cpuRecolorToPngBytes(
+  originBgr: Uint8Array,
+  pickBuffer: Uint8Array,
+  paintedRegions: Map<number, BgrColor>,
+  cols: number,
+  rows: number,
+): Uint8Array {
+  const pixelCount = cols * rows;
+  const rgba = new Uint8Array(pixelCount * 4);
+  const colorByPickCode = new Map<number, BgrColor>();
+  for (const [regionId, color] of paintedRegions) {
+    colorByPickCode.set(regionId + 1, color);
+  }
+  for (let i = 0; i < pixelCount; i++) {
+    const code = pickBuffer[i];
+    const color = code > 0 ? colorByPickCode.get(code) : undefined;
+    const d = i * 4;
+    if (color) {
+      rgba[d] = color.r;
+      rgba[d + 1] = color.g;
+      rgba[d + 2] = color.b;
+      rgba[d + 3] = 255;
+    } else {
+      const s = i * 3;
+      rgba[d] = originBgr[s + 2]; // RGB <- BGR
+      rgba[d + 1] = originBgr[s + 1];
+      rgba[d + 2] = originBgr[s];
+      rgba[d + 3] = 255;
+    }
+  }
+  const png = UPNG.encode([rgba.buffer], cols, rows, 0);
+  return new Uint8Array(png as ArrayBuffer);
+}
+
+/** 将上色区域导出为 recolored PNG。
+ * 优先级（从好到保底）：
+ * 1. exportPngBytes（调用方用 makeImageSnapshot 在高分辨率 Canvas 上捕获的完整 shader 结果）—— 推荐的“保存快照”路径，无 CPU 逐像素，无二次 drawAsImage。
+ * 2. shaderTextures + render*（通过 renderPaintedImageOffscreen / drawAsImage 重建同一套 PaintShaderLayer + SkSL）。
+ * 3. CPU 逐像素 recolor（flat，无光照/纹理，仅作最后兜底，保证保存不中断）。
+ */
+export async function compositePaintedImage(
+  input: CompositePaintInput,
+): Promise<SavePaintResult> {
+  const {
+    originBuffer,
+    cols,
+    rows,
+    pickBuffer,
+    paintedRegions,
+    destDir,
+    exportPngBase64,
+    exportPngBytes,
+    shaderTextures,
+    renderWidth,
+    renderHeight,
+  } = input;
+  if (paintedRegions.size === 0) {
+    throw new Error('No painted regions, cannot save');
+  }
+
+  if (pickBuffer.length !== cols * rows) {
+    const msg = 'pickMap size does not match image';
+    console.error('[VIZ-SAVE] composite will throw:', msg, { pickLen: pickBuffer.length, expected: cols * rows, cols, rows });
+    throw new Error(msg);
+  }
+  if (originBuffer.length !== cols * rows * 3) {
+    const msg = 'Original buffer size does not match image';
+    console.error('[VIZ-SAVE] composite will throw:', msg, { originLen: originBuffer.length, expected: cols * rows * 3, cols, rows });
+    throw new Error(msg);
+  }
+
+  let pngBytesForWrite: Uint8Array | undefined;
+  let pngBase64ForWrite: string | undefined;
+  let usedSnapshot = false;
+  let usedRichShader = false;
+
+  // 1) Highest priority: pre-encoded PNG base64 from makeImageSnapshot (no extra conversion).
+  if (exportPngBase64 && exportPngBase64.length > 0) {
+    pngBase64ForWrite = exportPngBase64;
+    usedSnapshot = true;
+  }
+  // 1b) Snapshot bytes fallback (legacy callers).
+  else if (exportPngBytes && exportPngBytes.length > 0) {
+    pngBytesForWrite = exportPngBytes;
+    usedSnapshot = true;
+  }
+  // 2) 回退 rich：用 live 纹理 + drawAsImage 重建与编辑器一致的 shader 结果（带光照 + 频率纹理）。
+  else if (shaderTextures && renderWidth && renderHeight) {
+    try {
+      const offImg = await renderPaintedImageOffscreen({
+        originImage: shaderTextures.originImage,
+        paintColorMap: shaderTextures.paintColorMap,
+        lowFreqImage: shaderTextures.lowFreqImage,
+        highFreqImage: shaderTextures.highFreqImage,
+        width: renderWidth,
+        height: renderHeight,
+        showOrigin: false,
+      });
+      if (offImg) {
+        let b64 = '';
+        try {
+          const enc = (offImg as any).encodeToBase64;
+          if (typeof enc === 'function') {
+            b64 = enc.call(offImg) || '';
+          }
+        } catch (encErr) {
+          console.warn('[VIZ-SAVE] offscreen encodeToBase64 failed:', encErr);
+        }
+        if (b64 && b64.length > 0) {
+          pngBytesForWrite = new Uint8Array(Buffer.from(b64, 'base64'));
+          usedRichShader = true;
+        }
+        try {
+          const disp = (offImg as any).dispose;
+          if (typeof disp === 'function') disp.call(offImg);
+        } catch {}
+      }
+    } catch (e) {
+      console.warn('[VIZ-SAVE] rich shader offscreen for export failed (will fallback):', e);
+    }
+  }
+
+  // 3) 最后兜底：CPU 逐像素（flat 颜色，无 editor 质感）。
+  if (!pngBase64ForWrite && !pngBytesForWrite) {
+    try {
+      pngBytesForWrite = cpuRecolorToPngBytes(originBuffer, pickBuffer, paintedRegions, cols, rows);
+    } catch (e) {
+      throw new Error('CPU recolor PNG decoding failed');
+    }
+  }
+  const dir = destDir ?? RNFS.CachesDirectoryPath;
+  const filePath = `${dir}/painted_${Date.now()}.png`;
+  try {
+    if (pngBase64ForWrite) {
+      await writePngBase64ToFile(filePath, pngBase64ForWrite);
+    } else {
+      await writePngBytesToFile(filePath, pngBytesForWrite!);
+    }
+    void usedSnapshot;
+    void usedRichShader;
+  } catch (e) {
+    console.error('[VIZ-SAVE] composite writeFile threw:', e, { filePath, dir });
+    throw e;
+  }
+
+  return {
+    filePath,
+    width: cols,
+    height: rows,
+    paintedCount: paintedRegions.size,
+  };
+}

package/src/utils/exportUtils.ts

@@ -0,0 +1,40 @@
+import { Buffer } from 'buffer';
+import RNFS from 'react-native-fs';
+import type { BgrColor } from '../components/MaskSegmentCanvas.types';
+
+/** Stable fingerprint for painted region colors — used to reuse cached exports. */
+export function paintedRegionsFingerprint(painted: Map<number, BgrColor>): string {
+  if (painted.size === 0) {
+    return '';
+  }
+  const entries = [...painted.entries()].sort((a, b) => a[0] - b[0]);
+  return entries.map(([id, c]) => `${id}:${c.r},${c.g},${c.b}`).join('|');
+}
+
+export async function writePngBase64ToFile(filePath: string, base64: string): Promise<void> {
+  await RNFS.writeFile(filePath, base64, 'base64');
+}
+
+export async function writePngBytesToFile(filePath: string, bytes: Uint8Array): Promise<void> {
+  await RNFS.writeFile(filePath, Buffer.from(bytes).toString('base64'), 'base64');
+}
+
+export function stripFilePrefix(uri: string): string {
+  return uri.startsWith('file://') ? uri.slice('file://'.length) : uri;
+}
+
+export async function resolveExportResultForDestDir(
+  cached: { filePath: string; width: number; height: number; paintedCount: number; previewPath?: string },
+  destDir?: string,
+): Promise<{ filePath: string; width: number; height: number; paintedCount: number; previewPath?: string }> {
+  if (!destDir) {
+    return cached;
+  }
+  const src = stripFilePrefix(cached.filePath);
+  if (src.startsWith(destDir)) {
+    return cached;
+  }
+  const filePath = `${destDir}/painted_${Date.now()}.png`;
+  await RNFS.copyFile(src, filePath);
+  return { ...cached, filePath };
+}