react-native-sherpa-onnx 0.3.5 → 0.3.7

package/src/download/validation.ts

@@ -3,6 +3,7 @@ import {
   stat,
   exists,
   readDir,
+  unlink,
   type ReadDirResItemT,
 } from '@dr.pogodin/react-native-fs';
 import SherpaOnnx from '../NativeSherpaOnnx';
@@ -25,6 +26,37 @@ export class ValidationResult {
   }
 }
 
+/**
+ * Delete a directory and all contents. No-op if the path is missing.
+ * Best-effort: continues on per-entry errors (permissions, race).
+ */
+export async function removeDirectoryRecursive(dirPath: string): Promise<void> {
+  if (!(await exists(dirPath))) return;
+  let entries: ReadDirResItemT[];
+  try {
+    entries = await readDir(dirPath);
+  } catch {
+    return;
+  }
+  for (const entry of entries) {
+    const childPath = `${dirPath}/${entry.name}`.replace(/\/+/g, '/');
+    try {
+      if (entry.isDirectory()) {
+        await removeDirectoryRecursive(childPath);
+      } else {
+        await unlink(childPath);
+      }
+    } catch {
+      // ignore
+    }
+  }
+  try {
+    await unlink(dirPath);
+  } catch {
+    // ignore
+  }
+}
+
 /**
  * Parse checksum.txt format into a Map of filename -> hash
  * Expected format:
@@ -148,12 +180,15 @@ export async function validateExtractedFiles(
 
       for (const entry of entries) {
         if (entry.isDirectory()) {
-          const nested = await collectFilesRecursive(
-            entry.path,
-            depth + 1,
-            maxDepth
-          );
-          files.push(...nested);
+          const subPath = entry.path;
+          if (subPath != null) {
+            const nested = await collectFilesRecursive(
+              subPath,
+              depth + 1,
+              maxDepth
+            );
+            files.push(...nested);
+          }
         } else {
           files.push(entry);
         }
@@ -175,13 +210,13 @@ export async function validateExtractedFiles(
     }
 
     let hasModelLikeFiles = actualFiles.some((file) =>
-      isModelLikeFile(file.name)
+      isModelLikeFile(file.name ?? '')
     );
 
     if (!hasModelLikeFiles) {
       const nestedFiles = await collectFilesRecursive(modelDir);
       hasModelLikeFiles = nestedFiles.some((file) =>
-        isModelLikeFile(file.name)
+        isModelLikeFile(file.name ?? '')
       );
     }
 
@@ -203,6 +238,77 @@ export async function validateExtractedFiles(
   }
 }
 
+/** True if the file is a native sherpa-onnx model file (e.g. encoder.onnx). Excludes our metadata (.ready, manifest.json). */
+function isNativeModelFileName(name: string): boolean {
+  const lower = name.toLowerCase();
+  return lower.endsWith('.onnx') || lower.endsWith('.bin');
+}
+
+/**
+ * Check if a directory contains native model files (.onnx or .bin) at the top level or one level of subdirectories.
+ * Used to find the actual model dir; ignores our metadata (manifest.json, .ready).
+ */
+async function dirContainsModelFiles(dir: string): Promise<boolean> {
+  const entries = await readDir(dir);
+  const files = entries.filter((e) => !e.isDirectory());
+  if (files.some((f) => isNativeModelFileName(f.name ?? ''))) return true;
+  const subdirs = entries.filter((e) => e.isDirectory());
+  for (const sub of subdirs) {
+    const subPath = sub?.path;
+    if (subPath == null) continue;
+    const subEntries = await readDir(subPath);
+    const subFiles = subEntries.filter((e) => !e.isDirectory());
+    if (subFiles.some((f) => isNativeModelFileName(f.name ?? ''))) return true;
+  }
+  return false;
+}
+
+/**
+ * Resolve the directory that actually contains model files.
+ * After extracting a tarball, model files often end up in a single top-level subdirectory
+ * (e.g. installDir/modelId/encoder.onnx). Native APIs expect the path to the folder
+ * that directly contains encoder.onnx, decoder.onnx, etc.
+ *
+ * - If installDir itself contains native model files (.onnx/.bin), returns installDir.
+ * - If installDir has exactly one subdirectory that contains native model files, returns that subdirectory path.
+ *   (Ignores our metadata: .ready, manifest.json.) This can produce paths like
+ *   .../tts/vits-piper-de_DE-thorsten-medium-int8/vits-piper-de_DE-thorsten-medium-int8 when the
+ *   archive extracts a top-level folder with the same name as the model id; that is intentional.
+ * - Otherwise returns installDir unchanged.
+ */
+export async function resolveActualModelDir(
+  installDir: string
+): Promise<string> {
+  try {
+    const dirExists = await exists(installDir);
+    if (!dirExists) return installDir;
+
+    const entries = await readDir(installDir);
+    const topLevelFiles = entries.filter((e) => !e.isDirectory());
+    if (topLevelFiles.some((f) => isNativeModelFileName(f.name ?? ''))) {
+      return installDir;
+    }
+
+    const subdirs = entries.filter((e) => e.isDirectory());
+    const firstSubdir = subdirs[0];
+    const singleSubdir = subdirs.length === 1 ? firstSubdir : undefined;
+
+    if (singleSubdir != null) {
+      const candidatePath = singleSubdir.path;
+      if (
+        candidatePath != null &&
+        (await dirContainsModelFiles(candidatePath))
+      ) {
+        return candidatePath;
+      }
+    }
+
+    return installDir;
+  } catch {
+    return installDir;
+  }
+}
+
 /**
  * Get available disk space (in bytes)
  * This is a simplified version. For accurate values on Android/iOS, use native modules.

package/src/{download → extraction}/extractTarBz2.ts

@@ -47,7 +47,9 @@ export async function extractTarBz2(
   if (signal) {
     const onAbort = () => {
       try {
-        SherpaOnnx.cancelExtractTarBz2();
+        // Use per-path cancel so aborting this extraction doesn't affect
+        // other extractions that may be running in parallel.
+        SherpaOnnx.cancelExtractBySourcePath(sourcePath);
       } catch {
         // Ignore cancel errors to avoid crashing on abort.
       }

package/src/extraction/extractTarZst.ts

@@ -0,0 +1,79 @@
+import { DeviceEventEmitter } from 'react-native';
+import SherpaOnnx from '../NativeSherpaOnnx';
+
+export type ExtractProgressEvent = {
+  bytes: number;
+  totalBytes: number;
+  percent: number;
+};
+
+type ExtractResult = {
+  success: boolean;
+  path?: string;
+  sha256?: string;
+  reason?: string;
+};
+
+export async function extractTarZst(
+  sourcePath: string,
+  targetPath: string,
+  force = true,
+  onProgress?: (event: ExtractProgressEvent) => void,
+  signal?: AbortSignal
+): Promise<ExtractResult> {
+  let subscription: { remove: () => void } | null = null;
+  let removeAbortListener: (() => void) | null = null;
+
+  if (signal?.aborted) {
+    const abortError = new Error('Extraction aborted');
+    abortError.name = 'AbortError';
+    throw abortError;
+  }
+
+  if (onProgress) {
+    subscription = DeviceEventEmitter.addListener(
+      'extractTarZstProgress',
+      (event: ExtractProgressEvent & { sourcePath?: string }) => {
+        if (event.sourcePath != null && event.sourcePath !== sourcePath) {
+          return;
+        }
+        const safePercent = Math.max(0, Math.min(100, event.percent));
+        onProgress({ ...event, percent: safePercent });
+      }
+    );
+  }
+
+  if (signal) {
+    const onAbort = () => {
+      try {
+        // Use per-path cancel so aborting this extraction doesn't affect
+        // other extractions that may be running in parallel.
+        SherpaOnnx.cancelExtractBySourcePath(sourcePath);
+      } catch {
+        // Ignore cancel errors to avoid crashing on abort.
+      }
+    };
+    signal.addEventListener('abort', onAbort);
+    removeAbortListener = () => signal.removeEventListener('abort', onAbort);
+  }
+
+  try {
+    const result = await SherpaOnnx.extractTarZst(
+      sourcePath,
+      targetPath,
+      force
+    );
+    if (!result.success) {
+      const message = result.reason || 'Extraction failed';
+      const error = new Error(message);
+      if (signal?.aborted || /cancel/i.test(message)) {
+        error.name = 'AbortError';
+      }
+      throw error;
+    }
+    return result;
+  } finally {
+    subscription?.remove();
+    removeAbortListener?.();
+  }
+}

package/src/extraction/index.ts

@@ -0,0 +1,269 @@
+/**
+ * Extraction subpath: list and extract compressed model archives (.tar.zst / .tar.bz2).
+ *
+ * Three entry points:
+ *  - getBundledArchives(packName)    – Android PAD packs (STORAGE_FILES or APK_ASSETS)
+ *  - listBundledArchives(dirPath)    – any filesystem directory (cross-platform)
+ *  - extractArchive(archive, target) – unified extraction (auto-selects path or asset-stream)
+ *
+ * After extraction, use listModelsAtPath / autoModelPath from the main package.
+ */
+
+import { DeviceEventEmitter, Platform } from 'react-native';
+import { readDir, stat, exists } from '@dr.pogodin/react-native-fs';
+import SherpaOnnx from '../NativeSherpaOnnx';
+import { extractTarZst } from './extractTarZst';
+import { extractTarBz2 } from './extractTarBz2';
+import type {
+  BundledArchive,
+  ExtractArchiveOptions,
+  ExtractResult,
+  ExtractProgressEvent,
+} from './types';
+
+export type {
+  BundledArchive,
+  ExtractArchiveOptions,
+  ExtractResult,
+  ExtractProgressEvent,
+} from './types';
+
+// ── Constants & helpers ───────────────────────────────────────────
+
+const TAR_ZST = '.tar.zst';
+const TAR_BZ2 = '.tar.bz2';
+
+function formatFromFilename(name: string): 'tar.zst' | 'tar.bz2' | null {
+  if (name.endsWith(TAR_ZST)) return 'tar.zst';
+  if (name.endsWith(TAR_BZ2)) return 'tar.bz2';
+  return null;
+}
+
+function modelIdFromFilename(filename: string): string {
+  if (filename.endsWith(TAR_ZST)) return filename.slice(0, -TAR_ZST.length);
+  if (filename.endsWith(TAR_BZ2)) return filename.slice(0, -TAR_BZ2.length);
+  return filename;
+}
+
+/**
+ * Scan a filesystem directory for .tar.zst / .tar.bz2 entries.
+ * Shared by getBundledArchives (STORAGE_FILES) and listBundledArchives.
+ */
+async function scanDirectoryForArchives(
+  directoryPath: string
+): Promise<BundledArchive[]> {
+  const dirExists = await exists(directoryPath);
+  if (!dirExists) return [];
+
+  const entries = await readDir(directoryPath);
+  const archives: BundledArchive[] = [];
+
+  for (const entry of entries) {
+    const format = formatFromFilename(entry.name);
+    if (!format || !entry.isFile()) continue;
+
+    let fileSize = 0;
+    try {
+      const s = await stat(entry.path);
+      fileSize = s.size ?? 0;
+    } catch {
+      // stat may fail on some filesystems; fileSize stays 0
+    }
+
+    archives.push({
+      modelId: modelIdFromFilename(entry.name),
+      archivePath: entry.path,
+      format,
+      fileSize,
+    });
+  }
+
+  return archives;
+}
+
+// ── Public API ────────────────────────────────────────────────────
+
+/**
+ * List compressed archives delivered via a **Play Asset Delivery** pack.
+ *
+ * - **STORAGE_FILES** packs: scans the pack directory on the filesystem.
+ * - **APK_ASSETS** packs: queries the Android AssetManager for embedded archive paths.
+ *   Archives returned with `fromAsset: true` are extracted by streaming from the APK
+ *   (no temp copy needed).
+ * - **iOS / unavailable pack**: returns `null`.
+ *
+ * @param packName  Name of the PAD asset pack (e.g. `"sherpa_models"`)
+ */
+export async function getBundledArchives(
+  packName: string
+): Promise<BundledArchive[] | null> {
+  if (Platform.OS !== 'android') {
+    return null;
+  }
+
+  const packPath = await SherpaOnnx.getAssetPackPath(packName);
+
+  if (packPath != null && packPath.length > 0) {
+    const archives = await scanDirectoryForArchives(packPath);
+    return archives.length > 0 ? archives : null;
+  }
+
+  const assetPaths = await SherpaOnnx.listBundledArchiveAssetPaths(packName);
+  if (assetPaths.length === 0) return null;
+
+  return assetPaths.map((archivePath) => {
+    const filename = archivePath.split('/').pop() ?? archivePath;
+    const format = formatFromFilename(filename) ?? 'tar.zst';
+    return {
+      modelId: modelIdFromFilename(filename),
+      archivePath,
+      format,
+      fromAsset: true,
+    };
+  });
+}
+
+/**
+ * List `.tar.zst` and `.tar.bz2` archives in a filesystem directory.
+ *
+ * Works on **all platforms** — use for:
+ * - iOS main-bundle archives (`MainBundlePath + '/models'`)
+ * - Archives downloaded to the documents directory
+ * - Any other folder containing compressed model archives
+ *
+ * @param directoryPath  Absolute path to the directory to scan
+ */
+export async function listBundledArchives(
+  directoryPath: string
+): Promise<BundledArchive[]> {
+  return scanDirectoryForArchives(directoryPath);
+}
+
+/**
+ * Extract a single archive to the target directory.
+ *
+ * Handles both source types transparently:
+ * - **Filesystem archives** (from `listBundledArchives` or PAD STORAGE_FILES) —
+ *   uses the regular path-based extraction.
+ * - **APK asset archives** (`fromAsset: true`, from PAD APK_ASSETS) —
+ *   streams directly from the APK without copying the archive to disk first.
+ *
+ * @param archive    Descriptor returned by `getBundledArchives` or `listBundledArchives`
+ * @param targetPath Directory to extract into (e.g. `DocumentDirectoryPath + '/models'`)
+ * @param options    `force` (default `true`), `onProgress`, `signal` (AbortSignal)
+ */
+export async function extractArchive(
+  archive: BundledArchive,
+  targetPath: string,
+  options?: ExtractArchiveOptions
+): Promise<ExtractResult> {
+  const force = options?.force !== false;
+  const onProgress = options?.onProgress;
+  const signal = options?.signal;
+
+  if (signal?.aborted) {
+    const err = new Error('Extraction aborted');
+    err.name = 'AbortError';
+    throw err;
+  }
+
+  const useAssetStream =
+    Platform.OS === 'android' &&
+    (archive.fromAsset === true ||
+      archive.archivePath.startsWith('asset_packs/'));
+
+  if (useAssetStream) {
+    return extractFromAsset(archive, targetPath, force, onProgress, signal);
+  }
+
+  if (archive.format === 'tar.zst') {
+    return extractTarZst(
+      archive.archivePath,
+      targetPath,
+      force,
+      onProgress,
+      signal
+    );
+  }
+  return extractTarBz2(
+    archive.archivePath,
+    targetPath,
+    force,
+    onProgress,
+    signal
+  );
+}
+
+// ── Internal: asset-stream extraction (Android APK_ASSETS) ───────
+
+async function extractFromAsset(
+  archive: BundledArchive,
+  targetPath: string,
+  force: boolean,
+  onProgress?: (event: ExtractProgressEvent) => void,
+  signal?: AbortSignal
+): Promise<ExtractResult> {
+  const eventName =
+    archive.format === 'tar.zst'
+      ? 'extractTarZstProgress'
+      : 'extractTarBz2Progress';
+
+  let subscription: { remove: () => void } | null = null;
+  let removeAbortListener: (() => void) | null = null;
+
+  if (onProgress) {
+    subscription = DeviceEventEmitter.addListener(
+      eventName,
+      (event: ExtractProgressEvent & { sourcePath?: string }) => {
+        if (
+          event.sourcePath != null &&
+          event.sourcePath !== archive.archivePath
+        ) {
+          return;
+        }
+        const safePercent = Math.max(0, Math.min(100, event.percent));
+        onProgress({ ...event, percent: safePercent });
+      }
+    );
+  }
+
+  if (signal) {
+    const onAbort = () => {
+      try {
+        SherpaOnnx.cancelExtractBySourcePath(archive.archivePath);
+      } catch {
+        // ignore
+      }
+    };
+    signal.addEventListener('abort', onAbort);
+    removeAbortListener = () => signal.removeEventListener('abort', onAbort);
+  }
+
+  try {
+    const result =
+      archive.format === 'tar.zst'
+        ? await SherpaOnnx.extractTarZstFromAsset(
+            archive.archivePath,
+            targetPath,
+            force
+          )
+        : await SherpaOnnx.extractTarBz2FromAsset(
+            archive.archivePath,
+            targetPath,
+            force
+          );
+
+    if (!result.success) {
+      const message = result.reason ?? 'Extraction failed';
+      const error = new Error(message);
+      if (signal?.aborted || /cancel/i.test(message)) {
+        error.name = 'AbortError';
+      }
+      throw error;
+    }
+    return result;
+  } finally {
+    subscription?.remove();
+    removeAbortListener?.();
+  }
+}

package/src/extraction/types.ts

@@ -0,0 +1,63 @@
+/**
+ * Types for the extraction subpath.
+ *
+ * A BundledArchive describes a compressed model archive (.tar.zst or .tar.bz2)
+ * that can come from two distinct sources:
+ *
+ *  1. **Filesystem** — a regular file on disk (PAD STORAGE_FILES, iOS bundle,
+ *     downloaded archive, etc.). `fromAsset` is absent or `false`.
+ *  2. **Android APK asset** — embedded in the APK via PAD APK_ASSETS.
+ *     `fromAsset` is `true`; extraction streams directly from the APK.
+ *
+ * The consumer does not need to distinguish between the two:
+ * `extractArchive()` handles both transparently.
+ */
+
+/** Describes one compressed model archive. */
+export type BundledArchive = {
+  /** Identifier derived from the archive filename (filename minus the extension). */
+  modelId: string;
+  /**
+   * Path to the archive.
+   * - Filesystem archives: absolute path (e.g. `/data/.../models/whisper-tiny.tar.zst`).
+   * - APK assets: asset path (e.g. `asset_packs/sherpa_models/assets/whisper-tiny.tar.zst`).
+   */
+  archivePath: string;
+  /** Compression format. */
+  format: 'tar.zst' | 'tar.bz2';
+  /** File size in bytes (available for filesystem archives; 0 or absent for APK assets). */
+  fileSize?: number;
+  /** `true` when the archive lives inside the APK (APK_ASSETS). Absent for filesystem archives. */
+  fromAsset?: boolean;
+};
+
+/** Progress event emitted during extraction. */
+export type ExtractProgressEvent = {
+  /** Bytes extracted so far. */
+  bytes: number;
+  /** Total bytes of the archive (may be 0 when unknown). */
+  totalBytes: number;
+  /** Progress percentage 0–100. */
+  percent: number;
+};
+
+/** Result returned by `extractArchive`. */
+export type ExtractResult = {
+  success: boolean;
+  /** Absolute path to the extracted directory (on success). */
+  path?: string;
+  /** SHA-256 hex digest of the source archive (when available). */
+  sha256?: string;
+  /** Error description (on failure). */
+  reason?: string;
+};
+
+/** Options for `extractArchive`. */
+export type ExtractArchiveOptions = {
+  /** Overwrite existing files. Defaults to `true`. */
+  force?: boolean;
+  /** Callback for extraction progress. */
+  onProgress?: (event: ExtractProgressEvent) => void;
+  /** AbortSignal to cancel the extraction. */
+  signal?: AbortSignal;
+};

package/src/index.tsx

@@ -18,10 +18,12 @@ export {
 
 export { copyFileToContentUri } from './tts';
 
+export { getModelLicenses, type ModelLicense } from './licenses';
 // Note: Feature-specific exports are available via subpath imports:
 // - import { createSTT, createStreamingSTT, ... } from 'react-native-sherpa-onnx/stt'
 // - import { createTTS, ... } from 'react-native-sherpa-onnx/tts'
 // - import { ... } from 'react-native-sherpa-onnx/download'
+// - import { getBundledArchives, listBundledArchives, extractArchive } from 'react-native-sherpa-onnx/extraction'
 // - import { ... } from 'react-native-sherpa-onnx/vad' (planned)
 // - import { ... } from 'react-native-sherpa-onnx/diarization' (planned)
 // - import { ... } from 'react-native-sherpa-onnx/enhancement' (planned)

package/src/licenses.ts

@@ -0,0 +1,100 @@
+import SherpaOnnx from './NativeSherpaOnnx';
+
+export interface ModelLicense {
+  asset_name: string;
+  license_type: string;
+  commercial_use: 'yes' | 'no' | 'conditional' | 'restricted' | 'unknown';
+  confidence: string;
+  detection_source: string;
+  license_file: string;
+}
+
+export async function getModelLicenses(): Promise<ModelLicense[]> {
+  const asrPath = 'model_licenses/asr-models-license-status.csv';
+  const qnnPath = 'model_licenses/qnn-asr-models-license-status.csv';
+  const ttsPath = 'model_licenses/tts-models-license-status.csv';
+
+  const results = await Promise.allSettled([
+    SherpaOnnx.readAssetFileAsUtf8(asrPath),
+    SherpaOnnx.readAssetFileAsUtf8(qnnPath),
+    SherpaOnnx.readAssetFileAsUtf8(ttsPath),
+  ]);
+
+  const [asrResult, qnnResult, ttsResult] = results;
+
+  const licenses: ModelLicense[] = [];
+
+  if (asrResult.status === 'fulfilled') {
+    licenses.push(...parseCsv(asrResult.value));
+  } else {
+    console.warn(
+      `[SherpaOnnx] Failed to load ASR model licenses: ${asrResult.reason}`
+    );
+  }
+
+  if (qnnResult.status === 'fulfilled') {
+    licenses.push(...parseCsv(qnnResult.value));
+  } else {
+    console.warn(
+      `[SherpaOnnx] Failed to load QNN model licenses: ${qnnResult.reason}`
+    );
+  }
+
+  if (ttsResult.status === 'fulfilled') {
+    licenses.push(...parseCsv(ttsResult.value));
+  } else {
+    console.warn(
+      `[SherpaOnnx] Failed to load TTS model licenses: ${ttsResult.reason}`
+    );
+  }
+
+  return licenses;
+}
+
+function parseCsv(csvString: string): ModelLicense[] {
+  const lines = csvString.split(/\r?\n/);
+  if (lines.length === 0) {
+    return [];
+  }
+
+  // The first line is the header
+  const headerLine = lines[0];
+  if (!headerLine) return [];
+
+  const headers = headerLine.split(',').map((h) => h.trim());
+
+  const results: ModelLicense[] = [];
+
+  for (let i = 1; i < lines.length; i++) {
+    const line = lines[i];
+    if (!line || line.trim() === '') continue;
+
+    // The CSV has 6 columns: asset_name, license_type, commercial_use,
+    // confidence, detection_source, license_file.  The last column
+    // (license_file) may itself contain commas (e.g. URLs with query strings),
+    // so split into at most 6 parts and join any excess back into the last field.
+    const COLUMN_COUNT = 6;
+    const rawParts = line.split(',');
+    const parts =
+      rawParts.length <= COLUMN_COUNT
+        ? rawParts
+        : [
+            ...rawParts.slice(0, COLUMN_COUNT - 1),
+            rawParts.slice(COLUMN_COUNT - 1).join(','),
+          ];
+
+    const entry: Record<string, string> = {};
+    for (let j = 0; j < headers.length; j++) {
+      const header = headers[j];
+      if (header) {
+        entry[header] = (parts[j] ?? '').trim();
+      }
+    }
+
+    if (entry['asset_name']) {
+      results.push(entry as unknown as ModelLicense);
+    }
+  }
+
+  return results;
+}