react-native-sherpa-onnx 0.3.5 → 0.3.7

package/src/download/localModels.ts

@@ -0,0 +1,234 @@
+import { Platform } from 'react-native';
+import {
+  DocumentDirectoryPath,
+  exists,
+  readFile,
+  readDir,
+  unlink,
+  writeFile,
+} from '@dr.pogodin/react-native-fs';
+import type {
+  ModelCategory,
+  ModelMetaBase,
+  ModelManifest,
+  ModelWithMetadata,
+} from './types';
+import {
+  getCachePath,
+  getModelsBaseDir,
+  getModelDir,
+  getManifestPath,
+  getNativeAssetExtractedModelDir,
+  getReadyMarkerPath,
+  getTarArchivePath,
+  getOnnxPath,
+} from './paths';
+import { resolveActualModelDir, removeDirectoryRecursive } from './validation';
+import { emitModelsListUpdated } from './downloadEvents';
+import { clearMemoryCacheForCategory } from './registry';
+
+export async function listDownloadedModelsByCategory<T extends ModelMetaBase>(
+  category: ModelCategory
+): Promise<T[]> {
+  const baseDir = getModelsBaseDir(category);
+  const existsResult = await exists(baseDir);
+  if (!existsResult) return [];
+
+  const entries = await readDir(baseDir);
+  const models: T[] = [];
+
+  for (const entry of entries) {
+    if (!entry.isDirectory()) continue;
+    const manifestPath = getManifestPath(category, entry.name);
+    const manifestExists = await exists(manifestPath);
+    if (!manifestExists) continue;
+    try {
+      const raw = await readFile(manifestPath, 'utf8');
+      const manifest = JSON.parse(raw) as ModelManifest<T>;
+      if (manifest.model) {
+        models.push(manifest.model);
+      }
+    } catch {
+      // ignore invalid manifest
+    }
+  }
+
+  return models;
+}
+
+export async function isModelDownloadedByCategory(
+  category: ModelCategory,
+  id: string
+): Promise<boolean> {
+  const readyPath = getReadyMarkerPath(category, id);
+  return exists(readyPath);
+}
+
+export async function getLocalModelPathByCategory(
+  category: ModelCategory,
+  id: string
+): Promise<string | null> {
+  const ready = await isModelDownloadedByCategory(category, id);
+  if (!ready) return null;
+
+  await updateModelLastUsed(category, id);
+
+  const installDir = getModelDir(category, id);
+  return resolveActualModelDir(installDir);
+}
+
+export async function updateModelLastUsed(
+  category: ModelCategory,
+  id: string
+): Promise<void> {
+  const manifestPath = getManifestPath(category, id);
+  const existsResult = await exists(manifestPath);
+  if (!existsResult) return;
+
+  try {
+    const raw = await readFile(manifestPath, 'utf8');
+    const manifest = JSON.parse(raw) as ModelManifest;
+    manifest.lastUsed = new Date().toISOString();
+    await writeFile(manifestPath, JSON.stringify(manifest), 'utf8');
+  } catch (error) {
+    console.warn(`Failed to update lastUsed for ${category}:${id}:`, error);
+  }
+}
+
+export async function listDownloadedModelsWithMetadata<T extends ModelMetaBase>(
+  category: ModelCategory
+): Promise<ModelWithMetadata<T>[]> {
+  const baseDir = getModelsBaseDir(category);
+  const existsResult = await exists(baseDir);
+  if (!existsResult) return [];
+
+  const entries = await readDir(baseDir);
+  const modelsWithMetadata: ModelWithMetadata<T>[] = [];
+
+  for (const entry of entries) {
+    if (!entry.isDirectory()) continue;
+
+    const manifestPath = getManifestPath(category, entry.name);
+    const manifestExists = await exists(manifestPath);
+
+    if (manifestExists) {
+      try {
+        const raw = await readFile(manifestPath, 'utf8');
+        const manifest = JSON.parse(raw) as ModelManifest<T>;
+        if (manifest.model) {
+          modelsWithMetadata.push({
+            model: manifest.model,
+            downloadedAt: manifest.downloadedAt,
+            lastUsed: manifest.lastUsed ?? null,
+            sizeOnDisk: manifest.sizeOnDisk ?? entry.size,
+            status: 'ready',
+          });
+        }
+      } catch (error) {
+        console.warn(
+          `Failed to read manifest for ${category}:${entry.name}:`,
+          error
+        );
+      }
+    }
+  }
+
+  return modelsWithMetadata;
+}
+
+export async function cleanupLeastRecentlyUsed(
+  category: ModelCategory,
+  options?: {
+    targetBytes?: number;
+    maxModelsToDelete?: number;
+    keepCount?: number;
+  }
+): Promise<string[]> {
+  const modelsWithMetadata = await listDownloadedModelsWithMetadata(category);
+
+  if (modelsWithMetadata.length === 0) {
+    return [];
+  }
+
+  const keepCount = options?.keepCount ?? 1;
+  if (modelsWithMetadata.length <= keepCount) {
+    return [];
+  }
+
+  const sorted = modelsWithMetadata.sort((a, b) => {
+    const aTime = a.lastUsed ?? a.downloadedAt;
+    const bTime = b.lastUsed ?? b.downloadedAt;
+    return new Date(aTime).getTime() - new Date(bTime).getTime();
+  });
+
+  const deletedIds: string[] = [];
+  let bytesFreed = 0;
+  const targetBytes = options?.targetBytes ?? 0;
+  const maxToDelete = options?.maxModelsToDelete ?? sorted.length - keepCount;
+
+  for (let i = 0; i < sorted.length - keepCount && i < maxToDelete; i++) {
+    const item = sorted[i];
+    if (!item) continue;
+
+    try {
+      await deleteModelByCategory(category, item.model.id);
+      deletedIds.push(item.model.id);
+      bytesFreed += item.sizeOnDisk ?? 0;
+
+      console.log(
+        `[LRU Cleanup] Deleted ${category}:${item.model.id} (freed ${
+          (item.sizeOnDisk ?? 0) / 1024 / 1024
+        } MB)`
+      );
+
+      if (targetBytes > 0 && bytesFreed >= targetBytes) {
+        break;
+      }
+    } catch (error) {
+      console.warn(
+        `[LRU Cleanup] Failed to delete ${category}:${item.model.id}:`,
+        error
+      );
+    }
+  }
+
+  return deletedIds;
+}
+
+export async function deleteModelByCategory(
+  category: ModelCategory,
+  id: string
+): Promise<void> {
+  const modelDir = getModelDir(category, id);
+  const tarPath = getTarArchivePath(category, id);
+  const onnxPath = getOnnxPath(category, id);
+  if (await exists(modelDir)) {
+    await unlink(modelDir);
+  }
+  if (await exists(tarPath)) {
+    await unlink(tarPath);
+  }
+  if (await exists(onnxPath)) {
+    await unlink(onnxPath);
+  }
+  await removeDirectoryRecursive(getNativeAssetExtractedModelDir(id));
+  const list = await listDownloadedModelsByCategory<ModelMetaBase>(category);
+  emitModelsListUpdated(category, list);
+}
+
+export async function clearModelCacheByCategory(
+  category: ModelCategory
+): Promise<void> {
+  const cachePath = getCachePath(category);
+  if (await exists(cachePath)) {
+    await unlink(cachePath);
+  }
+  clearMemoryCacheForCategory(category);
+}
+
+export async function getDownloadStorageBase(): Promise<string> {
+  if (Platform.OS === 'ios') {
+    return DocumentDirectoryPath;
+  }
+  return DocumentDirectoryPath;
+}

package/src/download/modelExtraction.ts

@@ -0,0 +1,244 @@
+import {
+  exists,
+  readFile,
+  readDir,
+  mkdir,
+  stat,
+  unlink,
+} from '@dr.pogodin/react-native-fs';
+import type {
+  ModelCategory,
+  ModelMetaBase,
+  ChecksumIssue,
+  DownloadProgress,
+  ExtractionState,
+} from './types';
+import type { DownloadResult } from './types';
+import {
+  getModelsBaseDir,
+  getModelDir,
+  getArchivePath,
+  getReadyMarkerPath,
+  getExtractionStatePath,
+  getNativeAssetExtractedModelDir,
+} from './paths';
+import { runPostDownloadProcessing } from './postDownloadProcessing';
+import { getModelByIdByCategory } from './registry';
+import {
+  listDownloadedModelsByCategory,
+  getLocalModelPathByCategory,
+} from './localModels';
+import { resolveActualModelDir, removeDirectoryRecursive } from './validation';
+
+const EXTRACTION_STATE_PREFIX = '.extraction-state-';
+const EXTRACTION_STATE_SUFFIX = '.json';
+
+/**
+ * Start extraction for a model: archive must already exist (e.g. after download or from PAD).
+ * Writes extraction state so that if the app crashes, extraction can be resumed via
+ * getIncompleteExtractions + resumeExtraction.
+ * Use signal to abort (pause) extraction.
+ */
+export async function extractModelByCategory<T extends ModelMetaBase>(
+  category: ModelCategory,
+  id: string,
+  opts?: {
+    onProgress?: (progress: DownloadProgress) => void;
+    signal?: AbortSignal;
+    onChecksumIssue?: (issue: ChecksumIssue) => Promise<boolean>;
+    deleteArchiveAfterExtract?: boolean;
+  }
+): Promise<DownloadResult> {
+  const model = await getModelByIdByCategory<T>(category, id);
+  if (!model) {
+    throw new Error(`Unknown model id: ${id}`);
+  }
+  if (model.archiveExt !== 'tar.bz2') {
+    throw new Error(
+      `Model ${id} is not a tar.bz2 archive; extraction is only for archived models.`
+    );
+  }
+
+  const downloadPath = getArchivePath(category, id, model.archiveExt);
+  const modelDir = getModelDir(category, id);
+  const statePath = getExtractionStatePath(category, id);
+
+  const archiveExists = await exists(downloadPath);
+  if (!archiveExists) {
+    throw new Error(
+      `Archive not found at ${downloadPath}. Download the model first or ensure the archive is present.`
+    );
+  }
+  try {
+    const archiveStat = await stat(downloadPath);
+    if (model.bytes > 0 && archiveStat.size < model.bytes) {
+      throw new Error(
+        `Archive is truncated (${archiveStat.size}/${model.bytes} bytes). Re-download or replace the file.`
+      );
+    }
+  } catch (statErr) {
+    if (statErr instanceof Error) throw statErr;
+    throw new Error('Failed to read archive size.');
+  }
+
+  await mkdir(getModelsBaseDir(category));
+
+  return runPostDownloadProcessing({
+    category,
+    id,
+    model,
+    downloadPath,
+    modelDir,
+    isArchive: true,
+    statePath,
+    signal: opts?.signal,
+    onChecksumIssue: opts?.onChecksumIssue,
+    deleteArchiveAfterExtract: opts?.deleteArchiveAfterExtract,
+    onProgress: opts?.onProgress,
+    getDownloadedList: () =>
+      listDownloadedModelsByCategory<ModelMetaBase>(category),
+  });
+}
+
+/**
+ * Returns models in the given category that have incomplete extractions (e.g. after app
+ * crash during extraction). Use with resumeExtraction to continue.
+ */
+export async function getIncompleteExtractions(
+  category: ModelCategory
+): Promise<ExtractionState[]> {
+  const baseDir = getModelsBaseDir(category);
+  const baseExists = await exists(baseDir);
+  if (!baseExists) return [];
+
+  let entries: { name: string; isDirectory: () => boolean }[];
+  try {
+    entries = await readDir(baseDir);
+  } catch {
+    return [];
+  }
+
+  const results: ExtractionState[] = [];
+  for (const entry of entries) {
+    const name = entry.name;
+    if (
+      !name.startsWith(EXTRACTION_STATE_PREFIX) ||
+      !name.endsWith(EXTRACTION_STATE_SUFFIX)
+    ) {
+      continue;
+    }
+    const modelId = name.slice(
+      EXTRACTION_STATE_PREFIX.length,
+      name.length - EXTRACTION_STATE_SUFFIX.length
+    );
+    const statePath = getExtractionStatePath(category, modelId);
+    let state: ExtractionState;
+    try {
+      const raw = await readFile(statePath, 'utf8');
+      state = JSON.parse(raw) as ExtractionState;
+    } catch {
+      continue;
+    }
+    const readyPath = getReadyMarkerPath(category, modelId);
+    if (await exists(readyPath)) continue;
+    try {
+      const archiveExists = await exists(state.archivePath);
+      if (!archiveExists) continue;
+      const st = await stat(state.archivePath);
+      if (
+        state.model.bytes > 0 &&
+        st.size != null &&
+        st.size < state.model.bytes
+      ) {
+        continue;
+      }
+    } catch {
+      continue;
+    }
+    results.push(state);
+  }
+  return results;
+}
+
+/**
+ * Resume an incomplete extraction (e.g. after app restart). Use getIncompleteExtractions
+ * to discover items to resume. Runs extraction from the start (archive is overwritten into
+ * model dir with force).
+ */
+export async function resumeExtraction<T extends ModelMetaBase>(
+  category: ModelCategory,
+  id: string,
+  opts?: {
+    onProgress?: (progress: DownloadProgress) => void;
+    signal?: AbortSignal;
+    onChecksumIssue?: (issue: ChecksumIssue) => Promise<boolean>;
+    deleteArchiveAfterExtract?: boolean;
+  }
+): Promise<DownloadResult> {
+  const statePath = getExtractionStatePath(category, id);
+  const stateExists = await exists(statePath);
+  if (!stateExists) {
+    return extractModelByCategory<T>(category, id, opts);
+  }
+  let state: ExtractionState;
+  try {
+    const raw = await readFile(statePath, 'utf8');
+    state = JSON.parse(raw) as ExtractionState;
+  } catch {
+    return extractModelByCategory<T>(category, id, opts);
+  }
+  if (state.modelId !== id || state.category !== category) {
+    return extractModelByCategory<T>(category, id, opts);
+  }
+  const readyPath = getReadyMarkerPath(category, id);
+  if (await exists(readyPath)) {
+    try {
+      await unlink(statePath);
+    } catch {
+      // non-fatal
+    }
+    const localPath =
+      (await getLocalModelPathByCategory(category, id)) ??
+      (await resolveActualModelDir(state.modelDir));
+    return { modelId: id, localPath };
+  }
+
+  return runPostDownloadProcessing({
+    category: state.category,
+    id: state.modelId,
+    model: state.model,
+    downloadPath: state.archivePath,
+    modelDir: state.modelDir,
+    isArchive: true,
+    statePath,
+    signal: opts?.signal,
+    onChecksumIssue: opts?.onChecksumIssue,
+    deleteArchiveAfterExtract: opts?.deleteArchiveAfterExtract,
+    onProgress: opts?.onProgress,
+    getDownloadedList: () =>
+      listDownloadedModelsByCategory<ModelMetaBase>(category),
+  });
+}
+
+/**
+ * Cancel/delete an incomplete extraction: removes extraction state and partial model dir.
+ * Does not delete the archive so the user can retry extraction later.
+ */
+export async function deleteIncompleteExtraction(
+  category: ModelCategory,
+  id: string
+): Promise<void> {
+  const statePath = getExtractionStatePath(category, id);
+  try {
+    if (await exists(statePath)) await unlink(statePath);
+  } catch {
+    // non-fatal
+  }
+  const modelDir = getModelDir(category, id);
+  try {
+    if (await exists(modelDir)) await unlink(modelDir);
+  } catch {
+    // non-fatal
+  }
+  await removeDirectoryRecursive(getNativeAssetExtractedModelDir(id));
+}

package/src/download/paths.ts

@@ -0,0 +1,134 @@
+import { DocumentDirectoryPath } from '@dr.pogodin/react-native-fs';
+import { ModelCategory } from './types';
+import type { ModelArchiveExt } from './types';
+import { RELEASE_API_BASE } from './constants';
+
+export const CATEGORY_CONFIG: Record<
+  ModelCategory,
+  { tag: string; cacheFile: string; baseDir: string }
+> = {
+  [ModelCategory.Tts]: {
+    tag: 'tts-models',
+    cacheFile: 'tts-models.json',
+    baseDir: `${DocumentDirectoryPath}/sherpa-onnx/models/tts`,
+  },
+  [ModelCategory.Stt]: {
+    tag: 'asr-models',
+    cacheFile: 'asr-models.json',
+    baseDir: `${DocumentDirectoryPath}/sherpa-onnx/models/stt`,
+  },
+  [ModelCategory.Vad]: {
+    tag: 'asr-models',
+    cacheFile: 'vad-models.json',
+    baseDir: `${DocumentDirectoryPath}/sherpa-onnx/models/vad`,
+  },
+  [ModelCategory.Diarization]: {
+    tag: 'speaker-segmentation-models',
+    cacheFile: 'diarization-models.json',
+    baseDir: `${DocumentDirectoryPath}/sherpa-onnx/models/diarization`,
+  },
+  [ModelCategory.Enhancement]: {
+    tag: 'speech-enhancement-models',
+    cacheFile: 'enhancement-models.json',
+    baseDir: `${DocumentDirectoryPath}/sherpa-onnx/models/enhancement`,
+  },
+  [ModelCategory.Separation]: {
+    tag: 'source-separation-models',
+    cacheFile: 'separation-models.json',
+    baseDir: `${DocumentDirectoryPath}/sherpa-onnx/models/separation`,
+  },
+  [ModelCategory.Qnn]: {
+    tag: 'asr-models-qnn-binary',
+    cacheFile: 'qnn-models.json',
+    baseDir: `${DocumentDirectoryPath}/sherpa-onnx/models/qnn`,
+  },
+};
+
+export function getCacheDir(): string {
+  return `${DocumentDirectoryPath}/sherpa-onnx/cache`;
+}
+
+export function getCachePath(category: ModelCategory): string {
+  return `${getCacheDir()}/${CATEGORY_CONFIG[category].cacheFile}`;
+}
+
+export function getModelsBaseDir(category: ModelCategory): string {
+  return CATEGORY_CONFIG[category].baseDir;
+}
+
+export function getModelDir(category: ModelCategory, modelId: string): string {
+  return `${getModelsBaseDir(category)}/${modelId}`;
+}
+
+export function getArchiveFilename(
+  modelId: string,
+  archiveExt: ModelArchiveExt
+): string {
+  return `${modelId}.${archiveExt}`;
+}
+
+export function getArchivePath(
+  category: ModelCategory,
+  modelId: string,
+  archiveExt: ModelArchiveExt
+): string {
+  const filename = getArchiveFilename(modelId, archiveExt);
+  if (archiveExt === 'onnx') {
+    return `${getModelDir(category, modelId)}/${filename}`;
+  }
+  return `${getModelsBaseDir(category)}/${filename}`;
+}
+
+export function getTarArchivePath(
+  category: ModelCategory,
+  modelId: string
+): string {
+  return getArchivePath(category, modelId, 'tar.bz2');
+}
+
+export function getOnnxPath(category: ModelCategory, modelId: string): string {
+  return getArchivePath(category, modelId, 'onnx');
+}
+
+export function getReadyMarkerPath(
+  category: ModelCategory,
+  modelId: string
+): string {
+  return `${getModelDir(category, modelId)}/.ready`;
+}
+
+export function getManifestPath(
+  category: ModelCategory,
+  modelId: string
+): string {
+  return `${getModelDir(category, modelId)}/manifest.json`;
+}
+
+export function getDownloadStatePath(
+  category: ModelCategory,
+  modelId: string
+): string {
+  return `${getModelsBaseDir(category)}/.download-state-${modelId}.json`;
+}
+
+/** Path to extraction state file; used to detect and resume incomplete extractions after app restart. */
+export function getExtractionStatePath(
+  category: ModelCategory,
+  modelId: string
+): string {
+  return `${getModelsBaseDir(category)}/.extraction-state-${modelId}.json`;
+}
+
+/**
+ * Directory where native `resolveAssetPath` materializes a bundled model folder
+ * (`DocumentDirectoryPath/models/{modelId}` — Android internal `files/models/...`).
+ * Separate from {@link getModelDir}. Remove on delete so empty dirs do not break detection.
+ */
+export function getNativeAssetExtractedModelDir(modelId: string): string {
+  const safeId = modelId.replace(/[/\\]/g, '');
+  return `${DocumentDirectoryPath}/models/${safeId}`.replace(/\/+/g, '/');
+}
+
+export function getReleaseUrl(category: ModelCategory): string {
+  return `${RELEASE_API_BASE}/${CATEGORY_CONFIG[category].tag}`;
+}