react-native-sherpa-onnx 0.3.6 → 0.3.8

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "react-native-sherpa-onnx",
-  "version": "0.3.6",
+  "version": "0.3.8",
   "description": "Offline Speech-to-text, text-to-speech, speaker diarization, speech enhancement, source separation, and VAD with sherpa-onnx for React NativeSpeech-to-Text with sherpa-onnx for React Native",
   "main": "./lib/module/index.js",
   "types": "./lib/typescript/src/index.d.ts",
@@ -58,6 +58,8 @@
     "./package.json": "./package.json"
   },
   "files": [
+    "THIRD_PARTY_LICENSES",
+    "THIRD_PARTY_LICENSES/**",
     "src",
     "lib",
     "android",
@@ -77,6 +79,8 @@
     "!ios/build",
     "!android/build",
     "!android/gradle",
+    "!android/**/build",
+    "!android/**/build/**",
     "!android/gradlew",
     "!android/gradlew.bat",
     "!android/local.properties",
@@ -162,6 +166,7 @@
     "typescript": "^5.9.2"
   },
   "dependencies": {
+    "@kesha-antonov/react-native-background-downloader": "^4.5.4",
     "buffer": "^6.0.3"
   },
   "peerDependencies": {

package/scripts/{check-model-csvs.sh → ci/check-model-csvs.sh}

@@ -10,6 +10,13 @@ REPO="${SHERPA_ONNX_REPO:-k2-fsa/sherpa-onnx}"
 ASR_CSV="${ASR_CSV:-test/fixtures/asr-models-expected.csv}"
 TTS_CSV="${TTS_CSV:-test/fixtures/tts-models-expected.csv}"
 
+# Optional: GITHUB_TOKEN or GH_TOKEN for api.github.com rate limits / private forks
+CURL_GH_API=(-sL)
+if [ -n "${GITHUB_TOKEN:-${GH_TOKEN:-}}" ]; then
+  _t="${GITHUB_TOKEN:-$GH_TOKEN}"
+  CURL_GH_API+=(-H "Authorization: Bearer ${_t}" -H "Accept: application/vnd.github+json")
+fi
+
 if [ ! -f "$ASR_CSV" ]; then
   echo "::warning::Missing $ASR_CSV (run from repo root or set ASR_CSV)"
   exit 0
@@ -21,7 +28,7 @@ fi
 
 # Fetch ASR release assets (.tar.bz2, .onnx)
 ASR_ASSETS=""
-ASR_RESP="${ASR_RESP:-$(curl -sL "https://api.github.com/repos/${REPO}/releases/tags/asr-models")}"
+ASR_RESP="${ASR_RESP:-$(curl "${CURL_GH_API[@]}" "https://api.github.com/repos/${REPO}/releases/tags/asr-models")}"
 if echo "$ASR_RESP" | jq -e '.assets' >/dev/null 2>&1; then
   ASR_ASSETS=$(echo "$ASR_RESP" | jq -r '.assets[] | select(.name | endswith(".tar.bz2") or endswith(".onnx")) | .name')
 else
@@ -30,7 +37,7 @@ fi
 
 # Fetch TTS release assets
 TTS_ASSETS=""
-TTS_RESP="${TTS_RESP:-$(curl -sL "https://api.github.com/repos/${REPO}/releases/tags/tts-models")}"
+TTS_RESP="${TTS_RESP:-$(curl "${CURL_GH_API[@]}" "https://api.github.com/repos/${REPO}/releases/tags/tts-models")}"
 if echo "$TTS_RESP" | jq -e '.assets' >/dev/null 2>&1; then
   TTS_ASSETS=$(echo "$TTS_RESP" | jq -r '.assets[] | select(.name | endswith(".tar.bz2") or endswith(".onnx")) | .name')
 else

package/scripts/ci/collect_all_sherpa_model_streams.sh

@@ -0,0 +1,101 @@
+#!/usr/bin/env bash
+# Run collect_one_sherpa_release_stream.sh for every entry in a streams JSON config.
+#
+# Usage:
+#   collect_all_sherpa_model_streams.sh [--config <file>]
+#   collect_all_sherpa_model_streams.sh [--config <file>] --print-git-paths
+#
+# Default config: scripts/ci/sherpa_asr_model_release_streams.json (ASR + QNN).
+# TTS: --config sherpa_tts_model_release_streams.json (relative to this dir or repo root, or absolute).
+#
+# --print-git-paths prints newline-separated paths to git-add (deduped), then exits.
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+CONFIG_REL="sherpa_asr_model_release_streams.json"
+PRINT_PATHS=0
+
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    --config)
+      CONFIG_REL="${2:-}"
+      [[ -n "$CONFIG_REL" ]] || { echo "--config requires a file path" >&2; exit 1; }
+      shift 2
+      ;;
+    --print-git-paths)
+      PRINT_PATHS=1
+      shift
+      ;;
+    *)
+      echo "Unknown option: $1 (use --config FILE and/or --print-git-paths)" >&2
+      exit 1
+      ;;
+  esac
+done
+
+resolve_config_path() {
+  local rel="$1"
+  if [[ "$rel" = /* ]]; then
+    echo "$rel"
+    return
+  fi
+  if [[ -f "$SCRIPT_DIR/$rel" ]]; then
+    echo "$SCRIPT_DIR/$rel"
+    return
+  fi
+  if [[ -f "$REPO_ROOT/$rel" ]]; then
+    echo "$REPO_ROOT/$rel"
+    return
+  fi
+  echo ""
+}
+
+CONFIG="$(resolve_config_path "$CONFIG_REL")"
+[[ -n "$CONFIG" && -f "$CONFIG" ]] || { echo "Config not found: $CONFIG_REL" >&2; exit 1; }
+
+if [[ "$PRINT_PATHS" -eq 1 ]]; then
+  {
+    jq -r '.streams[] | .structure_file, .expected_csv' "$CONFIG"
+    jq -r '.streams[] | .license_csv | strings' "$CONFIG"
+    jq -r '.streams[] | .license_csv | strings | select(startswith("android/src/main/assets/model_licenses/")) | ("ios/Resources/model_licenses/" + (split("/") | .[-1]))' "$CONFIG"
+  } | sort -u
+  exit 0
+fi
+
+if ! command -v jq >/dev/null 2>&1; then
+  echo "jq is required (install jq or use CI)." >&2
+  exit 1
+fi
+
+GITHUB_REPO="$(jq -r '.repo // "k2-fsa/sherpa-onnx"' "$CONFIG")"
+
+n="$(jq '.streams | length' "$CONFIG")"
+for ((i = 0; i < n; i++)); do
+  row="$(jq -c ".streams[$i]" "$CONFIG")"
+  tag="$(echo "$row" | jq -r '.release_tag')"
+  tree="$(echo "$row" | jq -r '.tree_cache_dir')"
+  struct="$(echo "$row" | jq -r '.structure_file')"
+  expected="$(echo "$row" | jq -r '.expected_csv')"
+  lic="$(echo "$row" | jq -r '.license_csv | strings')"
+  sid="$(echo "$row" | jq -r '.id // empty')"
+
+  args=(
+    "$SCRIPT_DIR/collect_one_sherpa_release_stream.sh"
+    --repo-root "$REPO_ROOT"
+    --github-repo "$GITHUB_REPO"
+    --release-tag "$tag"
+    --structure-file "$struct"
+    --expected-csv "$expected"
+    --tree-cache-dir "$tree"
+  )
+  if [[ -n "$lic" ]]; then
+    args+=(--license-csv "$lic")
+  fi
+  if [[ -n "$sid" ]]; then
+    args+=(--stream-id "$sid")
+  fi
+  bash "${args[@]}"
+done
+
+echo "=== All streams processed ($n) config=$(basename "$CONFIG") ==="

package/scripts/ci/collect_one_sherpa_release_stream.sh

@@ -0,0 +1,189 @@
+#!/usr/bin/env bash
+# Fetch one GitHub release from k2-fsa/sherpa-onnx (or --github-repo), refresh tree listings,
+# aggregate structure + expected CSV, optionally run update_model_license_csv.sh (in this directory).
+# Paths are relative to repository root (--repo-root).
+set -euo pipefail
+
+GITHUB_REPO="k2-fsa/sherpa-onnx"
+REPO_ROOT=""
+RELEASE_TAG=""
+STRUCTURE_FILE=""
+EXPECTED_CSV=""
+TREE_CACHE_DIR=""
+LICENSE_CSV=""
+STREAM_ID=""
+
+usage() {
+  echo "Usage: $0 --repo-root <dir> --release-tag <tag> --structure-file <rel> --expected-csv <rel> \\"
+  echo "         --tree-cache-dir <rel> [--github-repo owner/name] [--license-csv <rel>] [--stream-id <id>]"
+  exit 1
+}
+
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    --repo-root) REPO_ROOT="$2"; shift 2 ;;
+    --github-repo) GITHUB_REPO="$2"; shift 2 ;;
+    --release-tag) RELEASE_TAG="$2"; shift 2 ;;
+    --structure-file) STRUCTURE_FILE="$2"; shift 2 ;;
+    --expected-csv) EXPECTED_CSV="$2"; shift 2 ;;
+    --tree-cache-dir) TREE_CACHE_DIR="$2"; shift 2 ;;
+    --license-csv) LICENSE_CSV="$2"; shift 2 ;;
+    --stream-id) STREAM_ID="$2"; shift 2 ;;
+    *) echo "Unknown option: $1"; usage ;;
+  esac
+done
+
+[[ -n "$REPO_ROOT" && -n "$RELEASE_TAG" && -n "$STRUCTURE_FILE" && -n "$EXPECTED_CSV" && -n "$TREE_CACHE_DIR" ]] || usage
+
+REPO_ROOT="$(cd "$REPO_ROOT" && pwd)"
+cd "$REPO_ROOT"
+
+CI_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+_abs_tree="$REPO_ROOT/$TREE_CACHE_DIR"
+_abs_structure="$REPO_ROOT/$STRUCTURE_FILE"
+_abs_expected="$REPO_ROOT/$EXPECTED_CSV"
+
+WORK="$(mktemp -d -t sherpa-collect-XXXXXX)"
+trap 'rm -rf "$WORK"' EXIT
+ASSET_LIST="$WORK/asset-list.txt"
+
+_GH_TOKEN="${GITHUB_TOKEN:-${GH_TOKEN:-}}"
+_ASSET_LIMIT="${ASSET_LIMIT:-}"
+
+echo "=== collect_one_sherpa_release_stream: tag=$RELEASE_TAG tree-cache=$TREE_CACHE_DIR ==="
+
+API_AUTH=()
+if [[ -n "$_GH_TOKEN" ]]; then
+  API_AUTH=(-H "Authorization: Bearer ${_GH_TOKEN}" -H "Accept: application/vnd.github+json")
+fi
+
+RESP="$(curl -sSL "${API_AUTH[@]}" "https://api.github.com/repos/${GITHUB_REPO}/releases/tags/${RELEASE_TAG}")"
+if echo "$RESP" | jq -e '.assets' >/dev/null 2>&1; then
+  LIST="$(echo "$RESP" | jq -r '.assets[] | select(.name | endswith(".tar.bz2") or endswith(".onnx")) | "\(.name)|\(.browser_download_url)"')"
+  if [[ -z "$_ASSET_LIMIT" || "$_ASSET_LIMIT" == "0" ]]; then
+    printf '%s\n' "$LIST" > "$ASSET_LIST"
+    echo "  Asset list: no limit ($(wc -l < "$ASSET_LIST" | tr -d ' ') lines)"
+  else
+    printf '%s\n' "$LIST" | head -n "$_ASSET_LIMIT" > "$ASSET_LIST"
+    echo "  Asset list: limit $_ASSET_LIMIT ($(wc -l < "$ASSET_LIST" | tr -d ' ') lines)"
+  fi
+else
+  echo "::warning::Release ${RELEASE_TAG} not found or has no assets (${GITHUB_REPO})" >&2
+  : > "$ASSET_LIST"
+fi
+
+mkdir -p "$_abs_tree"
+
+if [[ -f "$_abs_structure" ]]; then
+  cur=""
+  safe=""
+  while IFS= read -r line || [[ -n "$line" ]]; do
+    line="${line%$'\r'}"
+    if [[ "$line" =~ ^#\ Asset:\ (.+)$ ]]; then
+      cur="${BASH_REMATCH[1]}"
+      safe="${cur//\//-}"
+      safe="${safe//\\/-}"
+      : > "${_abs_tree}/${safe}.txt"
+    elif [[ -n "$cur" ]]; then
+      printf '%s\n' "$line" >> "${_abs_tree}/${safe}.txt"
+    fi
+  done < "$_abs_structure"
+  echo "  Parsed existing structure into ${_abs_tree}"
+else
+  echo "  No existing structure file; tree-cache will be filled from downloads only"
+fi
+
+mkdir -p "$WORK/dl"
+while IFS='|' read -r name url; do
+  [[ -z "$name" ]] && continue
+  name="${name%$'\r'}"
+  url="${url%$'\r'}"
+  safe="${name//\//-}"
+  safe="${safe//\\/-}"
+  cache_file="${_abs_tree}/${safe}.txt"
+  if [[ -f "$cache_file" ]]; then
+    echo "  Skip (cache hit): $name"
+    continue
+  fi
+  echo "  Download: $name"
+  dl="$WORK/dl/$safe"
+  DL_ARGS=(-sSL)
+  if [[ -n "$_GH_TOKEN" && "$url" == *"github.com"* ]]; then
+    DL_ARGS+=(-H "Authorization: Bearer ${_GH_TOKEN}" -H "Accept: application/octet-stream")
+  fi
+  if ! curl "${DL_ARGS[@]}" -o "$dl" "$url"; then
+    echo "::warning::Download failed for $name" >&2
+    rm -f "$dl"
+    continue
+  fi
+  if [[ "$name" == *.tar.bz2 ]]; then
+    if ! tar -tjf "$dl" > "$cache_file" 2>/dev/null; then
+      echo "::warning::tar -tjf failed for $name" >&2
+      rm -f "$cache_file" "$dl"
+      continue
+    fi
+  elif [[ "$name" == *.onnx ]]; then
+    echo "single file: $name" > "$cache_file"
+  else
+    echo "::warning::Unexpected asset $name" >&2
+    rm -f "$dl"
+    continue
+  fi
+  rm -f "$dl"
+done < "$ASSET_LIST"
+
+mkdir -p "$(dirname "$_abs_structure")"
+: > "$_abs_structure"
+while IFS='|' read -r name _; do
+  [[ -z "$name" ]] && continue
+  name="${name%$'\r'}"
+  safe="${name//\//-}"
+  safe="${safe//\\/-}"
+  cache_file="${_abs_tree}/${safe}.txt"
+  [[ -f "$cache_file" ]] || continue
+  echo "# Asset: $name" >> "$_abs_structure"
+  cat "$cache_file" >> "$_abs_structure"
+done < "$ASSET_LIST"
+echo "  Wrote aggregated structure ($(wc -l < "$_abs_structure" | tr -d ' ') lines)"
+
+if [[ ! -s "$ASSET_LIST" ]]; then
+  echo "  No assets; skipping expected CSV and license update"
+  exit 0
+fi
+
+mkdir -p "$(dirname "$_abs_expected")"
+if [[ ! -f "$_abs_expected" ]]; then
+  echo "asset_name,model_type" > "$_abs_expected"
+fi
+tmp="$(mktemp)"
+echo "asset_name,model_type" > "$tmp"
+while IFS='|' read -r asset _; do
+  [[ -z "$asset" ]] && continue
+  asset="${asset%$'\r'}"
+  esc="${asset//./\\.}"
+  line="$(grep -E "^(\"${esc}\"|${esc})," "$_abs_expected" 2>/dev/null | head -1 || true)"
+  if [[ -n "$line" ]]; then
+    echo "$line" >> "$tmp"
+  else
+    echo "${asset}," >> "$tmp"
+  fi
+done < "$ASSET_LIST"
+mv "$tmp" "$_abs_expected"
+echo "  Expected CSV rows: $(($(wc -l < "$_abs_expected" | tr -d ' ') - 1)) data (+ header)"
+
+if [[ -n "$LICENSE_CSV" ]]; then
+  _abs_license="$REPO_ROOT/$LICENSE_CSV"
+  echo "  License CSV: $LICENSE_CSV"
+  _lic_args=(
+    --asset-list "$ASSET_LIST"
+    --tree-cache-dir "$_abs_tree"
+    --csv "$_abs_license"
+  )
+  if [[ -n "$STREAM_ID" ]]; then
+    _lic_args+=(--stream-id "$STREAM_ID")
+  fi
+  bash "$CI_DIR/update_model_license_csv.sh" "${_lic_args[@]}"
+fi
+
+echo "=== done: $RELEASE_TAG ==="

package/scripts/ci/sherpa_asr_model_release_streams.json

@@ -0,0 +1,21 @@
+{
+  "repo": "k2-fsa/sherpa-onnx",
+  "streams": [
+    {
+      "id": "asr-models",
+      "release_tag": "asr-models",
+      "tree_cache_dir": "tree-cache/asr-models",
+      "structure_file": "test/fixtures/asr-models-structure.txt",
+      "expected_csv": "test/fixtures/asr-models-expected.csv",
+      "license_csv": "android/src/main/assets/model_licenses/asr-models-license-status.csv"
+    },
+    {
+      "id": "asr-models-qnn-binary",
+      "release_tag": "asr-models-qnn-binary",
+      "tree_cache_dir": "tree-cache/asr-models-qnn-binary",
+      "structure_file": "test/fixtures/qnn-asr-models-structure.txt",
+      "expected_csv": "test/fixtures/qnn-asr-models-expected.csv",
+      "license_csv": "android/src/main/assets/model_licenses/qnn-asr-models-license-status.csv"
+    }
+  ]
+}

package/scripts/ci/sherpa_tts_model_release_streams.json

@@ -0,0 +1,13 @@
+{
+  "repo": "k2-fsa/sherpa-onnx",
+  "streams": [
+    {
+      "id": "tts-models",
+      "release_tag": "tts-models",
+      "tree_cache_dir": "tree-cache/tts-models",
+      "structure_file": "test/fixtures/tts-models-structure.txt",
+      "expected_csv": "test/fixtures/tts-models-expected.csv",
+      "license_csv": "android/src/main/assets/model_licenses/tts-models-license-status.csv"
+    }
+  ]
+}