parakeet.js 0.0.2 → 0.0.3

package/docs/parakeet-transformers-js/src/utils/video.js

@@ -1,128 +0,0 @@
-import { RawImage } from "./image.js";
-import { apis } from "../env.js";
-
-export class RawVideoFrame {
-
-    /**
-     * @param {RawImage} image
-     * @param {number} timestamp
-     */
-    constructor(image, timestamp) {
-        this.image = image;
-        this.timestamp = timestamp;
-    }
-}
-
-export class RawVideo {
-    /**
-     * @param {RawVideoFrame[]|RawImage[]} frames
-     * @param {number} duration
-     */
-    constructor(frames, duration) {
-        if (frames.length > 0 && frames[0] instanceof RawImage) {
-            // Assume uniform timestamps
-            frames = frames.map((image, i) => new RawVideoFrame(image, (i + 1) / (frames.length + 1) * duration));
-        }
-        this.frames = /** @type {RawVideoFrame[]} */ (frames);
-        this.duration = duration;
-    }
-
-    get width() {
-        return this.frames[0].image.width;
-    }
-    get height() {
-        return this.frames[0].image.height;
-    }
-
-    get fps() {
-        return this.frames.length / this.duration;
-    }
-}
-
-
-/**
- * Loads a video.
- *
- * @param {string|Blob|HTMLVideoElement} src The video to process.
- * @param {Object} [options] Optional parameters.
- * @param {number} [options.num_frames=null] The number of frames to sample uniformly.
- * @param {number} [options.fps=null] The number of frames to sample per second.
- *
- * @returns {Promise<RawVideo>} The loaded video.
- */
-export async function load_video(src, { num_frames = null, fps = null } = {}) {
-    if (!apis.IS_BROWSER_ENV) {
-        throw new Error("`load_video` is currently only supported in browser environments.");
-    }
-
-    // TODO: Support efficiently loading all frames using the WebCodecs API.
-    // Specfically, https://developer.mozilla.org/en-US/docs/Web/API/VideoDecoder
-    if (num_frames == null && fps == null) {
-        throw new Error("Either num_frames or fps must be provided.");
-    }
-
-    const frames = [];
-
-    const video = document.createElement("video");
-    video.crossOrigin = "anonymous";
-    video.muted = true; // mute to allow autoplay and seeking
-
-    if (typeof src === 'string') {
-        video.src = src;
-    } else if (src instanceof Blob) {
-        video.src = URL.createObjectURL(src);
-    } else if (src instanceof HTMLVideoElement) {
-        video.src = src.src;
-    } else {
-        throw new Error("Invalid URL or video element provided.");
-    }
-    // Wait for metadata to load to obtain duration
-    await new Promise((resolve) => video.onloadedmetadata = resolve);
-
-    if (video.seekable.start(0) === video.seekable.end(0)) {
-        // Fallback: Download entire video if not seekable
-        const response = await fetch(video.src);
-        const blob = await response.blob();
-        video.src = URL.createObjectURL(blob);
-        await new Promise((resolve) => video.onloadedmetadata = resolve);
-    }
-
-    const duration = video.duration;
-
-    let count, step;
-    if (num_frames != null) {
-        count = num_frames;
-        step = num_frames === 1 ? 0 : duration / (num_frames - 1);
-    } else {
-        step = 1 / fps;
-        count = Math.floor(duration / step);
-    }
-
-    // Build an array of sample times based on num_frames or fps
-    let sampleTimes = [];
-    for (let i = 0; i < count; ++i) {
-        sampleTimes.push(num_frames === 1 ? duration / 2 : i * step);
-    }
-
-    const canvas = document.createElement("canvas");
-    canvas.width = video.videoWidth;
-    canvas.height = video.videoHeight;
-    const ctx = canvas.getContext("2d", { willReadFrequently: true });
-    for (const t of sampleTimes) {
-        video.currentTime = t;
-        await new Promise((resolve) => {
-            video.onseeked = resolve;
-        });
-        ctx.drawImage(video, 0, 0, canvas.width, canvas.height);
-        const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height);
-        const frameData = new RawImage(imageData.data, canvas.width, canvas.height, 4);
-
-        const frame = new RawVideoFrame(frameData, t);
-        frames.push(frame);
-    }
-
-    // Clean up video element.
-    video.remove();
-
-    return new RawVideo(frames, duration);
-}

package/docs/parakeet-transformers-js/test/decoder.test.js

@@ -1,114 +0,0 @@
-import fs from 'fs';
-import path from 'path';
-import { spawnSync } from 'child_process';
-import { ParakeetModel } from '../src/parakeet.js';
-import { MEL_BINS } from '../src/audio_features.js';
-import { fileURLToPath } from 'url';
-
-const CHUNK = 128;
-const BLANK_ID = 1024;
-
-const __dirname = path.dirname(fileURLToPath(import.meta.url));
-
-test('Decoder joint first step logits match python reference (tolerance 1e-3)', async () => {
-  const projectRoot = path.resolve(__dirname, '..');
-  const assetsDir = path.join(projectRoot, 'assets');
-  const preprocPath = path.join(assetsDir, 'nemo128.onnx');
-  if (!fs.existsSync(preprocPath)) {
-    console.warn('Preprocessor not found, skipping');
-    return;
-  }
-
-  const modelDir = path.resolve(projectRoot, '..', 'parakeet-tdt-0.6b-v2-onnx');
-  let encoderPath = path.join(modelDir, 'encoder-model.int8.onnx');
-  if (!fs.existsSync(encoderPath)) encoderPath = path.join(modelDir, 'encoder-model.onnx');
-  let decJointPath = path.join(modelDir, 'decoder_joint-model.int8.onnx');
-  if (!fs.existsSync(decJointPath)) decJointPath = path.join(modelDir, 'decoder_joint-model.onnx');
-  if (!fs.existsSync(encoderPath) || !fs.existsSync(decJointPath)) {
-    console.warn('Model files not found, skipping');
-    return;
-  }
-
-  const wavPath = path.resolve(projectRoot, '..', 'jfk.wav');
-  if (!fs.existsSync(wavPath)) {
-    console.warn('WAV not found, skipping');
-    return;
-  }
-
-  const tmpJson = path.join(projectRoot, '.tmp_dec.json');
-  const pyScript = path.join(projectRoot, '..', 'scripts', 'decoder_step.py');
-  const pyRes = spawnSync('python', [pyScript, encoderPath, decJointPath, preprocPath, wavPath, tmpJson, BLANK_ID.toString(), CHUNK.toString()], { stdio: 'inherit' });
-  expect(pyRes.status).toBe(0);
-  const ref = JSON.parse(fs.readFileSync(tmpJson, 'utf8'));
-  fs.unlinkSync(tmpJson);
-
-  // JS path via ParakeetModel
-  const model = await ParakeetModel.fromDirectory(modelDir);
-  const audioData = fs.readFileSync(wavPath);
-  // quick wav parse using node-wav? Instead reuse test helper for reading audio; replicate minimal parse.
-  const wavBuf = audioData;
-  // To avoid dependency, skip JS compute; just validate length of logits using model.joinerSession dims
-  // Instead replicate encoder+preproc as in previous test but using model
-  // We'll reuse OnnxPreprocessor from model.preprocessor
-  const wav = await import('wav');
-  const { Readable } = await import('stream');
-  function readWavFloat(file) {
-    return new Promise((resolve, reject) => {
-      const reader = new wav.Reader();
-      const samples = [];
-      let channels = 0;
-      reader.on('format', fmt=>{ channels=fmt.channels; });
-      reader.on('data', chunk=>{ for(let i=0;i<chunk.length;i+=2) samples.push(chunk.readInt16LE(i)); });
-      reader.on('end', ()=>{
-        const mono = new Float32Array(Math.floor(samples.length/channels));
-        for(let i=0;i<mono.length;++i){
-          let sum=0; for(let c=0;c<channels;++c){ sum+=samples[i*channels+c]; }
-          mono[i]=sum/channels/32768;
-        }
-        resolve(mono);
-      });
-      reader.on('error', reject);
-      Readable.from(file).pipe(reader);
-    });
-  }
-  const audioFloat = await readWavFloat(fs.readFileSync(wavPath));
-  const featRes = await model.preprocessor.process(audioFloat);
-  const T = Math.min(featRes.length, CHUNK);
-  const bufMel = new Float32Array(MEL_BINS * T);
-  for (let t=0;t<T;++t){
-    for (let m=0;m<MEL_BINS;++m){
-      bufMel[m*T+t]=featRes.features[t*MEL_BINS+m];
-    }
-  }
-  const ortMod = await import('onnxruntime-node');
-  const ort = ortMod.default ?? ortMod;
-  const encTensor = new ort.Tensor('float32', bufMel, [1, MEL_BINS, T]);
-  const lenTensor = new ort.Tensor('int64', BigInt64Array.from([BigInt(T)]), [1]);
-  const encOut = await model.encoderSession.run({ 'audio_signal': encTensor, 'length': lenTensor });
-  const encTensorFrame = (encOut['outputs']||Object.values(encOut)[0]);
-  const hidden = encTensorFrame.dims[1];
-  // slice first frame
-  const frameBuf = new Float32Array(hidden);
-  for(let i=0;i<hidden;++i) frameBuf[i]=encTensorFrame.data[i];
-  const encFrame = new ort.Tensor('float32', frameBuf, [1, hidden, 1]);
-  // zero states
-  const numLayers = model.predLayers||2;
-  const state1 = new ort.Tensor('float32', new Float32Array(numLayers*1*model.predHidden), [numLayers,1,model.predHidden]);
-  const state2 = new ort.Tensor('float32', new Float32Array(numLayers*1*model.predHidden), [numLayers,1,model.predHidden]);
-  const out = await model.joinerSession.run({
-    encoder_outputs: encFrame,
-    targets: new ort.Tensor('int32', Int32Array.from([BLANK_ID]), [1,1]),
-    target_length: new ort.Tensor('int32', Int32Array.from([1]), [1]),
-    input_states_1: state1,
-    input_states_2: state2,
-  });
-  const logitsJS = (out['outputs']||out[0]||Object.values(out)[0]).data;
-
-  expect(logitsJS.length).toBe(ref.logits.length);
-  let maxDiff=0;
-  for(let i=0;i<logitsJS.length;++i){
-    const diff=Math.abs(logitsJS[i]-ref.logits[i]);
-    if(diff>maxDiff) maxDiff=diff;
-  }
-  expect(maxDiff).toBeLessThan(1e-3);
-}); 

package/docs/parakeet-transformers-js/test/encoder.test.js

@@ -1,108 +0,0 @@
-import fs from 'fs';
-import path from 'path';
-import { spawnSync } from 'child_process';
-import { OnnxPreprocessor } from '../src/parakeet.js';
-import * as ort from 'onnxruntime-node';
-import wav from 'wav';
-import { MEL_BINS } from '../src/audio_features.js';
-import { fileURLToPath } from 'url';
-
-const __dirname = path.dirname(fileURLToPath(import.meta.url));
-
-const CHUNK = 128;
-
-function readWavFloat32(filePath) {
-  return new Promise((resolve, reject) => {
-    const file = fs.createReadStream(filePath);
-    const reader = new wav.Reader();
-    const samples = [];
-    let sampleRate = 0;
-    let channels = 0;
-
-    reader.on('format', (format) => {
-      sampleRate = format.sampleRate;
-      channels = format.channels;
-    });
-
-    reader.on('data', (chunk) => {
-      for (let i = 0; i < chunk.length; i += 2) {
-        samples.push(chunk.readInt16LE(i));
-      }
-    });
-
-    reader.on('end', () => {
-      const mono = new Float32Array(Math.floor(samples.length / channels));
-      for (let i = 0; i < mono.length; ++i) {
-        let sum = 0;
-        for (let c = 0; c < channels; ++c) {
-          sum += samples[i * channels + c];
-        }
-        mono[i] = (sum / channels) / 32768;
-      }
-      resolve({ audio: mono, sampleRate });
-    });
-
-    reader.on('error', reject);
-    file.on('error', reject);
-
-    file.pipe(reader);
-  });
-}
-
-test('Encoder first vector matches python reference (tolerance 1e-3)', async () => {
-  const projectRoot = path.resolve(__dirname, '..');
-  const assetsDir = path.join(projectRoot, 'assets');
-  const preprocPath = path.join(assetsDir, 'nemo128.onnx');
-  if (!fs.existsSync(preprocPath)) {
-    console.warn('Preprocessor not found, skipping');
-    return;
-  }
-
-  const modelDir = path.resolve(projectRoot, '..', 'parakeet-tdt-0.6b-v2-onnx');
-  let encoderPath = path.join(modelDir, 'encoder-model.int8.onnx');
-  if (!fs.existsSync(encoderPath)) encoderPath = path.join(modelDir, 'encoder-model.onnx');
-  if (!fs.existsSync(encoderPath)) {
-    console.warn('Encoder model not found, skipping');
-    return;
-  }
-
-  const wavPath = path.resolve(projectRoot, '..', 'jfk.wav');
-  if (!fs.existsSync(wavPath)) {
-    console.warn('Test WAV not found, skipping');
-    return;
-  }
-
-  // Run python helper
-  const tmpJson = path.join(projectRoot, '.tmp_enc.json');
-  const pyScript = path.join(projectRoot, '..', 'scripts', 'encode_chunk.py');
-  const pyRes = spawnSync('python', [pyScript, encoderPath, preprocPath, wavPath, tmpJson, CHUNK.toString()], { stdio: 'inherit' });
-  expect(pyRes.status).toBe(0);
-  const ref = JSON.parse(fs.readFileSync(tmpJson, 'utf8'));
-  fs.unlinkSync(tmpJson);
-
-  // JS pipeline
-  const { audio } = await readWavFloat32(wavPath);
-  const preproc = new OnnxPreprocessor(preprocPath);
-  const procRes = await preproc.process(audio);
-  const T = Math.min(procRes.length, CHUNK);
-  const buf = new Float32Array(MEL_BINS * T);
-  for (let t = 0; t < T; ++t) {
-    for (let m = 0; m < MEL_BINS; ++m) {
-      buf[m * T + t] = procRes.features[t * MEL_BINS + m];
-    }
-  }
-  const inputTensor = new ort.Tensor('float32', buf, [1, MEL_BINS, T]);
-  const lenTensor = new ort.Tensor('int64', BigInt64Array.from([BigInt(T)]), [1]);
-  const encSession = await ort.InferenceSession.create(encoderPath);
-  const encOut = await encSession.run({ 'audio_signal': inputTensor, 'length': lenTensor });
-  const enc = encOut['outputs'] || Object.values(encOut)[0];
-  const jsVec = enc.data.slice(0, ref.hidden);
-
-  expect(jsVec.length).toBe(ref.vector.length);
-  let maxDiff = 0;
-  for (let i = 0; i < jsVec.length; ++i) {
-    const diff = Math.abs(jsVec[i] - ref.vector[i]);
-    if (diff > maxDiff) maxDiff = diff;
-  }
-  expect(maxDiff).toBeLessThan(1e-3);
-}); 

package/docs/parakeet-transformers-js/test/preprocessor.test.js

@@ -1,85 +0,0 @@
-import fs from 'fs';
-import path from 'path';
-import { spawnSync } from 'child_process';
-import { OnnxPreprocessor } from '../src/parakeet.js';
-import wav from 'wav';
-import { fileURLToPath } from 'url';
-
-const __dirname = path.dirname(fileURLToPath(import.meta.url));
-
-function readWavFloat32(filePath) {
-  return new Promise((resolve, reject) => {
-    const file = fs.createReadStream(filePath);
-    const reader = new wav.Reader();
-    const samples = [];
-    let sampleRate = 0;
-    let channels = 0;
-
-    reader.on('format', (format) => {
-      sampleRate = format.sampleRate;
-      channels = format.channels;
-    });
-
-    reader.on('data', (chunk) => {
-      for (let i = 0; i < chunk.length; i += 2) {
-        samples.push(chunk.readInt16LE(i));
-      }
-    });
-
-    reader.on('end', () => {
-      const mono = new Float32Array(Math.floor(samples.length / channels));
-      for (let i = 0; i < mono.length; ++i) {
-        let sum = 0;
-        for (let c = 0; c < channels; ++c) {
-          sum += samples[i * channels + c];
-        }
-        mono[i] = (sum / channels) / 32768;
-      }
-      resolve({ audio: mono, sampleRate });
-    });
-
-    reader.on('error', reject);
-    file.on('error', reject);
-
-    file.pipe(reader);
-  });
-}
-
-test('ONNX preprocessor matches python reference output', async () => {
-  const projectRoot = path.resolve(__dirname, '..');
-  const assetsDir = path.join(projectRoot, 'assets');
-  const preprocPath = path.join(assetsDir, 'nemo128.onnx');
-  if (!fs.existsSync(preprocPath)) {
-    console.warn('Preprocessor model not found, skipping test');
-    return;
-  }
-
-  const wavPath = path.resolve(projectRoot, '..', 'jfk.wav');
-  if (!fs.existsSync(wavPath)) {
-    console.warn('Test WAV not found, skipping test');
-    return;
-  }
-
-  const tmpJson = path.join(projectRoot, '.tmp_features.json');
-  const pythonScript = path.join(projectRoot, '..', 'scripts', 'compute_features.py');
-  const pyRes = spawnSync('python', [pythonScript, preprocPath, wavPath, tmpJson], { stdio: 'inherit' });
-  expect(pyRes.status).toBe(0);
-  const ref = JSON.parse(fs.readFileSync(tmpJson, 'utf8'));
-  fs.unlinkSync(tmpJson);
-
-  const { audio } = await readWavFloat32(wavPath);
-  const jsPreproc = new OnnxPreprocessor(preprocPath);
-  const jsRes = await jsPreproc.process(audio);
-
-  expect(jsRes.length).toBe(ref.length);
-  const refFeat = Float32Array.from(ref.features);
-  const jsFeat = jsRes.features;
-  expect(jsFeat.length).toBe(refFeat.length);
-
-  let maxDiff = 0;
-  for (let i = 0; i < jsFeat.length; ++i) {
-    const diff = Math.abs(jsFeat[i] - refFeat[i]);
-    if (diff > maxDiff) maxDiff = diff;
-  }
-  expect(maxDiff).toBeLessThan(1e-4);
-}); 

package/docs/parakeet-transformers-js/test/tokenizer.test.js

@@ -1,24 +0,0 @@
-import { ParakeetTokenizer } from '../src/parakeet.js';
-
-describe('ParakeetTokenizer', () => {
-  test('decode joins tokens with spaces correctly', () => {
-    const id2token = ['<blk>', '▁hello', '▁world', '!'];
-    const tok = new ParakeetTokenizer(id2token);
-    const text = tok.decode([1, 2, 3]);
-    expect(text).toBe('hello world!');
-  });
-
-  test('blank tokens are skipped', () => {
-    const id2token = ['<blk>', '▁test'];
-    const tok = new ParakeetTokenizer(id2token);
-    const text = tok.decode([0, 1, 0]);
-    expect(text).toBe('test');
-  });
-
-  test('unknown ids are ignored', () => {
-    const id2token = ['<blk>', '▁foo'];
-    const tok = new ParakeetTokenizer(id2token);
-    const text = tok.decode([1, 99, 0]);
-    expect(text).toBe('foo');
-  });
-}); 

package/docs/parakeet-transformers-js/test/transcribe.js

@@ -1,89 +0,0 @@
-// Quick smoke test to transcribe sherpa-onnx-parakeet-model/test_wavs/0.wav
-import fs from 'fs';
-import path from 'path';
-import { ParakeetModel } from '../src/parakeet.js';
-import wav from 'wav';
-const { Reader } = wav;
-import { Writable } from 'stream';
-
-function readWav(filePath) {
-  return new Promise((resolve, reject) => {
-    const file = fs.createReadStream(filePath);
-    const reader = new Reader();
-    const samples = [];
-    let sampleRate = 0;
-    let channels = 0;
-
-    const writer = new Writable({
-      write(chunk, encoding, callback) {
-        for (let i = 0; i < chunk.length; i += 2) {
-          samples.push(chunk.readInt16LE(i));
-        }
-        callback();
-      }
-    });
-
-    reader.on('format', (format) => {
-      sampleRate = format.sampleRate;
-      channels = format.channels;
-      console.log(`[Test] WAV format: ${sampleRate} Hz, ${channels} channels, ${format.bitDepth}-bit`);
-    });
-
-    file.pipe(reader).pipe(writer);
-
-    writer.on('finish', () => {
-      let monoSamples = new Float32Array(Math.floor(samples.length / channels));
-      for (let i = 0; i < monoSamples.length; i++) {
-        let sum = 0;
-        for (let c = 0; c < channels; c++) {
-          sum += samples[i * channels + c];
-        }
-        monoSamples[i] = (sum / channels) / 32768;
-      }
-      resolve({ audio: monoSamples, sampleRate });
-    });
-
-    writer.on('error', reject);
-    reader.on('error', reject);
-    file.on('error', reject);
-  });
-}
-
-async function main() {
-  const modelDir = path.resolve('../parakeet-transformers-js-model');
-  const model = await ParakeetModel.fromDirectory(modelDir);
-  const wavPath = path.resolve('../jfk.wav');
-  let { audio, sampleRate } = await readWav(wavPath);
-
-  // Resample if necessary (simple linear interpolation)
-  if (sampleRate !== 16000) {
-    console.log(`[Test] Resampling from ${sampleRate} Hz to 16000 Hz...`);
-    const ratio = sampleRate / 16000;
-    const newLength = Math.floor(audio.length / ratio);
-    const resampled = new Float32Array(newLength);
-    for (let i = 0; i < newLength; i++) {
-      const index = i * ratio;
-      const i_low = Math.floor(index);
-      const i_high = Math.ceil(index);
-      if (i_high >= audio.length) {
-          resampled[i] = audio[i_low];
-          continue;
-      }
-      const w = index - i_low;
-      resampled[i] = audio[i_low] * (1 - w) + audio[i_high] * w;
-    }
-    audio = resampled;
-    sampleRate = 16000;
-  }
-
-  // Tail padding 2 s
-  const pad = new Float32Array(sampleRate * 2);
-  const padded = new Float32Array(audio.length + pad.length);
-  padded.set(audio);
-  padded.set(pad, audio.length);
-
-  const result = await model.transcribe(padded, sampleRate, { returnTimestamps: true, returnConfidences: true, temperature: 2.5, debug: true, skipCMVN: true });
-  console.log(JSON.stringify(result, null, 2));
-}
-
-main(); 

package/docs/parakeet-transformers-js/tsconfig.json

@@ -1,21 +0,0 @@
-{
-  // Only include files in the src directory
-  "include": ["src/**/*"],
-  "compilerOptions": {
-    // Tells the compiler to check JS files
-    "checkJs": true,
-    "target": "esnext",
-    "module": "nodenext",
-    "moduleResolution": "nodenext",
-    "outDir": "types",
-    "strict": false,
-    "skipLibCheck": true,
-    "declaration": true,
-    "declarationMap": true,
-    "noEmit": false,
-    "emitDeclarationOnly": true
-  },
-  "typeAcquisition": {
-    "include": ["jest"]
-  }
-}