runonweb 0.0.1

package/README.md

@@ -0,0 +1,165 @@
+# runonweb
+
+Free browser ML modules. Models run **locally in the browser** (WebGPU preferred, WASM fallback). No API keys, no servers. MIT.
+
+```bash
+pnpm add runonweb
+npm install runonweb
+bun add runonweb
+yarn add runonweb
+```
+
+Every module follows the same shape:
+
+```ts
+const m = new Module({ onProgress })
+await m.load()          // downloads + caches weights, safe to call twice
+await m.<task>(input)   // the actual work
+m.dispose()             // free memory
+```
+
+Each module also exports a one-shot function (`transcribe`, `removeBackground`, `caption`, …) that loads, runs and disposes.
+
+## Modules
+
+### Speech-to-text: `runonweb/stt`
+
+```ts
+import { SpeechToText } from 'runonweb/stt'
+
+const stt = new SpeechToText()
+await stt.load()
+const { text } = await stt.transcribe(audioBlob, {
+  onPartial: (t) => console.log(t), // words as Whisper emits them
+})
+```
+
+Base: Whisper tiny.en (OpenAI, Apache-2.0). Pass `model` + `language` for multilingual Whisper variants.
+
+### Background removal: `runonweb/remove-bg`
+
+```ts
+import { RemoveBackground } from 'runonweb/remove-bg'
+
+const remover = new RemoveBackground()
+const png = await remover.remove(imageFile) // PNG Blob with alpha
+```
+
+Base: BEN2 (Prama LLC, MIT, ~219 MB fp16). Hair, objects and hard edges. WASM only until onnxruntime-web ships the LayerNorm shader fix (onnxruntime#32629).
+
+### Image captioning: `runonweb/caption`
+
+```ts
+import { ImageCaptioner } from 'runonweb/caption'
+
+const captioner = new ImageCaptioner({ language: 'es', detail: 'short' })
+const { text } = await captioner.caption(imageFile)
+```
+
+Base: LFM2.5-VL-450M (Liquid AI, ~316 MB on WebGPU). Captions in `en`, `es`, `pt`, `fr`, `de`, `ar`, `zh`, `ja`, `ko`; `detail` is `short` | `detailed` | `more`; `onPartial` streams.
+
+> **License:** LFM Open License v1.0. Free for individuals and companies under USD 10M annual revenue; larger companies need a commercial license from Liquid AI. Not OSI-approved. See <https://huggingface.co/LiquidAI/LFM2.5-VL-450M/blob/main/LICENSE>.
+
+### Depth estimation: `runonweb/depth`
+
+```ts
+import { DepthEstimator } from 'runonweb/depth'
+
+const { depth, width, height } = await new DepthEstimator().estimate(imageFile)
+// depth: grayscale PNG Blob, brighter = closer
+```
+
+Base: Depth Anything V2 Small (Apache-2.0).
+
+### Object detection: `runonweb/detect`
+
+```ts
+import { ObjectDetector } from 'runonweb/detect'
+
+const objects = await new ObjectDetector({ threshold: 0.5 }).detect(imageFile)
+// [{ label: 'cat', score: 0.98, box: { xmin, ymin, xmax, ymax } }]
+```
+
+Base: DETR ResNet-50 (Meta, Apache-2.0). Boxes in pixel coordinates.
+
+### OCR: `runonweb/ocr`
+
+```ts
+import { OCR } from 'runonweb/ocr'
+
+const ocr = new OCR() // size: 'small', best size/quality
+// const ocr = new OCR({ size: 'tiny' })   // ~6 MB
+// const ocr = new OCR({ size: 'medium' }) // ~139 MB
+const { text, lines } = await ocr.read(imageFile)
+```
+
+Base: PP-OCRv6 (PaddlePaddle, Apache-2.0). `tiny` / `small` / `medium`. Default is `small` (~31 MB).
+
+### Text embeddings: `runonweb/embed`
+
+```ts
+import { TextEmbedder, cosineSimilarity } from 'runonweb/embed'
+
+const { embeddings, dimensions } = await new TextEmbedder().embed(['a', 'b'])
+cosineSimilarity(embeddings[0], embeddings[1])
+```
+
+Base: all-MiniLM-L6-v2 (Apache-2.0). 384 dims, normalized.
+
+### Translation: `runonweb/translate`
+
+```ts
+import { Translator, PAIRS, hasPair } from 'runonweb/translate'
+
+const t = new Translator({ from: 'en', to: 'es' }) // Firefox Translations en-es, ~37 MB
+const { text } = await t.translate('Hello world')
+
+await t.translate('Bonjour', { from: 'fr', to: 'en' }) // same instance, downloaded on demand
+await t.translate('<b>Hello</b> world', { html: true }) // keeps the markup
+hasPair('es', 'fr') // true, pivots through English
+
+// one Transformers.js model for 100 languages (MIT, ~630 MB)
+const multi = new Translator({ model: 'Xenova/m2m100_418M', from: 'fr', to: 'en' })
+```
+
+Base: Mozilla's [Firefox Translations](https://github.com/mozilla/translations) models (MPL-2.0), the same Marian NMT students Firefox ships. 17–44 MB per pair (int8), 106 direct pairs between English and 58 languages; other pairs pivot through English. They run in a Web Worker through the Bergamot WASM runtime (no COOP/COEP headers needed) and are cached in Cache Storage. `PAIRS` lists every pair with architecture, size and COMET score.
+
+Weights come from the runonweb mirror on the Hugging Face Hub by default. To self-host, run `node scripts/translate-models.mjs fetch en-es es-en` (or `--all`) and pass `modelPath` pointing at wherever you upload that folder; the script also copies the runtime so you can pass `runtimePath: '<modelPath>/runtime/'` instead of loading it from jsDelivr. The registry is regenerated from Mozilla's model list with `node scripts/translate-models.mjs registry`.
+
+### Text-to-speech: `runonweb/tts`
+
+```ts
+import { TextToSpeech } from 'runonweb/tts'
+
+const tts = new TextToSpeech({ size: 'small', voice: 'af_heart' })
+await tts.load()
+
+for await (const chunk of tts.speakStream('Hello from the browser')) {
+  // chunk.audio: 24 kHz PCM, one sentence at a time
+}
+
+const wav = await tts.speakToBlob('Hola mundo', { voice: 'ef_dora' })
+
+// Supertonic 2: one model for en · ko · es · pt · fr, 44.1 kHz
+const multi = new TextToSpeech({ size: 'multi', voice: 'st_f1' })
+const pt = await multi.speakToBlob('Olá do navegador', { language: 'pt' })
+```
+
+Base: Kokoro 82M (hexgrad, Apache-2.0) by default. English, Spanish, French, on Transformers.js directly. Pass `size: 'multi'` for Supertonic 2 (Supertone, OpenRAIL-M, ~262 MB): English, Korean, Spanish, Portuguese, French with 10 shared voices `st_f1`…`st_m5` and a `language` option. Pass `size: 'tiny'` for KittenTTS nano (~28 MB, 8 English voices). Audio streams via `speakStream`. Kokoro Spanish/French download a local eSpeak-NG WASM (~18 MB) on first use.
+
+### Core helpers: `runonweb/core`
+
+```ts
+import { isWebGPUAvailable, resolveDevice } from 'runonweb/core'
+
+const ok = await isWebGPUAvailable()
+const device = await resolveDevice() // 'webgpu' | 'wasm'
+```
+
+## Notes
+
+- First run downloads model weights from Hugging Face; the browser caches them afterward.
+- Each module picks its quantization per device (`fp16`/`fp32` on WebGPU, `q8` on WASM). `q8` on WebGPU is never used because it produces wrong results.
+- Models that fail on WebGPU in ONNX Runtime Web (detect) are pinned to WASM through `supportedDevices` and ignore `device: 'webgpu'`. Translation runs on CPU (Bergamot WASM) by design. Kokoro and Supertonic TTS use WebGPU; KittenTTS (`size: 'tiny'`) is WASM-only.
+- Nothing leaves the browser. Audio, images and text stay on the user's device.
+- All default models are Apache-2.0 or MIT except `runonweb/caption` (LFM Open License v1.0, free under USD 10M revenue). Attribution to the original authors is listed above and on every demo page.

package/package.json

@@ -0,0 +1,126 @@
+{
+  "name": "runonweb",
+  "version": "0.0.1",
+  "description": "Free browser ML modules: speech, vision, text. Runs locally with WebGPU or WASM.",
+  "type": "module",
+  "license": "MIT",
+  "exports": {
+    "./core": {
+      "types": "./src/core/index.ts",
+      "import": "./src/core/index.ts",
+      "default": "./src/core/index.ts"
+    },
+    "./stt": {
+      "types": "./src/stt/index.ts",
+      "import": "./src/stt/index.ts",
+      "default": "./src/stt/index.ts"
+    },
+    "./remove-bg": {
+      "types": "./src/remove-bg/index.ts",
+      "import": "./src/remove-bg/index.ts",
+      "default": "./src/remove-bg/index.ts"
+    },
+    "./caption": {
+      "types": "./src/caption/index.ts",
+      "import": "./src/caption/index.ts",
+      "default": "./src/caption/index.ts"
+    },
+    "./depth": {
+      "types": "./src/depth/index.ts",
+      "import": "./src/depth/index.ts",
+      "default": "./src/depth/index.ts"
+    },
+    "./detect": {
+      "types": "./src/detect/index.ts",
+      "import": "./src/detect/index.ts",
+      "default": "./src/detect/index.ts"
+    },
+    "./ocr": {
+      "types": "./src/ocr/index.ts",
+      "import": "./src/ocr/index.ts",
+      "default": "./src/ocr/index.ts"
+    },
+    "./ocr/sizes": {
+      "types": "./src/ocr/sizes.ts",
+      "import": "./src/ocr/sizes.ts",
+      "default": "./src/ocr/sizes.ts"
+    },
+    "./embed": {
+      "types": "./src/embed/index.ts",
+      "import": "./src/embed/index.ts",
+      "default": "./src/embed/index.ts"
+    },
+    "./translate": {
+      "types": "./src/translate/index.ts",
+      "import": "./src/translate/index.ts",
+      "default": "./src/translate/index.ts"
+    },
+    "./tts": {
+      "types": "./src/tts/index.ts",
+      "import": "./src/tts/index.ts",
+      "default": "./src/tts/index.ts"
+    },
+    "./tts/voices": {
+      "types": "./src/tts/voices.ts",
+      "import": "./src/tts/voices.ts",
+      "default": "./src/tts/voices.ts"
+    },
+    "./tts/sizes": {
+      "types": "./src/tts/sizes.ts",
+      "import": "./src/tts/sizes.ts",
+      "default": "./src/tts/sizes.ts"
+    },
+    "./clean": {
+      "types": "./src/clean/index.ts",
+      "import": "./src/clean/index.ts",
+      "default": "./src/clean/index.ts"
+    },
+    "./emoji": {
+      "types": "./src/emoji/index.ts",
+      "import": "./src/emoji/index.ts",
+      "default": "./src/emoji/index.ts"
+    },
+    "./image": {
+      "types": "./src/image/index.ts",
+      "import": "./src/image/index.ts",
+      "default": "./src/image/index.ts"
+    },
+    "./image/sizes": {
+      "types": "./src/image/sizes.ts",
+      "import": "./src/image/sizes.ts",
+      "default": "./src/image/sizes.ts"
+    }
+  },
+  "files": [
+    "src"
+  ],
+  "sideEffects": false,
+  "dependencies": {
+    "@huggingface/transformers": "^4.3.0",
+    "onnxruntime-web": "^1.30.0",
+    "paddleocr": "^1.2.0",
+    "phonemizer": "^1.2.1"
+  },
+  "keywords": [
+    "webgpu",
+    "wasm",
+    "onnx",
+    "transformers",
+    "browser",
+    "ml",
+    "ai",
+    "whisper",
+    "speech-to-text",
+    "background-removal",
+    "ocr",
+    "embeddings",
+    "translation",
+    "tts",
+    "emoji",
+    "transcript-cleanup",
+    "dictation",
+    "image-generation",
+    "text-to-image",
+    "bonsai"
+  ]
+}

package/src/caption/index.ts

@@ -0,0 +1,310 @@
+import type { Device, ProgressCallback, ProgressInfo, ResolvedDevice } from '../core/index.ts'
+import { resolveDevice } from '../core/device.ts'
+import { imageToPipelineInput } from '../core/pipeline.ts'
+import { toProgressInfo } from '../core/progress.ts'
+
+/**
+ * Default: LFM2.5-VL-450M (Liquid AI, Nov 2025). Multilingual vision-language model.
+ *
+ * License: LFM Open License v1.0. Free for individuals and for companies under
+ * USD 10M annual revenue. Larger companies need a commercial license from Liquid AI.
+ * https://huggingface.co/LiquidAI/LFM2.5-VL-450M/blob/main/LICENSE
+ */
+const DEFAULT_MODEL = 'onnx-community/LFM2.5-VL-450M-ONNX'
+
+/** Languages the default model was trained on. Any other value is passed to the prompt as-is. */
+export const CAPTION_LANGUAGES = {
+  en: 'English',
+  es: 'Spanish',
+  pt: 'Portuguese',
+  fr: 'French',
+  de: 'German',
+  ar: 'Arabic',
+  zh: 'Chinese',
+  ja: 'Japanese',
+  ko: 'Korean',
+} as const
+
+export type CaptionLanguage = keyof typeof CAPTION_LANGUAGES
+
+const PROMPTS = {
+  short: 'Describe this image in one short sentence, suitable as alt text. Do not start with "This image" or "The image".',
+  detailed: 'Describe this image in two or three sentences.',
+  more: 'Describe this image in detail: subjects, setting, colors, composition and any visible text.',
+} as const
+
+const MAX_TOKENS = {
+  short: 48,
+  detailed: 128,
+  more: 320,
+} as const
+
+export type CaptionDetail = keyof typeof PROMPTS
+
+export type CaptionOptions = {
+  model?: string
+  device?: Device
+  /** Caption verbosity. `short` is alt-text style. */
+  detail?: CaptionDetail
+  /**
+   * Caption language. A code from `CAPTION_LANGUAGES` or a plain language name ("Catalan").
+   * Default `en`. The model was trained on the 9 listed languages; others are best-effort.
+   */
+  language?: CaptionLanguage | (string & {})
+  /** Replace the built-in instruction. `language` is still appended. */
+  prompt?: string
+  /** Max new tokens for generation. Defaults depend on `detail`. */
+  maxNewTokens?: number
+  onProgress?: ProgressCallback
+}
+
+export type CaptionRunOptions = Pick<CaptionOptions, 'detail' | 'language' | 'prompt' | 'maxNewTokens'> & {
+  /** Receives the caption as it is generated. */
+  onPartial?: (text: string) => void
+}
+
+export type CaptionImageInput = Blob | File | string | HTMLImageElement | HTMLCanvasElement | ImageData
+
+export type CaptionResult = {
+  text: string
+}
+
+type Tensor = {
+  dims: number[]
+  slice: (...args: unknown[]) => Tensor
+}
+
+type VlmModel = {
+  generate: (inputs: Record<string, unknown>) => Promise<Tensor>
+  dispose?: () => Promise<void>
+}
+
+type VlmProcessor = {
+  (image: unknown, text: string, options?: Record<string, unknown>): Promise<Record<string, unknown>>
+  apply_chat_template: (messages: unknown, options?: Record<string, unknown>) => string
+  batch_decode: (ids: Tensor, options: { skip_special_tokens: boolean }) => string[]
+  tokenizer: unknown
+  image_processor: { do_image_splitting?: boolean }
+}
+
+type RawImageLike = {
+  width: number
+  height: number
+}
+
+async function supportsShaderF16(): Promise<boolean> {
+  try {
+    const gpu = (globalThis as { navigator?: { gpu?: GPU } }).navigator?.gpu
+    const adapter = await gpu?.requestAdapter()
+    return Boolean(adapter?.features.has('shader-f16'))
+  } catch {
+    return false
+  }
+}
+
+function dtypeFor(device: ResolvedDevice, fp16: boolean) {
+  if (device === 'webgpu') {
+    // Official LFM2.5-VL WebGPU recipe: fp16 vision + embeddings, q4f16 decoder.
+    return fp16
+      ? ({ embed_tokens: 'fp16', vision_encoder: 'fp16', decoder_model_merged: 'q4f16' } as const)
+      : ({ embed_tokens: 'fp32', vision_encoder: 'fp32', decoder_model_merged: 'q4' } as const)
+  }
+  // WASM: q8/q4 embeddings use GatherBlockQuantized, which ONNX Runtime Web lacks on WASM. fp16 works.
+  return { embed_tokens: 'fp16', vision_encoder: 'q8', decoder_model_merged: 'q4' } as const
+}
+
+async function toRawImage(image: CaptionImageInput): Promise<RawImageLike> {
+  const input = await imageToPipelineInput(image)
+  if (typeof input !== 'string') return input as unknown as RawImageLike
+  const { RawImage } = await import('@huggingface/transformers')
+  return RawImage.fromURL(input) as unknown as Promise<RawImageLike>
+}
+
+function languageName(language: string): string {
+  return (CAPTION_LANGUAGES as Record<string, string>)[language] ?? language
+}
+
+function buildPrompt(detail: CaptionDetail, language: string, custom?: string): string {
+  const base = custom ?? PROMPTS[detail]
+  return `${base} Answer in ${languageName(language)}.`
+}
+
+/**
+ * LFM2.5-VL ships a chat template that uses the `{% generation %}` tag, which the Jinja
+ * engine bundled with Transformers.js 4.2 does not parse yet. Try the template first (so a
+ * custom `model` keeps its own format) and fall back to the documented ChatML layout.
+ */
+function formatChat(processor: VlmProcessor, prompt: string): string {
+  try {
+    const messages = [{ role: 'user', content: [{ type: 'image' }, { type: 'text', text: prompt }] }]
+    return processor.apply_chat_template(messages, { add_generation_prompt: true })
+  } catch {
+    return `<|startoftext|><|im_start|>user\n<image>${prompt}<|im_end|>\n<|im_start|>assistant\n`
+  }
+}
+
+function cleanCaption(text: string): string {
+  const t = text.trim().replace(/^["'“”]+|["'“”]+$/g, '').trim()
+  return t.charAt(0).toUpperCase() + t.slice(1)
+}
+
+/**
+ * Image captioning in the browser (LFM2.5-VL-450M, multilingual).
+ *
+ * @example
+ * ```ts
+ * import { ImageCaptioner } from 'runonweb/caption'
+ *
+ * const captioner = new ImageCaptioner({ language: 'es' })
+ * await captioner.load()
+ * const { text } = await captioner.caption(imageFile)
+ * ```
+ */
+export class ImageCaptioner {
+  #modelId: string
+  #device: Device
+  #detail: CaptionDetail
+  #language: string
+  #prompt?: string
+  #maxNewTokens?: number
+  #onProgress?: ProgressCallback
+  #model: VlmModel | null = null
+  #processor: VlmProcessor | null = null
+  #loading: Promise<void> | null = null
+  #resolvedDevice: 'webgpu' | 'wasm' | null = null
+
+  constructor(options: CaptionOptions = {}) {
+    this.#modelId = options.model ?? DEFAULT_MODEL
+    this.#device = options.device ?? 'auto'
+    this.#detail = options.detail ?? 'short'
+    this.#language = options.language ?? 'en'
+    this.#prompt = options.prompt
+    this.#maxNewTokens = options.maxNewTokens
+    this.#onProgress = options.onProgress
+  }
+
+  get device(): 'webgpu' | 'wasm' | null {
+    return this.#resolvedDevice
+  }
+
+  async load(): Promise<void> {
+    if (this.#model) return
+    if (this.#loading) return this.#loading
+
+    this.#loading = (async () => {
+      const preferred = await resolveDevice(this.#device)
+      try {
+        await this.#loadOn(preferred)
+      } catch (err) {
+        if (preferred === 'wasm' || this.#device === 'wasm') throw err
+        await this.#loadOn('wasm')
+      }
+    })()
+
+    try {
+      await this.#loading
+    } finally {
+      this.#loading = null
+    }
+  }
+
+  async #loadOn(device: ResolvedDevice): Promise<void> {
+    this.#onProgress?.({ status: 'loading', progress: 0 })
+
+    const { AutoModelForImageTextToText, AutoProcessor, env } = await import('@huggingface/transformers')
+    env.allowLocalModels = false
+
+    const fp16 = device === 'webgpu' && (await supportsShaderF16())
+    const progress_callback = (data: Record<string, unknown>) => {
+      this.#onProgress?.(toProgressInfo(data))
+    }
+
+    const [model, processor] = await Promise.all([
+      AutoModelForImageTextToText.from_pretrained(this.#modelId, {
+        device,
+        dtype: dtypeFor(device, fp16),
+        progress_callback,
+      }),
+      AutoProcessor.from_pretrained(this.#modelId, { progress_callback }),
+    ])
+
+    const proc = processor as unknown as VlmProcessor
+    // Images are resized to fit 512×512 anyway; tiling only adds tokens and latency for captions.
+    if (proc.image_processor) proc.image_processor.do_image_splitting = false
+
+    this.#model = model as unknown as VlmModel
+    this.#processor = proc
+    this.#resolvedDevice = device
+
+    this.#onProgress?.({ status: 'ready', progress: 100 })
+  }
+
+  async caption(image: CaptionImageInput, options: CaptionRunOptions = {}): Promise<CaptionResult> {
+    await this.load()
+    const model = this.#model
+    const processor = this.#processor
+    if (!model || !processor) {
+      throw new Error('ImageCaptioner model failed to load')
+    }
+
+    this.#onProgress?.({ status: 'captioning' })
+
+    const detail = options.detail ?? this.#detail
+    const language = options.language ?? this.#language
+    const prompt = buildPrompt(detail, language, options.prompt ?? this.#prompt)
+    const maxNewTokens = options.maxNewTokens ?? this.#maxNewTokens ?? MAX_TOKENS[detail]
+
+    const chat = formatChat(processor, prompt)
+
+    const rawImage = await toRawImage(image)
+    const inputs = await processor(rawImage, chat, { add_special_tokens: false })
+
+    let streamer: unknown
+    if (options.onPartial) {
+      const { TextStreamer } = await import('@huggingface/transformers')
+      let partial = ''
+      streamer = new TextStreamer(processor.tokenizer as never, {
+        skip_prompt: true,
+        skip_special_tokens: true,
+        callback_function: (chunk: string) => {
+          partial += chunk
+          options.onPartial?.(cleanCaption(partial))
+        },
+      })
+    }
+
+    const output = await model.generate({
+      ...inputs,
+      max_new_tokens: maxNewTokens,
+      do_sample: false,
+      repetition_penalty: 1.05,
+      ...(streamer ? { streamer } : {}),
+    })
+
+    const promptLength = (inputs.input_ids as Tensor).dims.at(-1) ?? 0
+    const decoded = processor.batch_decode(output.slice(null, [promptLength, null]), { skip_special_tokens: true })
+    const text = cleanCaption(decoded[0] ?? '')
+
+    this.#onProgress?.({ status: 'done' })
+    return { text }
+  }
+
+  dispose(): void {
+    const model = this.#model
+    this.#model = null
+    this.#processor = null
+    this.#resolvedDevice = null
+    void model?.dispose?.()
+  }
+}
+
+export async function caption(image: CaptionImageInput, options?: CaptionOptions): Promise<CaptionResult> {
+  const captioner = new ImageCaptioner(options)
+  try {
+    return await captioner.caption(image)
+  } finally {
+    captioner.dispose()
+  }
+}
+
+export type { ProgressInfo, Device }