runonweb 0.0.1

package/src/depth/index.ts

@@ -0,0 +1,127 @@
+import type { Device, ProgressCallback, ProgressInfo } from '../core/index.ts'
+import { imageToPipelineInput, loadPipeline } from '../core/pipeline.ts'
+
+const DEFAULT_MODEL = 'onnx-community/depth-anything-v2-small'
+
+export type DepthOptions = {
+  model?: string
+  device?: Device
+  onProgress?: ProgressCallback
+}
+
+export type DepthImageInput = Blob | File | string | HTMLImageElement | HTMLCanvasElement | ImageData
+
+export type DepthResult = {
+  /** Grayscale depth visualization as PNG. */
+  depth: Blob
+  width: number
+  height: number
+}
+
+/**
+ * Monocular depth estimation in the browser (Depth Anything).
+ *
+ * @example
+ * ```ts
+ * import { DepthEstimator } from 'runonweb/depth'
+ *
+ * const depth = new DepthEstimator()
+ * await depth.load()
+ * const { depth: png } = await depth.estimate(imageFile)
+ * ```
+ */
+export class DepthEstimator {
+  #model: string
+  #device: Device
+  #onProgress?: ProgressCallback
+  #pipe: (CallableFunction & { dispose?: () => Promise<void> }) | null = null
+  #loading: Promise<void> | null = null
+  #resolvedDevice: 'webgpu' | 'wasm' | null = null
+
+  constructor(options: DepthOptions = {}) {
+    this.#model = options.model ?? DEFAULT_MODEL
+    this.#device = options.device ?? 'auto'
+    this.#onProgress = options.onProgress
+  }
+
+  get device(): 'webgpu' | 'wasm' | null {
+    return this.#resolvedDevice
+  }
+
+  async load(): Promise<void> {
+    if (this.#pipe) return
+    if (this.#loading) return this.#loading
+
+    this.#loading = (async () => {
+      const { pipe, device } = await loadPipeline({
+        task: 'depth-estimation',
+        model: this.#model,
+        device: this.#device,
+        // fp16 on WebGPU (~50 MB); q8 on WASM (~27 MB)
+        dtype: (d) => (d === 'webgpu' ? 'fp16' : 'q8'),
+        onProgress: this.#onProgress,
+      })
+      this.#pipe = pipe
+      this.#resolvedDevice = device
+    })()
+
+    try {
+      await this.#loading
+    } finally {
+      this.#loading = null
+    }
+  }
+
+  async estimate(image: DepthImageInput): Promise<DepthResult> {
+    await this.load()
+    if (!this.#pipe) throw new Error('DepthEstimator model failed to load')
+
+    this.#onProgress?.({ status: 'processing' })
+    const input = await imageToPipelineInput(image)
+    const raw = (await this.#pipe(input)) as {
+      depth: {
+        width: number
+        height: number
+        toBlob?: () => Promise<Blob>
+        toCanvas?: () => HTMLCanvasElement
+      }
+    }
+
+    const depthImage = raw.depth
+    let blob: Blob
+    if (typeof depthImage.toBlob === 'function') {
+      blob = await depthImage.toBlob()
+    } else if (typeof depthImage.toCanvas === 'function') {
+      const canvas = depthImage.toCanvas()
+      blob = await new Promise<Blob>((resolve, reject) => {
+        canvas.toBlob((b) => (b ? resolve(b) : reject(new Error('toBlob failed'))), 'image/png')
+      })
+    } else {
+      throw new Error('Depth output has no toBlob/toCanvas')
+    }
+
+    this.#onProgress?.({ status: 'done' })
+    return { depth: blob, width: depthImage.width, height: depthImage.height }
+  }
+
+  dispose(): void {
+    const pipe = this.#pipe
+    this.#pipe = null
+    this.#resolvedDevice = null
+    void pipe?.dispose?.()
+  }
+}
+
+export async function estimateDepth(
+  image: DepthImageInput,
+  options?: DepthOptions
+): Promise<DepthResult> {
+  const estimator = new DepthEstimator(options)
+  try {
+    return await estimator.estimate(image)
+  } finally {
+    estimator.dispose()
+  }
+}
+
+export type { ProgressInfo, Device }

package/src/detect/index.ts

@@ -0,0 +1,274 @@
+import type { Device, ProgressCallback, ProgressInfo } from '../core/index.ts'
+import { imageToPipelineInput, loadPipeline } from '../core/pipeline.ts'
+
+const DEFAULT_MODEL = 'onnx-community/rfdetr_nano-ONNX'
+
+/** RT-DETR family uses per-class sigmoid, not DETR's softmax + background class. */
+const SIGMOID_TYPES = new Set(['rf_detr', 'rt_detr', 'rt_detr_v2', 'd_fine'])
+
+function isSigmoidModelId(model: string): boolean {
+  return /rfdetr|rt_detr|rtdetr|dfine|d_fine/i.test(model)
+}
+
+export type DetectOptions = {
+  model?: string
+  device?: Device
+  /** Minimum confidence score (0–1). Default 0.5. */
+  threshold?: number
+  /** Return boxes as fractions of the image (0–1) instead of pixels. Default false. */
+  normalized?: boolean
+  onProgress?: ProgressCallback
+}
+
+export type DetectImageInput = Blob | File | string | HTMLImageElement | HTMLCanvasElement | ImageData
+
+/** Box in pixel coordinates of the input image (or 0–1 fractions with `normalized: true`). */
+export type DetectionBox = {
+  xmin: number
+  ymin: number
+  xmax: number
+  ymax: number
+}
+
+export type Detection = {
+  label: string
+  score: number
+  box: DetectionBox
+}
+
+/**
+ * Object detection in the browser (RF-DETR Nano by default).
+ *
+ * @example
+ * ```ts
+ * import { ObjectDetector } from 'runonweb/detect'
+ *
+ * const detector = new ObjectDetector()
+ * await detector.load()
+ * const objects = await detector.detect(imageFile)
+ * // [{ label: 'cat', score: 0.98, box: { xmin, ymin, xmax, ymax } }]
+ * ```
+ */
+export class ObjectDetector {
+  #model: string
+  #device: Device
+  #threshold: number
+  #normalized: boolean
+  #onProgress?: ProgressCallback
+  #pipe: DetectionPipe | null = null
+  #loading: Promise<void> | null = null
+  #resolvedDevice: 'webgpu' | 'wasm' | null = null
+
+  constructor(options: DetectOptions = {}) {
+    this.#model = options.model ?? DEFAULT_MODEL
+    this.#device = options.device ?? 'auto'
+    this.#threshold = options.threshold ?? 0.5
+    this.#normalized = options.normalized ?? false
+    this.#onProgress = options.onProgress
+  }
+
+  get device(): 'webgpu' | 'wasm' | null {
+    return this.#resolvedDevice
+  }
+
+  async load(): Promise<void> {
+    if (this.#pipe) return
+    if (this.#loading) return this.#loading
+
+    this.#loading = (async () => {
+      const { pipe, device } = await loadPipeline({
+        task: 'object-detection',
+        model: this.#model,
+        device: this.#device,
+        // These ONNX exports collapse logits on WebGPU with the current runtime.
+        ...(isSigmoidModelId(this.#model)
+          ? { supportedDevices: ['wasm'] as const, dtype: 'q8' as const }
+          : { dtype: (d: 'webgpu' | 'wasm') => (d === 'webgpu' ? 'fp16' : 'q8') }),
+        onProgress: this.#onProgress,
+      })
+      this.#pipe = pipe as DetectionPipe
+      this.#resolvedDevice = device
+    })()
+
+    try {
+      await this.#loading
+    } finally {
+      this.#loading = null
+    }
+  }
+
+  async detect(image: DetectImageInput): Promise<Detection[]> {
+    await this.load()
+    if (!this.#pipe) throw new Error('ObjectDetector model failed to load')
+
+    this.#onProgress?.({ status: 'processing' })
+    const input = await imageToPipelineInput(image)
+    const detections = usesSigmoid(this.#pipe)
+      ? await detectSigmoid(this.#pipe, input, this.#threshold, this.#normalized)
+      : await detectSoftmax(this.#pipe, input, this.#threshold, this.#normalized)
+
+    this.#onProgress?.({ status: 'done' })
+    return detections
+  }
+
+  dispose(): void {
+    const pipe = this.#pipe
+    this.#pipe = null
+    this.#resolvedDevice = null
+    void pipe?.dispose?.()
+  }
+}
+
+export async function detect(
+  image: DetectImageInput,
+  options?: DetectOptions
+): Promise<Detection[]> {
+  const detector = new ObjectDetector(options)
+  try {
+    return await detector.detect(image)
+  } finally {
+    detector.dispose()
+  }
+}
+
+export type { ProgressInfo, Device }
+
+type DetectionPipe = CallableFunction & {
+  dispose?: () => Promise<void>
+  model?: {
+    config?: { model_type?: string; id2label?: Record<string, string> }
+    (inputs: Record<string, unknown>): Promise<{ logits: DetTensor; pred_boxes: DetTensor }>
+  }
+  processor?: (images: unknown) => Promise<{
+    pixel_values: unknown
+    pixel_mask?: unknown
+    original_sizes?: Array<[number, number]>
+  }>
+}
+
+type DetTensor = {
+  dims: number[]
+  data: ArrayLike<number>
+  [index: number]: DetTensor
+}
+
+function usesSigmoid(pipe: DetectionPipe): boolean {
+  const type = pipe.model?.config?.model_type
+  return Boolean(type && SIGMOID_TYPES.has(type) && pipe.model && pipe.processor)
+}
+
+async function detectSoftmax(
+  pipe: DetectionPipe,
+  input: unknown,
+  threshold: number,
+  normalized: boolean
+): Promise<Detection[]> {
+  const raw = (await pipe(input, {
+    threshold,
+    percentage: normalized,
+  })) as Array<{ label: string; score: number; box: DetectionBox }>
+  return raw.map((d) => ({ label: d.label, score: d.score, box: d.box }))
+}
+
+async function detectSigmoid(
+  pipe: DetectionPipe,
+  input: unknown,
+  threshold: number,
+  normalized: boolean
+): Promise<Detection[]> {
+  const images = Array.isArray(input) ? input : [input]
+  const processed = await pipe.processor!(images)
+  const output = await pipe.model!({
+    pixel_values: processed.pixel_values,
+    pixel_mask: processed.pixel_mask,
+  })
+
+  const size = normalized
+    ? null
+    : sizeFrom(input, processed.original_sizes)
+  return decodeSigmoid(output.logits, output.pred_boxes, {
+    threshold,
+    id2label: pipe.model!.config?.id2label ?? {},
+    size,
+  })
+}
+
+function sizeFrom(
+  input: unknown,
+  originalSizes?: Array<[number, number]>
+): { width: number; height: number } | null {
+  if (originalSizes?.[0]) {
+    const [height, width] = originalSizes[0]
+    return { width, height }
+  }
+  if (input && typeof input === 'object' && 'width' in input && 'height' in input) {
+    const { width, height } = input as { width: number; height: number }
+    if (width && height) return { width, height }
+  }
+  return null
+}
+
+function decodeSigmoid(
+  logits: DetTensor,
+  boxes: DetTensor,
+  options: {
+    threshold: number
+    id2label: Record<string, string>
+    size: { width: number; height: number } | null
+  }
+): Detection[] {
+  const dims = logits.dims
+  const numQueries = dims.length === 3 ? dims[1]! : dims[0]!
+  const numClasses = dims.length === 3 ? dims[2]! : dims[1]!
+  const batch = dims.length === 3 ? logits[0]! : logits
+  const boxBatch = dims.length === 3 ? boxes[0]! : boxes
+  const labels = labelList(options.id2label, numClasses)
+
+  const out: Detection[] = []
+  for (let j = 0; j < numQueries; j++) {
+    const scores = batch[j]!.data
+    let best = -Infinity
+    let bestIdx = 0
+    for (let k = 0; k < numClasses; k++) {
+      const p = 1 / (1 + Math.exp(-scores[k]!))
+      if (p > best) {
+        best = p
+        bestIdx = k
+      }
+    }
+    if (best < options.threshold) continue
+    const label = labels[bestIdx]
+    if (!label) continue
+
+    const [cx, cy, bw, bh] = Array.from(boxBatch[j]!.data)
+    let xmin = cx! - bw! / 2
+    let ymin = cy! - bh! / 2
+    let xmax = cx! + bw! / 2
+    let ymax = cy! + bh! / 2
+    if (options.size) {
+      xmin *= options.size.width
+      xmax *= options.size.width
+      ymin *= options.size.height
+      ymax *= options.size.height
+    }
+    out.push({
+      label,
+      score: best,
+      box: { xmin, ymin, xmax, ymax },
+    })
+  }
+  return out
+}
+
+function labelList(id2label: Record<string, string>, numClasses: number): Array<string | undefined> {
+  const byId = Object.entries(id2label)
+    .map(([id, name]) => [Number(id), name] as const)
+    .sort((a, b) => a[0] - b[0])
+  if (byId.length === numClasses) return byId.map(([, name]) => name)
+
+  const sparse: Array<string | undefined> = Array.from({ length: numClasses })
+  for (const [id, name] of byId) {
+    if (id >= 0 && id < numClasses) sparse[id] = name
+  }
+  return sparse
+}

package/src/embed/index.ts

@@ -0,0 +1,172 @@
+import type { Device, ProgressCallback, ProgressInfo } from '../core/index.ts'
+import { loadPipeline } from '../core/pipeline.ts'
+
+const DEFAULT_MODEL = 'Xenova/all-MiniLM-L6-v2'
+
+export type EmbedOptions = {
+  model?: string
+  device?: Device
+  /** Pooling strategy. Default `mean`. */
+  pooling?: 'mean' | 'cls' | 'none'
+  /** L2-normalize vectors. Default true. */
+  normalize?: boolean
+  onProgress?: ProgressCallback
+}
+
+export type EmbedResult = {
+  /** Embedding vector(s). One vector if input was a string; one per item if array. */
+  embeddings: Float32Array[]
+  dimensions: number
+}
+
+/**
+ * Text embeddings in the browser (sentence-transformers style).
+ *
+ * @example
+ * ```ts
+ * import { TextEmbedder } from 'runonweb/embed'
+ *
+ * const embedder = new TextEmbedder()
+ * await embedder.load()
+ * const { embeddings } = await embedder.embed('hello world')
+ * const sim = cosineSimilarity(embeddings[0], other)
+ * ```
+ */
+export class TextEmbedder {
+  #model: string
+  #device: Device
+  #pooling: 'mean' | 'cls' | 'none'
+  #normalize: boolean
+  #onProgress?: ProgressCallback
+  #pipe: (CallableFunction & { dispose?: () => Promise<void> }) | null = null
+  #loading: Promise<void> | null = null
+  #resolvedDevice: 'webgpu' | 'wasm' | null = null
+
+  constructor(options: EmbedOptions = {}) {
+    this.#model = options.model ?? DEFAULT_MODEL
+    this.#device = options.device ?? 'auto'
+    this.#pooling = options.pooling ?? 'mean'
+    this.#normalize = options.normalize ?? true
+    this.#onProgress = options.onProgress
+  }
+
+  get device(): 'webgpu' | 'wasm' | null {
+    return this.#resolvedDevice
+  }
+
+  async load(): Promise<void> {
+    if (this.#pipe) return
+    if (this.#loading) return this.#loading
+
+    this.#loading = (async () => {
+      const { pipe, device } = await loadPipeline({
+        task: 'feature-extraction',
+        model: this.#model,
+        device: this.#device,
+        // fp16 on WebGPU (~45 MB); q8 on WASM (~23 MB).
+        // q8 on WebGPU produces garbage similarities. Never use it there.
+        dtype: (d) => (d === 'webgpu' ? 'fp16' : 'q8'),
+        onProgress: this.#onProgress,
+      })
+      this.#pipe = pipe
+      this.#resolvedDevice = device
+    })()
+
+    try {
+      await this.#loading
+    } finally {
+      this.#loading = null
+    }
+  }
+
+  async embed(text: string | string[]): Promise<EmbedResult> {
+    await this.load()
+    if (!this.#pipe) throw new Error('TextEmbedder model failed to load')
+
+    this.#onProgress?.({ status: 'embedding' })
+    const inputs = Array.isArray(text) ? text : [text]
+    if (inputs.length === 0) throw new Error('No text provided')
+
+    const embeddings: Float32Array[] = []
+    for (const item of inputs) {
+      const output = await this.#pipe(item, {
+        pooling: this.#pooling,
+        normalize: this.#normalize,
+      })
+      // Transformers.js returns a Tensor-like with .data or nested arrays
+      const vector = tensorToFloat32(output)
+      embeddings.push(vector)
+    }
+
+    this.#onProgress?.({ status: 'done' })
+    return {
+      embeddings,
+      dimensions: embeddings[0]?.length ?? 0,
+    }
+  }
+
+  dispose(): void {
+    const pipe = this.#pipe
+    this.#pipe = null
+    this.#resolvedDevice = null
+    void pipe?.dispose?.()
+  }
+}
+
+export async function embed(
+  text: string | string[],
+  options?: EmbedOptions
+): Promise<EmbedResult> {
+  const embedder = new TextEmbedder(options)
+  try {
+    return await embedder.embed(text)
+  } finally {
+    embedder.dispose()
+  }
+}
+
+/** Cosine similarity between two equal-length vectors. */
+export function cosineSimilarity(a: Float32Array, b: Float32Array): number {
+  if (a.length !== b.length) throw new Error('Vector length mismatch')
+  let dot = 0
+  let na = 0
+  let nb = 0
+  for (let i = 0; i < a.length; i++) {
+    const x = a[i]!
+    const y = b[i]!
+    dot += x * y
+    na += x * x
+    nb += y * y
+  }
+  const denom = Math.sqrt(na) * Math.sqrt(nb)
+  return denom === 0 ? 0 : dot / denom
+}
+
+export type { ProgressInfo, Device }
+
+function tensorToFloat32(output: unknown): Float32Array {
+  if (output instanceof Float32Array) return output
+  if (output && typeof output === 'object') {
+    const o = output as { data?: ArrayLike<number>; tolist?: () => unknown }
+    if (o.data) return new Float32Array(o.data as ArrayLike<number>)
+    if (typeof o.tolist === 'function') {
+      const list = o.tolist()
+      return new Float32Array(flattenNumbers(list))
+    }
+  }
+  if (Array.isArray(output)) {
+    return new Float32Array(flattenNumbers(output))
+  }
+  throw new Error('Unexpected embedding output shape')
+}
+
+function flattenNumbers(value: unknown): number[] {
+  if (typeof value === 'number') return [value]
+  if (!Array.isArray(value)) return []
+  const out: number[] = []
+  for (const item of value) {
+    if (typeof item === 'number') out.push(item)
+    else out.push(...flattenNumbers(item))
+  }
+  return out
+}

package/src/emoji/index.ts

@@ -0,0 +1,132 @@
+import type { Device, ProgressCallback, ProgressInfo } from '../core/index.ts'
+import { loadPipeline } from '../core/pipeline.ts'
+
+/**
+ * Default weights: `text2emoji-tiny`, a 2.4M-parameter T5 trained from scratch for runonweb
+ * on the Text2Emoji dataset. 3.9 MB as q8 (encoder 1.7 MB + decoder 2.1 MB), MIT.
+ * Trained with `training/text2emoji` in the runonweb repo.
+ */
+export const DEFAULT_MODEL = 'text2emoji-tiny'
+
+export type EmojiOptions = {
+  /** Model folder name (under `modelPath`) or Hugging Face repo id. */
+  model?: string
+  /**
+   * Base URL the weights are served from, e.g. `/models/`. The runonweb site self-hosts them
+   * in `public/models/`. Omit to download `model` from the Hugging Face Hub.
+   */
+  modelPath?: string
+  device?: Device
+  /** Maximum number of emojis to generate. Default 12. */
+  maxEmojis?: number
+  onProgress?: ProgressCallback
+}
+
+export type EmojiResult = {
+  /** Emojis joined without separators, e.g. "🍕❤️🐶". */
+  text: string
+  /** One entry per emoji (grapheme cluster). */
+  emojis: string[]
+}
+
+/**
+ * Text → emoji translation in the browser with a 4 MB model.
+ *
+ * @example
+ * ```ts
+ * import { Emojifier } from 'runonweb/emoji'
+ *
+ * const emojifier = new Emojifier({ modelPath: '/models/' })
+ * await emojifier.load()
+ * const { text } = await emojifier.emojify('I love pizza and my dog')
+ * // "🍕❤️🐶"
+ * ```
+ */
+export class Emojifier {
+  #model: string
+  #modelPath?: string
+  #device: Device
+  #maxEmojis: number
+  #onProgress?: ProgressCallback
+  #pipe: (CallableFunction & { dispose?: () => Promise<void> }) | null = null
+  #loading: Promise<void> | null = null
+  #resolvedDevice: 'webgpu' | 'wasm' | null = null
+
+  constructor(options: EmojiOptions = {}) {
+    this.#model = options.model ?? DEFAULT_MODEL
+    this.#modelPath = options.modelPath
+    this.#device = options.device ?? 'auto'
+    this.#maxEmojis = options.maxEmojis ?? 12
+    this.#onProgress = options.onProgress
+  }
+
+  get device(): 'webgpu' | 'wasm' | null {
+    return this.#resolvedDevice
+  }
+
+  async load(): Promise<void> {
+    if (this.#pipe) return
+    if (this.#loading) return this.#loading
+
+    this.#loading = (async () => {
+      const { pipe, device } = await loadPipeline({
+        task: 'text2text-generation',
+        model: this.#model,
+        modelPath: this.#modelPath,
+        device: this.#device,
+        // q8 everywhere: the whole model is 3.9 MB and encoder-decoder graphs fail on WebGPU.
+        dtype: 'q8',
+        supportedDevices: ['wasm'],
+        onProgress: this.#onProgress,
+      })
+      this.#pipe = pipe
+      this.#resolvedDevice = device
+    })()
+
+    try {
+      await this.#loading
+    } finally {
+      this.#loading = null
+    }
+  }
+
+  /** Translate a sentence (English) into a short emoji sequence. */
+  async emojify(text: string): Promise<EmojiResult> {
+    await this.load()
+    if (!this.#pipe) throw new Error('Emoji model failed to load')
+    if (!text.trim()) throw new Error('Text is empty')
+
+    this.#onProgress?.({ status: 'translating' })
+
+    const raw = (await this.#pipe(text, {
+      max_new_tokens: this.#maxEmojis + 1,
+      // Each emoji is one token; never emit the same emoji twice.
+      no_repeat_ngram_size: 1,
+    })) as Array<{ generated_text: string }> | { generated_text: string }
+
+    const out = Array.isArray(raw) ? raw[0] : raw
+    // The tokenizer separates emojis with spaces; drop them and split back into graphemes.
+    const emojis = (out?.generated_text ?? '').split(/\s+/).filter(Boolean)
+
+    this.#onProgress?.({ status: 'done' })
+    return { text: emojis.join(''), emojis }
+  }
+
+  dispose(): void {
+    const pipe = this.#pipe
+    this.#pipe = null
+    this.#resolvedDevice = null
+    void pipe?.dispose?.()
+  }
+}
+
+export async function emojify(text: string, options?: EmojiOptions): Promise<EmojiResult> {
+  const emojifier = new Emojifier(options)
+  try {
+    return await emojifier.emojify(text)
+  } finally {
+    emojifier.dispose()
+  }
+}
+
+export type { ProgressInfo, Device }