react-native-nobodywho 2.5.0 → 4.0.0

package/package.json

@@ -1,21 +1,25 @@
 {
   "name": "react-native-nobodywho",
-  "version": "2.5.0",
-  "description": "Run LLMs locally with offline inference — React Native bindings",
+  "version": "4.0.0",
+  "description": "Run LLMs locally and offline with React Native & Expo - Tool calling, RAG, Speech-to-Text, Text-to-Speech, and VAD",
   "main": "src/wrapper.ts",
   "types": "src/wrapper.ts",
   "exports": {
     ".": "./src/wrapper.ts"
   },
   "license": "EUPL-1.2",
-  "homepage": "https://nobodywho.ooo",
+  "homepage": "https://www.nobodywho.ai",
   "bugs": "https://github.com/nobodywho-ooo/nobodywho/issues",
   "keywords": [
     "react-native",
+    "expo",
     "ai",
     "llm",
     "local-ai",
     "inference",
+    "stt",
+    "tts",
+    "vad",
     "gguf",
     "on-device",
     "offline"

package/src/chat.ts

@@ -10,6 +10,18 @@ import { TokenStream } from "./streaming";
 import type { Prompt } from "./prompt";
 import type { Tool } from "./tool";
 
+/**
+ * Settings to apply before a `complete` turn. An omitted field keeps what the
+ * chat has; a set one stays set, like a leading system message.
+ */
+export type Options = {
+  sampler?: SamplerConfig;
+  /** Replaces the chat's template variables wholesale. */
+  templateVariables?: Record<string, boolean>;
+  /** Re-selects the chat template, so the turn re-prefills from near token zero. */
+  tools?: Tool[];
+};
+
 /**
  * A chat session for local LLM inference.
  *
@@ -107,6 +119,29 @@ export class Chat {
     return new TokenStream(this._inner.askWithPrompt(message._parts!));
   }
 
+  /**
+   * Answer a full list of messages, replacing the chat history.
+   *
+   * The list is the whole conversation, used as given: it must be non-empty, end
+   * in a user or tool message, and carry a system message only first. That system
+   * message sets the chat's system prompt; leave it out and the prompt already on
+   * the chat is kept. The response is appended, and the next `ask` continues from
+   * there.
+   *
+   * `options` follows the same rule for the chat's other settings.
+   */
+  complete(messages: Message[], options: Options = {}): TokenStream {
+    return new TokenStream(
+      this._inner.complete(messages.map(toInternal), {
+        sampler: options.sampler,
+        templateVariables: options.templateVariables
+          ? new Map(Object.entries(options.templateVariables))
+          : undefined,
+        tools: options.tools?.map((t) => t._inner),
+      }),
+    );
+  }
+
   /** Stop the current generation. */
   stopGeneration(): void {
     this._inner.stopGeneration();

package/src/encoder.ts

@@ -46,6 +46,11 @@ export class Encoder {
     return this._inner.encode(text);
   }
 
+  /** Encode multiple texts, preserving input order. */
+  async encodeBatch(texts: string[]): Promise<number[][]> {
+    return this._inner.encodeBatch(texts);
+  }
+
   /**
    * Immediately free the underlying Rust resources.
    * After calling this, the Encoder instance is no longer usable.

package/src/message.ts

@@ -1,19 +1,45 @@
 import {
+  ContentPart as InternalContentPart,
+  ContentPart_Tags,
   Message as InternalMessage,
   Message_Tags,
-  type Asset,
+  MessageContent as InternalMessageContent,
+  MessageContent_Tags,
   type ToolCall,
 } from "../generated/ts/nobodywho";
 
+/** One piece of a message: a run of text, or a media file at this position. */
+export type ContentPart =
+  | { type: "text"; text: string }
+  | { type: "image"; path: string }
+  | { type: "audio"; path: string };
+
+/**
+ * Message content: plain text, interleaved text and media, or a raw JSON value
+ * handed straight to the chat template.
+ *
+ * @example
+ * ```typescript
+ * const content: Content = [
+ *   { type: "text", text: "What is in this image?" },
+ *   { type: "image", path: "./dog.png" },
+ * ];
+ * ```
+ */
+export type Content = string | ContentPart[] | { type: "json"; json: string };
+
 /**
  * A chat message. The variant determines the message type:
  *
- * - **User message:** `{ role: "user", content, assets? }`
+ * - **User message:** `{ role: "user", content }`
  * - **Assistant message:** `{ role: "assistant", content }`
  * - **Assistant tool call:** `{ role: "assistant", content, toolCalls }`
  * - **System message:** `{ role: "system", content }`
  * - **Tool response:** `{ role: "tool", name, content }`
  *
+ * `content` is a string, a list of parts for multimodal input, or a
+ * `{ type: "json", json }` wrapper for raw template passthrough.
+ *
  * @example
  * ```typescript
  * const history = await chat.getChatHistory();
@@ -29,33 +55,75 @@ import {
  * ```
  */
 export type Message =
-  | { role: "user"; content: string; assets?: Asset[] }
-  | { role: "assistant"; content: string }
-  | { role: "assistant"; content: string; toolCalls: ToolCall[] }
-  | { role: "system"; content: string }
-  | { role: "tool"; name: string; content: string };
+  | { role: "user"; content: Content }
+  | { role: "assistant"; content: Content }
+  | { role: "assistant"; content: Content; toolCalls: ToolCall[] }
+  | { role: "system"; content: Content }
+  | { role: "tool"; name: string; content: Content };
+
+function partFromInternal(part: InternalContentPart): ContentPart {
+  if (part.tag === ContentPart_Tags.Text) {
+    return { type: "text", text: part.inner.text };
+  } else if (part.tag === ContentPart_Tags.Image) {
+    return { type: "image", path: part.inner.path };
+  } else {
+    return { type: "audio", path: part.inner.path };
+  }
+}
+
+function partToInternal(part: ContentPart): InternalContentPart {
+  if (part.type === "text") {
+    return new InternalContentPart.Text({ text: part.text });
+  } else if (part.type === "image") {
+    return new InternalContentPart.Image({ path: part.path });
+  } else {
+    return new InternalContentPart.Audio({ path: part.path });
+  }
+}
+
+/** @internal */
+export function contentFromInternal(content: InternalMessageContent): Content {
+  if (content.tag === MessageContent_Tags.Text) {
+    return content.inner.text;
+  } else if (content.tag === MessageContent_Tags.Parts) {
+    return content.inner.parts.map(partFromInternal);
+  } else {
+    return { type: "json", json: content.inner.json };
+  }
+}
+
+/** @internal */
+export function contentToInternal(content: Content): InternalMessageContent {
+  if (typeof content === "string") {
+    return new InternalMessageContent.Text({ text: content });
+  }
+  if (Array.isArray(content)) {
+    return new InternalMessageContent.Parts({
+      parts: content.map(partToInternal),
+    });
+  }
+  return new InternalMessageContent.Json({ json: content.json });
+}
 
 /** @internal Convert internal Message to Message */
 export function fromInternal(msg: InternalMessage): Message {
   if (msg.tag === Message_Tags.User) {
-    const { content, assets } = msg.inner;
-    return {
-      role: "user",
-      content,
-      ...(assets.length > 0 ? { assets } : {}),
-    };
+    return { role: "user", content: contentFromInternal(msg.inner.content) };
   } else if (msg.tag === Message_Tags.Assistant) {
     const { content, toolCalls } = msg.inner;
     if (toolCalls != null && toolCalls.length > 0) {
-      return { role: "assistant", content, toolCalls };
+      return {
+        role: "assistant",
+        content: contentFromInternal(content),
+        toolCalls,
+      };
     }
-    return { role: "assistant", content };
+    return { role: "assistant", content: contentFromInternal(content) };
   } else if (msg.tag === Message_Tags.System) {
-    const { content } = msg.inner;
-    return { role: "system", content };
+    return { role: "system", content: contentFromInternal(msg.inner.content) };
   } else {
     const { name, content } = msg.inner;
-    return { role: "tool", name, content };
+    return { role: "tool", name, content: contentFromInternal(content) };
   }
 }
 
@@ -63,27 +131,26 @@ export function fromInternal(msg: InternalMessage): Message {
 export function toInternal(msg: Message): InternalMessage {
   if (msg.role === "user") {
     return new InternalMessage.User({
-      content: msg.content,
-      assets: msg.assets ?? [],
+      content: contentToInternal(msg.content),
     });
   } else if (msg.role === "assistant" && "toolCalls" in msg) {
     return new InternalMessage.Assistant({
-      content: msg.content,
+      content: contentToInternal(msg.content),
       toolCalls: msg.toolCalls,
     });
   } else if (msg.role === "assistant") {
     return new InternalMessage.Assistant({
-      content: msg.content,
+      content: contentToInternal(msg.content),
       toolCalls: undefined,
     });
   } else if (msg.role === "system") {
     return new InternalMessage.System({
-      content: msg.content,
+      content: contentToInternal(msg.content),
     });
   } else {
     return new InternalMessage.Tool({
       name: msg.name,
-      content: msg.content,
+      content: contentToInternal(msg.content),
     });
   }
 }

package/src/prompt.ts

@@ -1,32 +1,32 @@
-import { PromptPart } from "../generated/ts/nobodywho";
+import { ContentPart } from "../generated/ts/nobodywho";
 
 /** A text part of a multimodal prompt. */
 class TextPart {
   /** @internal */
-  readonly _inner: PromptPart;
+  readonly _inner: ContentPart;
 
   constructor(content: string) {
-    this._inner = new PromptPart.Text({ content });
+    this._inner = new ContentPart.Text({ text: content });
   }
 }
 
 /** An image part of a multimodal prompt. */
 class ImagePart {
   /** @internal */
-  readonly _inner: PromptPart;
+  readonly _inner: ContentPart;
 
   constructor(path: string) {
-    this._inner = new PromptPart.Image({ path });
+    this._inner = new ContentPart.Image({ path });
   }
 }
 
 /** An audio part of a multimodal prompt. */
 class AudioPart {
   /** @internal */
-  readonly _inner: PromptPart;
+  readonly _inner: ContentPart;
 
   constructor(path: string) {
-    this._inner = new PromptPart.Audio({ path });
+    this._inner = new ContentPart.Audio({ path });
   }
 }
 
@@ -53,7 +53,7 @@ type Part = TextPart | ImagePart | AudioPart;
  */
 export class Prompt {
   /** @internal */
-  readonly _parts: PromptPart[] | null;
+  readonly _parts: ContentPart[] | null;
   /** @internal */
   readonly _jsonString: string | null;
 

package/src/{stt.ts → speech_to_text.ts}

@@ -1,8 +1,8 @@
-import type { RustSttInterface } from "../generated/ts/nobodywho";
+import type { RustSpeechToTextInterface } from "../generated/ts/nobodywho";
 import * as nobodywho from "../generated/ts/nobodywho";
 import { TokenStream } from "./streaming";
 
-export type SttOptions = {
+export type SpeechToTextOptions = {
   source: string;
   language?: string;
   quantization?: string;
@@ -13,7 +13,7 @@ export type SttOptions = {
  *
  * @example
  * ```typescript
- * const stt = new STT({
+ * const stt = await SpeechToText.load({
  *   source: "hf://onnx-community/whisper-base",
  *   language: "en",
  * });
@@ -26,24 +26,32 @@ export type SttOptions = {
  * const text = await stream.completed();
  * ```
  */
-export class STT {
+export class SpeechToText {
   /** @internal */
-  private readonly _inner: RustSttInterface;
+  private readonly _inner: RustSpeechToTextInterface;
+
+  /** @internal */
+  private constructor(inner: RustSpeechToTextInterface) {
+    this._inner = inner;
+  }
 
   /**
-   * @param opts - See {@link SttOptions}.
+   * Load a Whisper SpeechToText model.
+   *
+   * @param opts - See {@link SpeechToTextOptions}.
    */
-  constructor(opts: SttOptions) {
-    this._inner = new nobodywho.RustStt(
+  static async load(opts: SpeechToTextOptions): Promise<SpeechToText> {
+    const inner = await nobodywho.loadSpeechToText(
       opts.source,
       opts.language,
       opts.quantization,
     );
+    return new SpeechToText(inner);
   }
 
   /**
    * Transcribe an audio file (WAV / MP3).
-   * Returns an `STTStream` to consume tokens as they arrive.
+   * Returns an `SpeechToTextStream` to consume tokens as they arrive.
    */
   transcribeFile(path: string): TokenStream {
     return new TokenStream(this._inner.transcribeFile(path));

package/src/{tts.ts → text_to_speech.ts}

@@ -1,15 +1,15 @@
 import {
-  RustTts,
-  loadTts,
-  type RustTtsInterface,
+  RustTextToSpeech,
+  loadTextToSpeech,
+  type RustTextToSpeechInterface,
 } from "../generated/ts/nobodywho";
 
-export type TtsArchitecture = "kokoro" | "pocket-tts" | "supertonic";
-export type TtsDevice = "auto" | "cpu" | "cuda";
+export type TextToSpeechArchitecture = "kokoro" | "pocket-tts" | "supertonic";
+export type TextToSpeechDevice = "auto" | "cpu" | "cuda";
 
-export type TtsOptions = {
+export type TextToSpeechOptions = {
   source: string;
-  architecture?: TtsArchitecture;
+  architecture?: TextToSpeechArchitecture;
   voice?: string;
   language?: string;
   speed?: number;
@@ -18,17 +18,17 @@ export type TtsOptions = {
   precision?: "int8" | "fp32";
   temperature?: number;
   huggingfaceToken?: string;
-  device?: TtsDevice;
+  device?: TextToSpeechDevice;
 };
 
 /** Text-to-speech synthesizer that returns WAV bytes. */
-export class Tts {
+export class TextToSpeech {
   /** @internal */
-  private _inner: RustTtsInterface;
+  private _inner: RustTextToSpeechInterface;
 
-  /** Create a TTS synthesizer synchronously. */
-  constructor(opts: TtsOptions) {
-    this._inner = new RustTts(
+  /** Create a TextToSpeech synthesizer synchronously. */
+  constructor(opts: TextToSpeechOptions) {
+    this._inner = new RustTextToSpeech(
       opts.source,
       opts.architecture,
       opts.voice,
@@ -43,15 +43,15 @@ export class Tts {
     );
   }
 
-  private static fromInner(inner: RustTtsInterface): Tts {
-    const tts = Object.create(Tts.prototype) as Tts;
+  private static fromInner(inner: RustTextToSpeechInterface): TextToSpeech {
+    const tts = Object.create(TextToSpeech.prototype) as TextToSpeech;
     tts._inner = inner;
     return tts;
   }
 
-  /** Create a TTS synthesizer asynchronously. */
-  static async load(opts: TtsOptions): Promise<Tts> {
-    const inner = await loadTts(
+  /** Create a TextToSpeech synthesizer asynchronously. */
+  static async load(opts: TextToSpeechOptions): Promise<TextToSpeech> {
+    const inner = await loadTextToSpeech(
       opts.source,
       opts.architecture,
       opts.voice,
@@ -64,7 +64,7 @@ export class Tts {
       opts.huggingfaceToken,
       opts.device ?? "auto",
     );
-    return Tts.fromInner(inner);
+    return TextToSpeech.fromInner(inner);
   }
 
   /** Synthesize text and return WAV bytes. */
@@ -79,7 +79,7 @@ export class Tts {
 
   /**
    * Immediately free the underlying Rust resources.
-   * After calling this, the Tts instance is no longer usable.
+   * After calling this, the TextToSpeech instance is no longer usable.
    */
   destroy(): void {
     (this._inner as any).uniffiDestroy();

package/src/vad.ts

@@ -0,0 +1,104 @@
+import type { RustVoiceActivityDetectionInterface, VoiceActivityDetectionEvent } from "../generated/ts/nobodywho";
+import * as nobodywho from "../generated/ts/nobodywho";
+
+export { VoiceActivityDetectionEvent } from "../generated/ts/nobodywho";
+
+export type VoiceActivityDetectionOptions = {
+  source?: string;
+  sampleRate: number;
+  threshold?: number;
+  minSilenceDurationMs?: number;
+  minSpeechDurationMs?: number;
+  prerollDurationMs?: number;
+};
+
+/**
+ * Voice activity detection from live, streaming audio, backed by Silero VAD.
+ *
+ * @example
+ * ```typescript
+ * const vad = await VoiceActivityDetection.load({ sampleRate: 16000 });
+ *
+ * // Feed each newest chunk as it arrives (not the whole buffer — VoiceActivityDetection
+ * // tracks the current turn internally).
+ * if (vad.push(chunk) === VoiceActivityDetectionEvent.SpeechEnded) {
+ *   const audio = vad.finish();
+ *   // audio: Int16Array-like number[] spanning SpeechStarted (with a
+ *   // small pre-roll) through SpeechEnded.
+ * }
+ * ```
+ */
+export class VoiceActivityDetection {
+  /** @internal */
+  private readonly _inner: RustVoiceActivityDetectionInterface;
+
+  /** @internal */
+  private constructor(inner: RustVoiceActivityDetectionInterface) {
+    this._inner = inner;
+  }
+
+  /**
+   * Load a voice activity detector.
+   *
+   * @param opts - See {@link VoiceActivityDetectionOptions}.
+   */
+  static async load(opts: VoiceActivityDetectionOptions): Promise<VoiceActivityDetection> {
+    const inner = await nobodywho.loadVoiceActivityDetection(
+      opts.source,
+      opts.sampleRate,
+      opts.threshold,
+      opts.minSilenceDurationMs,
+      opts.minSpeechDurationMs,
+      opts.prerollDurationMs,
+      undefined,
+    );
+    return new VoiceActivityDetection(inner);
+  }
+
+  /**
+   * Feed the newest chunk of audio (not the whole accumulated buffer —
+   * `VoiceActivityDetection` tracks the current turn internally). Always
+   * returns the current confirmed state: `Speech`/`Silence` if unchanged
+   * since the last call, or `SpeechStarted`/`SpeechEnded` on the call that
+   * confirmed the transition.
+   *
+   * @param chunk - Flat array of signed 16-bit samples (mono).
+   */
+  push(chunk: Int16Array | number[]): VoiceActivityDetectionEvent {
+    const arr = chunk instanceof Int16Array ? Array.from(chunk) : chunk;
+    return this._inner.push(arr);
+  }
+
+  /**
+   * Return the current turn's captured audio (from the confirmed
+   * `SpeechStarted`, including a small pre-roll, through to `SpeechEnded`)
+   * and reset internal state for the next turn. Empty if speech was never
+   * confirmed.
+   */
+  finish(): number[] {
+    return this._inner.finish();
+  }
+
+  /**
+   * Detect every speech segment in a complete audio buffer, returning
+   * each segment's audio (with a short pre-roll) in order. Unlike `push`,
+   * correctly finds every segment regardless of buffer size — use this
+   * for offline/batch processing instead of live streaming.
+   *
+   * @example
+   * ```typescript
+   * for (const audio of vad.segment(fullRecording)) {
+   *   transcribe(audio);
+   * }
+   * ```
+   */
+  segment(samples: Int16Array | number[]): number[][] {
+    const arr = samples instanceof Int16Array ? Array.from(samples) : samples;
+    return this._inner.segment(arr);
+  }
+
+  /** Release native resources. Call when done with this handle. */
+  destroy(): void {
+    (this._inner as any).uniffiDestroy?.();
+  }
+}

package/src/wrapper.ts

@@ -2,9 +2,9 @@
  * NobodyWho — React Native bindings for local LLM inference.
  *
  * This is the public entry point. It re-exports the public API:
- * - Wrapper classes: Model, Chat, Encoder, CrossEncoder, Tts, Tool, TokenStream, Prompt
+ * - Wrapper classes: Model, Chat, Encoder, CrossEncoder, TextToSpeech, Tool, TokenStream, Prompt
  * - Direct re-exports: SamplerBuilder, SamplerConfig
- * - Types: Message, Asset, ToolCall
+ * - Types: Message, Content, ContentPart, ToolCall
  * - Utilities: SamplerPresets, cosineSimilarity, downloadModel
  *
  * This file is NOT generated — it is safe to edit.
@@ -57,16 +57,18 @@ export async function downloadModel(opts: {
 // Wrapper classes
 export { Model } from "./model";
 export { Chat } from "./chat";
-export { STT } from "./stt";
+export type { Options } from "./chat";
+export { SpeechToText } from "./speech_to_text";
+export { VoiceActivityDetection } from "./vad";
 export { Encoder } from "./encoder";
 export { CrossEncoder } from "./cross_encoder";
-export { Tts } from "./tts";
+export { TextToSpeech } from "./text_to_speech";
 export { Prompt } from "./prompt";
 export { Tool } from "./tool";
 export { TokenStream } from "./streaming";
 
 // Message types
-export type { Message } from "./message";
+export type { Message, Content, ContentPart } from "./message";
 
 // Re-export types that don't need wrapping.
 //
@@ -80,13 +82,14 @@ export {
 } from "../generated/ts/nobodywho";
 
 export type {
-  Asset,
   ToolCall,
   CachedModel,
   ChatStats,
 } from "../generated/ts/nobodywho";
-export type { TtsArchitecture, TtsDevice, TtsOptions } from "./tts";
-export type { SttOptions } from "./stt";
+export type { TextToSpeechArchitecture, TextToSpeechDevice, TextToSpeechOptions } from "./text_to_speech";
+export type { SpeechToTextOptions } from "./speech_to_text";
+export { VoiceActivityDetectionEvent } from "./vad";
+export type { VoiceActivityDetectionOptions } from "./vad";
 
 // Ergonomic wrapper additions.
 export { SamplerPresets } from "./sampler_presets";