react-native-nobodywho 2.3.0 → 2.5.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "react-native-nobodywho",
-  "version": "2.3.0",
+  "version": "2.5.0",
   "description": "Run LLMs locally with offline inference — React Native bindings",
   "main": "src/wrapper.ts",
   "types": "src/wrapper.ts",

package/src/chat.ts

@@ -1,6 +1,8 @@
 import {
   RustChat,
   SamplerConfig,
+  MtpConfig,
+  type ChatStats,
 } from "../generated/ts/nobodywho";
 import { Model } from "./model";
 import { type Message, fromInternal, toInternal } from "./message";
@@ -30,6 +32,12 @@ export class Chat {
   /** @internal */
   private readonly _inner: RustChat;
 
+  /**
+   * `threadCount` is the number of CPU threads used for inference. Omit it to detect the
+   * device's physical core count (performance cores only, on Apple silicon) — hyperthreads
+   * and efficiency cores make inference slower, not faster. Lower it to leave CPU headroom
+   * for the rest of the app.
+   */
   constructor(opts: {
     model: Model;
     systemPrompt?: string;
@@ -37,6 +45,8 @@ export class Chat {
     templateVariables?: Record<string, boolean>;
     tools?: Tool[];
     sampler?: SamplerConfig;
+    mtp?: Partial<MtpConfig>;
+    threadCount?: number;
   }) {
     this._inner = new RustChat(
       opts.model._inner,
@@ -45,6 +55,8 @@ export class Chat {
       opts.templateVariables ? new Map(Object.entries(opts.templateVariables)) : undefined,
       opts.tools?.map((t) => t._inner) ?? undefined,
       opts.sampler ?? undefined,
+      opts.mtp !== undefined ? MtpConfig.create(opts.mtp) : undefined,
+      opts.threadCount ?? undefined,
     );
   }
 
@@ -64,17 +76,21 @@ export class Chat {
     modelPath: string;
     useGpu?: boolean;
     projectionModelPath?: string;
+    draftModelPath?: string;
     systemPrompt?: string;
     contextSize?: number;
     templateVariables?: Record<string, boolean>;
     tools?: Tool[];
     sampler?: SamplerConfig;
+    mtp?: Partial<MtpConfig>;
+    threadCount?: number;
     onDownloadProgress?: (downloaded: number, total: number) => void;
   }): Promise<Chat> {
     const model = await Model.load({
       modelPath: opts.modelPath,
       useGpu: opts.useGpu,
       projectionModelPath: opts.projectionModelPath,
+      draftModelPath: opts.draftModelPath,
       onDownloadProgress: opts.onDownloadProgress,
     });
     return new Chat({ model, ...opts });
@@ -85,7 +101,10 @@ export class Chat {
     if (typeof message === "string") {
       return new TokenStream(this._inner.ask(message));
     }
-    return new TokenStream(this._inner.askWithPrompt(message._parts));
+    if (message._jsonString !== null) {
+      return new TokenStream(this._inner.askWithJsonPrompt(message._jsonString!));
+    }
+    return new TokenStream(this._inner.askWithPrompt(message._parts!));
   }
 
   /** Stop the current generation. */
@@ -125,6 +144,15 @@ export class Chat {
     return this._inner.getSystemPrompt();
   }
 
+  /** Tokenize a text message or multimodal prompt and return the token IDs.
+   * Each element is a token ID (number) for text, or null for image/audio embedding slots. */
+  async tokenize(message: string | Prompt): Promise<(number | null)[]> {
+    if (typeof message === "string") {
+      return this._inner.tokenize(message);
+    }
+    return this._inner.tokenizeWithPrompt(message._parts);
+  }
+
   /** Set the system prompt. */
   async setSystemPrompt(systemPrompt: string | undefined): Promise<void> {
     return this._inner.setSystemPrompt(systemPrompt);
@@ -155,6 +183,15 @@ export class Chat {
     return this._inner.getSamplerConfigJson();
   }
 
+  /** Get context usage statistics. */
+  async getStats(): Promise<ChatStats> {
+    return this._inner.getStats();
+  }
+
+  async mtpAcceptanceRate(): Promise<number | undefined> {
+    return this._inner.mtpAcceptanceRate();
+  }
+
   /**
    * Immediately free the underlying Rust resources (model context, KV cache, etc.).
    * After calling this, the Chat instance is no longer usable.

package/src/message.ts

@@ -74,7 +74,7 @@ export function toInternal(msg: Message): InternalMessage {
   } else if (msg.role === "assistant") {
     return new InternalMessage.Assistant({
       content: msg.content,
-      toolCalls: null,
+      toolCalls: undefined,
     });
   } else if (msg.role === "system") {
     return new InternalMessage.System({

package/src/model.ts

@@ -36,15 +36,19 @@ export class Model {
   /**
    * Load a GGUF model from a local path or remote URL.
    *
-   * @param opts.modelPath - Path to the GGUF model file, or a `hf://owner/repo/file.gguf` / `https://` URL to download and cache automatically
+   * @param opts.modelPath - Local path, `hf://` or `https://` URL, or `auto` for memory-based model selection
    * @param opts.useGpu - Use GPU acceleration if available (default: true)
    * @param opts.projectionModelPath - Path to a multimodal projector file for vision models
+   * @param opts.draftModelPath - Optional MTP draft-heads gguf. Loading it lets `Chat`
+   *   instances opt into MTP speculative decoding via `new Chat({ ..., mtp: true })`.
+   *   Adds around 5% to VRAM usage.
    * @param opts.onDownloadProgress - Invoked with `(downloaded, total)` byte counts while a remote model is being downloaded. Throttled to ~10 Hz with a guaranteed final emit on completion. Not invoked for cached/local files.
    */
   static async load(opts: {
     modelPath: string;
     useGpu?: boolean;
     projectionModelPath?: string;
+    draftModelPath?: string;
     onDownloadProgress?: (downloaded: number, total: number) => void;
   }): Promise<Model> {
     const cb = opts.onDownloadProgress;
@@ -55,11 +59,20 @@ export class Model {
       opts.modelPath,
       opts.useGpu ?? true,
       opts.projectionModelPath,
+      opts.draftModelPath,
       progressCallback,
     );
     return new Model(inner);
   }
 
+  /**
+   * The maximum context size this model was trained with.
+   * Use this to avoid setting `contextSize` above what the model supports.
+   */
+  get maxCtx(): number {
+    return Number((this._inner as any).maxCtx());
+  }
+
   /**
    * Immediately free the underlying Rust resources (model weights, GPU memory).
    * After calling this, the Model instance is no longer usable.

package/src/prompt.ts

@@ -45,14 +45,21 @@ type Part = TextPart | ImagePart | AudioPart;
  * ]);
  *
  * const stream = chat.ask(prompt);
+ *
+ * // JSON-structured prompt
+ * const jsonPrompt = Prompt.fromJson({ role: "user", content: "Hello" });
+ * const stream = chat.ask(jsonPrompt);
  * ```
  */
 export class Prompt {
   /** @internal */
-  readonly _parts: PromptPart[];
+  readonly _parts: PromptPart[] | null;
+  /** @internal */
+  readonly _jsonString: string | null;
 
   constructor(parts: Part[]) {
     this._parts = parts.map((p) => p._inner);
+    this._jsonString = null;
   }
 
   /** Create a text part. */
@@ -69,4 +76,11 @@ export class Prompt {
   static Audio(path: string): AudioPart {
     return new AudioPart(path);
   }
+
+  /** Create a prompt from any JSON-serializable value (object, array, string, etc.). */
+  static fromJson(data: unknown): Prompt {
+    const p: Prompt = Object.create(Prompt.prototype);
+    Object.assign(p, { _parts: null, _jsonString: JSON.stringify(data) });
+    return p;
+  }
 }

package/src/stt.ts

@@ -0,0 +1,69 @@
+import type { RustSttInterface } from "../generated/ts/nobodywho";
+import * as nobodywho from "../generated/ts/nobodywho";
+import { TokenStream } from "./streaming";
+
+export type SttOptions = {
+  source: string;
+  language?: string;
+  quantization?: string;
+};
+
+/**
+ * Speech-to-text using a local Whisper ONNX model.
+ *
+ * @example
+ * ```typescript
+ * const stt = new STT({
+ *   source: "hf://onnx-community/whisper-base",
+ *   language: "en",
+ * });
+ * for await (const piece of stt.transcribeFile("recording.mp3")) {
+ *   process.stdout.write(piece);
+ * }
+ *
+ * // From microphone (react-native-audio-pcm-stream gives Int16Array):
+ * const stream = stt.transcribePcm(samplesI16, 44100);
+ * const text = await stream.completed();
+ * ```
+ */
+export class STT {
+  /** @internal */
+  private readonly _inner: RustSttInterface;
+
+  /**
+   * @param opts - See {@link SttOptions}.
+   */
+  constructor(opts: SttOptions) {
+    this._inner = new nobodywho.RustStt(
+      opts.source,
+      opts.language,
+      opts.quantization,
+    );
+  }
+
+  /**
+   * Transcribe an audio file (WAV / MP3).
+   * Returns an `STTStream` to consume tokens as they arrive.
+   */
+  transcribeFile(path: string): TokenStream {
+    return new TokenStream(this._inner.transcribeFile(path));
+  }
+
+  /**
+   * Transcribe raw i16 PCM samples (e.g. from a microphone).
+   * The backend resamples to 16 kHz internally, so pass whatever rate your
+   * mic captures at (typically 44100 or 48000).
+   *
+   * @param samples - Flat array of signed 16-bit samples (mono).
+   * @param sampleRate - Capture rate in Hz.
+   */
+  transcribePcm(samples: Int16Array | number[], sampleRate: number): TokenStream {
+    const arr = samples instanceof Int16Array ? Array.from(samples) : samples;
+    return new TokenStream(this._inner.transcribePcm(arr, sampleRate));
+  }
+
+  /** Release native resources. Call when done with this handle. */
+  destroy(): void {
+    (this._inner as any).uniffiDestroy?.();
+  }
+}

package/src/tts.ts

@@ -0,0 +1,87 @@
+import {
+  RustTts,
+  loadTts,
+  type RustTtsInterface,
+} from "../generated/ts/nobodywho";
+
+export type TtsArchitecture = "kokoro" | "pocket-tts" | "supertonic";
+export type TtsDevice = "auto" | "cpu" | "cuda";
+
+export type TtsOptions = {
+  source: string;
+  architecture?: TtsArchitecture;
+  voice?: string;
+  language?: string;
+  speed?: number;
+  steps?: number;
+  silenceDuration?: number;
+  precision?: "int8" | "fp32";
+  temperature?: number;
+  huggingfaceToken?: string;
+  device?: TtsDevice;
+};
+
+/** Text-to-speech synthesizer that returns WAV bytes. */
+export class Tts {
+  /** @internal */
+  private _inner: RustTtsInterface;
+
+  /** Create a TTS synthesizer synchronously. */
+  constructor(opts: TtsOptions) {
+    this._inner = new RustTts(
+      opts.source,
+      opts.architecture,
+      opts.voice,
+      opts.language,
+      opts.speed,
+      opts.steps,
+      opts.silenceDuration,
+      opts.precision,
+      opts.temperature,
+      opts.huggingfaceToken,
+      opts.device ?? "auto",
+    );
+  }
+
+  private static fromInner(inner: RustTtsInterface): Tts {
+    const tts = Object.create(Tts.prototype) as Tts;
+    tts._inner = inner;
+    return tts;
+  }
+
+  /** Create a TTS synthesizer asynchronously. */
+  static async load(opts: TtsOptions): Promise<Tts> {
+    const inner = await loadTts(
+      opts.source,
+      opts.architecture,
+      opts.voice,
+      opts.language,
+      opts.speed,
+      opts.steps,
+      opts.silenceDuration,
+      opts.precision,
+      opts.temperature,
+      opts.huggingfaceToken,
+      opts.device ?? "auto",
+    );
+    return Tts.fromInner(inner);
+  }
+
+  /** Synthesize text and return WAV bytes. */
+  async synthesize(text: string): Promise<Uint8Array> {
+    return new Uint8Array(await this._inner.synthesizeAsync(text));
+  }
+
+  /** Synthesize text synchronously and return WAV bytes. */
+  synthesizeSync(text: string): Uint8Array {
+    return new Uint8Array(this._inner.synthesize(text));
+  }
+
+  /**
+   * Immediately free the underlying Rust resources.
+   * After calling this, the Tts instance is no longer usable.
+   */
+  destroy(): void {
+    (this._inner as any).uniffiDestroy();
+  }
+}

package/src/wrapper.ts

@@ -2,7 +2,7 @@
  * NobodyWho — React Native bindings for local LLM inference.
  *
  * This is the public entry point. It re-exports the public API:
- * - Wrapper classes: Model, Chat, Encoder, CrossEncoder, Tool, TokenStream, Prompt
+ * - Wrapper classes: Model, Chat, Encoder, CrossEncoder, Tts, Tool, TokenStream, Prompt
  * - Direct re-exports: SamplerBuilder, SamplerConfig
  * - Types: Message, Asset, ToolCall
  * - Utilities: SamplerPresets, cosineSimilarity, downloadModel
@@ -10,7 +10,23 @@
  * This file is NOT generated — it is safe to edit.
  */
 
-import { downloadModel as _downloadModel } from "../generated/ts/nobodywho";
+// Native crate install + bindings initialize. These run as side effects of
+// evaluating this module, which the React Native runtime is guaranteed to do
+// before any FFI call because wrapper.ts is the package main.
+//
+// The install logic was previously reached only through lazy re-exports from
+// ./index. In release mode Metro's inlineRequires defers lazy re-exports
+// until first use, so consumers whose first interaction was Model.load /
+// downloadModel / Chat never triggered ./index and FFI calls then threw
+// "Cannot read property 'ubrn_…' of undefined". Inlining the install here
+// removes the ./index indirection so this can't recur.
+import installer from "./NativeNobodywho";
+import nobodywhoDefault, {
+  downloadModel as _downloadModel,
+} from "../generated/ts/nobodywho";
+
+installer.installRustCrate();
+nobodywhoDefault.initialize();
 
 /**
  * Download a GGUF model from a remote URL or HuggingFace path and return the local file path.
@@ -41,8 +57,10 @@ export async function downloadModel(opts: {
 // Wrapper classes
 export { Model } from "./model";
 export { Chat } from "./chat";
+export { STT } from "./stt";
 export { Encoder } from "./encoder";
 export { CrossEncoder } from "./cross_encoder";
+export { Tts } from "./tts";
 export { Prompt } from "./prompt";
 export { Tool } from "./tool";
 export { TokenStream } from "./streaming";
@@ -59,13 +77,16 @@ export {
   SamplerConfig,
   cosineSimilarity,
   getCachedModels,
-} from "./index";
+} from "../generated/ts/nobodywho";
 
 export type {
   Asset,
   ToolCall,
   CachedModel,
-} from "./index";
+  ChatStats,
+} from "../generated/ts/nobodywho";
+export type { TtsArchitecture, TtsDevice, TtsOptions } from "./tts";
+export type { SttOptions } from "./stt";
 
 // Ergonomic wrapper additions.
 export { SamplerPresets } from "./sampler_presets";