pi-web-voice 0.1.0

package/lib/providers.cjs

@@ -0,0 +1,202 @@
+"use strict";
+
+/**
+ * Speech-to-text backends. Each one declares how many vocabulary entries it
+ * can usefully take, and receives the terms the caller mined for this request.
+ *
+ * The browser always uploads 16 kHz mono PCM WAV, which every backend accepts,
+ * so no server-side ffmpeg or format negotiation is needed.
+ */
+
+async function withTimeout(ms, run) {
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), ms);
+  try {
+    return await run(controller.signal);
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+async function readError(response) {
+  const body = await response.text().catch(() => "");
+  return new Error(`${response.status} ${response.statusText} ${body}`.trim());
+}
+
+function audioBlob(audio) {
+  return new Blob([audio], { type: "audio/wav" });
+}
+
+/**
+ * Azure AI Speech — fast transcription, including MAI-Transcribe-2.
+ * Phrase list gives real decode-time keyword biasing, and leaving `locales`
+ * unset keeps automatic language identification and code switching on.
+ */
+async function azureSpeech(audio, config, terms) {
+  const { endpoint, key, model, apiVersion, style } = config.azureSpeech;
+  if (!endpoint || !key) throw new Error("azure-speech needs AZURE_SPEECH_ENDPOINT and AZURE_SPEECH_KEY");
+
+  const url = `${endpoint}/speechtotext/transcriptions:transcribe?api-version=${encodeURIComponent(apiVersion)}`;
+
+  const send = async (phrases) => {
+    const definition = {
+      enhancedMode: { enabled: true, model, modelOptions: { transcribeStyle: style } },
+    };
+    if (phrases.length > 0) definition.phraseList = { phrases };
+    // `locales` is deliberately never set: leaving it off keeps automatic
+    // language identification and mid-sentence code switching enabled.
+
+    const form = new FormData();
+    form.append("audio", audioBlob(audio), "clip.wav");
+    form.append("definition", JSON.stringify(definition));
+
+    return withTimeout(config.limits.timeoutMs, (signal) =>
+      fetch(url, { method: "POST", headers: { "Ocp-Apim-Subscription-Key": key }, body: form, signal }),
+    );
+  };
+
+  let response = await send(terms);
+
+  // A rejected vocabulary must never cost someone their recording: drop the
+  // phrase list and transcribe anyway, slightly less accurately.
+  if (response.status === 400 && terms.length > 0) {
+    console.error("[pi-web-voice] phrase list rejected, retrying without it");
+    response = await send([]);
+  }
+
+  if (!response.ok) throw await readError(response);
+
+  const result = await response.json();
+  return (result.combinedPhrases ?? []).map((phrase) => phrase.text).join(" ").trim();
+}
+
+/**
+ * Whisper-style models read a free-form `prompt`. Whisper only keeps the final
+ * 224 tokens of it, so the list goes last and stays short.
+ */
+function vocabularyPrompt(terms) {
+  if (terms.length === 0) return "";
+  let prompt = "";
+  for (const term of terms) {
+    const next = prompt ? `${prompt}, ${term}` : term;
+    if (next.length > 700) break;
+    prompt = next;
+  }
+  return `Terms that may appear: ${prompt}.`;
+}
+
+/**
+ * Azure OpenAI.
+ *
+ * `AZURE_OPENAI_ENDPOINT` may be a resource name, a resource URL, or the whole
+ * transcriptions URL copied from the portal — Azure has shipped the classic
+ * `/openai/deployments/<name>/audio/transcriptions?api-version=...` path and
+ * the newer `/openai/v1/audio/transcriptions?api-version=preview` surface, and
+ * which one a resource serves is not worth guessing.
+ *
+ * `gpt-transcribe` accepts structured `keywords[]` and `languages[]`, which
+ * suits a mined vocabulary far better than stuffing it into a free-form
+ * prompt. Older models get the prompt instead. If the service rejects the
+ * structured fields, the request is retried with the prompt so a recording is
+ * never lost to a parameter mismatch.
+ */
+async function azureOpenAI(audio, config, terms, languages) {
+  const { endpoint, key, deployment, apiVersion } = config.azureOpenAI;
+  if (!endpoint || !key) throw new Error("azure-openai needs AZURE_OPENAI_ENDPOINT and AZURE_OPENAI_API_KEY");
+
+  const explicit = /\/audio\/transcriptions/.test(endpoint);
+  const url = explicit
+    ? endpoint
+    : `${endpoint}/openai/deployments/${encodeURIComponent(deployment)}/audio/transcriptions?api-version=${encodeURIComponent(apiVersion)}`;
+  const needsModel = /\/openai\/v1\//.test(url);
+  const structured = /gpt-transcribe/i.test(url) || /gpt-transcribe/i.test(deployment);
+
+  const send = async (withKeywords) => {
+    const form = new FormData();
+    form.append("file", audioBlob(audio), "clip.wav");
+    form.append("response_format", "json");
+    if (needsModel) form.append("model", deployment);
+
+    if (withKeywords) {
+      for (const term of terms) form.append("keywords[]", term);
+      for (const language of languages) form.append("languages[]", language);
+    } else {
+      const prompt = vocabularyPrompt(terms);
+      if (prompt) form.append("prompt", prompt);
+    }
+
+    return withTimeout(config.limits.timeoutMs, (signal) =>
+      fetch(url, { method: "POST", headers: { "api-key": key }, body: form, signal }),
+    );
+  };
+
+  let response = await send(structured);
+  if (!response.ok && response.status === 400 && structured) {
+    console.error("[pi-web-voice] keywords rejected, retrying with a prompt");
+    response = await send(false);
+  }
+  if (!response.ok) throw await readError(response);
+
+  return String((await response.json()).text ?? "").trim();
+}
+
+/** OpenAI, Groq, or any server exposing /audio/transcriptions. */
+async function openAICompatible(audio, config, terms) {
+  const { baseUrl, key, model } = config.openai;
+  if (!baseUrl) throw new Error("openai needs PI_VOICE_OPENAI_BASE_URL");
+
+  const form = new FormData();
+  form.append("file", audioBlob(audio), "clip.wav");
+  form.append("model", model);
+  form.append("response_format", "json");
+  const prompt = vocabularyPrompt(terms);
+  if (prompt) form.append("prompt", prompt);
+
+  const response = await withTimeout(config.limits.timeoutMs, (signal) =>
+    fetch(`${baseUrl.replace(/\/+$/, "")}/audio/transcriptions`, {
+      method: "POST",
+      headers: key ? { authorization: `Bearer ${key}` } : {},
+      body: form,
+      signal,
+    }),
+  );
+  if (!response.ok) throw await readError(response);
+
+  return String((await response.json()).text ?? "").trim();
+}
+
+/** No credentials required. Used to verify the plumbing end to end. */
+async function mock(audio, config, terms, languages) {
+  const seconds = Math.max(1, Math.round(audio.length / (16000 * 2)));
+  const vocabulary = terms.length > 0 ? ` · ${terms.length} terms: ${terms.slice(0, 8).join(", ")}` : "";
+  return `[pi-web-voice mock] received ${audio.length} bytes (~${seconds}s of audio) · ${languages.join("/")}${vocabulary}`;
+}
+
+// How many vocabulary entries each backend can usefully absorb.
+// MAI-Transcribe rejects a phrase list longer than 50 outright, and
+// whisper-style prompts are bounded by their token budget.
+const TERM_BUDGET = {
+  "azure-speech": 50,
+  "azure-openai": 60,
+  openai: 60,
+  mock: 50,
+};
+
+const PROVIDERS = {
+  "azure-speech": azureSpeech,
+  "azure-openai": azureOpenAI,
+  openai: openAICompatible,
+  mock,
+};
+
+function termBudget(provider) {
+  return TERM_BUDGET[provider] ?? 50;
+}
+
+async function transcribe(audio, config, terms = [], languages = ["en"]) {
+  const provider = PROVIDERS[config.provider];
+  if (!provider) throw new Error(`unknown provider: ${config.provider}`);
+  return provider(audio, config, terms, languages);
+}
+
+module.exports = { transcribe, termBudget, PROVIDERS };

package/lib/routes.cjs

@@ -0,0 +1,134 @@
+"use strict";
+
+const fs = require("node:fs/promises");
+const path = require("node:path");
+const { transcribe, termBudget } = require("./providers.cjs");
+const { collectTerms } = require("./context.cjs");
+
+const INJECT_FILE = path.join(__dirname, "..", "public", "inject.js");
+
+function json(res, status, body) {
+  const payload = JSON.stringify(body);
+  res.writeHead(status, {
+    "content-type": "application/json; charset=utf-8",
+    "content-length": Buffer.byteLength(payload),
+    "cache-control": "no-store",
+  });
+  res.end(payload);
+}
+
+function readBody(req, maxBytes) {
+  return new Promise((resolve, reject) => {
+    const chunks = [];
+    let size = 0;
+    req.on("data", (chunk) => {
+      size += chunk.length;
+      if (size > maxBytes) {
+        reject(new Error(`audio exceeds ${maxBytes} bytes`));
+        req.destroy();
+        return;
+      }
+      chunks.push(chunk);
+    });
+    req.on("end", () => resolve(Buffer.concat(chunks)));
+    req.on("error", reject);
+  });
+}
+
+/**
+ * Settings the browser needs. Credentials never leave the server: the page
+ * only learns which provider is configured, never its key.
+ */
+function browserConfig(config) {
+  return { prefix: config.prefix, provider: config.provider };
+}
+
+/**
+ * Language hints for the models that accept several. The browser already says
+ * what it speaks, so nothing needs configuring; English is always included
+ * because the terms themselves are English.
+ */
+function languageHints(req) {
+  const header = String(req.headers["accept-language"] ?? "");
+  const languages = header
+    .split(",")
+    .map((part) => part.split(";")[0].trim().split("-")[0].toLowerCase())
+    .filter((tag) => /^[a-z]{2,3}$/.test(tag));
+  return [...new Set([...languages, "en"])].slice(0, 3);
+}
+
+function createRouter(config) {
+  return async function handleRoute(req, res) {
+    const url = new URL(req.url, "http://localhost");
+    const route = url.pathname.slice(config.prefix.length) || "/";
+
+    if (route === "/inject.js") {
+      // Read on every request so editing inject.js takes effect on reload,
+      // with no restart of pi-web.
+      const source = await fs.readFile(INJECT_FILE, "utf8");
+      const preamble = `window.__PI_WEB_VOICE__=${JSON.stringify(browserConfig(config))};\n`;
+      const body = preamble + source;
+      res.writeHead(200, {
+        "content-type": "text/javascript; charset=utf-8",
+        "content-length": Buffer.byteLength(body),
+        "cache-control": "no-store",
+      });
+      res.end(body);
+      return;
+    }
+
+    if (route === "/health") {
+      json(res, 200, { ok: true, provider: config.provider });
+      return;
+    }
+
+    // Shows exactly which vocabulary a session would send. Useful for tuning,
+    // and for seeing what leaves the machine before it does.
+    if (route === "/terms") {
+      const sessionId = url.searchParams.get("session") ?? "";
+      const cwd = url.searchParams.get("cwd") ?? "";
+      const limit = Math.min(config.context.maxTerms, termBudget(config.provider));
+      const terms = collectTerms(sessionId, cwd, config, limit);
+      json(res, 200, { sessionId, cwd, count: terms.length, terms });
+      return;
+    }
+
+    if (route === "/transcribe") {
+      if (req.method !== "POST") return json(res, 405, { error: "POST only" });
+      const started = Date.now();
+      try {
+        const audio = await readBody(req, config.limits.maxBytes);
+        if (audio.length === 0) return json(res, 400, { error: "empty audio" });
+
+        const sessionId = url.searchParams.get("session") ?? "";
+        const cwd = url.searchParams.get("cwd") ?? "";
+        const limit = Math.min(config.context.maxTerms, termBudget(config.provider));
+        const terms = collectTerms(sessionId, cwd, config, limit);
+        const languages = languageHints(req);
+
+        const text = await transcribe(audio, config, terms, languages);
+        // Metadata only, never the transcript: enough to see how it is doing
+        // over a week without writing anything you said into a log file.
+        console.log(
+          `[pi-web-voice] ${((Date.now() - started) / 1000).toFixed(1)}s · ` +
+            `${(audio.length / 32000).toFixed(1)}s audio · ${terms.length} terms · ` +
+            `${text.length} chars · ${languages.join("/")}`,
+        );
+        json(res, 200, {
+          text,
+          ms: Date.now() - started,
+          provider: config.provider,
+          terms: terms.length,
+        });
+      } catch (error) {
+        console.error("[pi-web-voice] transcribe failed:", error.message);
+        json(res, 502, { error: error.message });
+      }
+      return;
+    }
+
+    json(res, 404, { error: "not found" });
+  };
+}
+
+module.exports = { createRouter };

package/package.json

@@ -0,0 +1,44 @@
+{
+  "name": "pi-web-voice",
+  "version": "0.1.0",
+  "description": "Voice input for pi-web. A NODE_OPTIONS hook that injects a microphone button into the chat composer and transcribes speech with Azure AI Speech, Azure OpenAI, or any OpenAI-compatible endpoint. No fork, no patching, no rebuild.",
+  "keywords": [
+    "pi",
+    "pi-web",
+    "pi-coding-agent",
+    "voice",
+    "dictation",
+    "speech-to-text",
+    "azure",
+    "whisper"
+  ],
+  "homepage": "https://github.com/lijunle/pi-web-voice#readme",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/lijunle/pi-web-voice.git"
+  },
+  "bugs": {
+    "url": "https://github.com/lijunle/pi-web-voice/issues"
+  },
+  "author": "Junle Li",
+  "license": "MIT",
+  "main": "hook.cjs",
+  "bin": {
+    "pi-web-voice": "bin/pi-web-voice.js"
+  },
+  "files": [
+    "hook.cjs",
+    "lib",
+    "bin",
+    "public",
+    "README.md",
+    "LICENSE"
+  ],
+  "engines": {
+    "node": ">=20.0.0"
+  },
+  "scripts": {
+    "test": "node test/run.mjs",
+    "test:e2e": "node test/e2e-edge.mjs"
+  }
+}