thursday-agent 0.1.2 → 0.3.0

package/hooks/use-wake-word.ts

@@ -0,0 +1,262 @@
+"use client";
+
+import { useEffect, useRef } from "react";
+
+/**
+ * Wake-word listener built on the browser's Web Speech API (no realtime socket
+ * while idle). It is a transcriber, not a KWS engine, so matching is fuzzy.
+ * Chrome streams idle audio to Google; Firefox has no recognizer (onError only).
+ */
+
+/** lib.dom types the result objects but not the recognizer itself. */
+type SpeechRecognitionLike = {
+  lang: string;
+  continuous: boolean;
+  interimResults: boolean;
+  maxAlternatives: number;
+  start(): void;
+  stop(): void;
+  abort(): void;
+  onresult:
+    | ((event: {
+        resultIndex: number;
+        results: SpeechRecognitionResultList;
+      }) => void)
+    | null;
+  onerror: ((event: { error: string }) => void) | null;
+  onend: (() => void) | null;
+};
+
+type SpeechRecognitionCtor = new () => SpeechRecognitionLike;
+
+/**
+ * Default recognizer language. English keeps one spelling to compare against
+ * regardless of the speaker's accent; callers with a non-English name pass
+ * `lang` and `phrases` together.
+ */
+export const WAKE_LANG = "en-US";
+
+/**
+ * Allowed edit distance per word as a fraction of its length ("hey" = 1 edit,
+ * "thursday" = 3). Lower misses accents; higher lets ordinary speech wake.
+ */
+const WAKE_TOLERANCE = 0.34;
+
+/** Alternatives per result; the second-ranked guess is often the right one. */
+const ALTERNATIVES = 3;
+
+/** Delay before restarting after `onend`, so error loops do not spin. */
+const RESTART_MS = 300;
+/** Backoff cap while the recognition service is unreachable. */
+const RESTART_MAX_MS = 10_000;
+
+/** Errors a restart cannot fix. */
+const FATAL: Record<string, string> = {
+  "not-allowed": "The microphone is not allowed",
+  "service-not-allowed": "The microphone is not allowed",
+  "audio-capture": "There is no microphone",
+  "language-not-supported": "This browser cannot listen in that language",
+};
+
+/** Case and punctuation vary per recognizer; word boundaries do not. */
+const normalize = (text: string) =>
+  text
+    .toLowerCase()
+    .replace(/[^a-z0-9 ]/g, "")
+    .split(/\s+/)
+    .filter(Boolean);
+
+/** Plain Levenshtein distance between two words. */
+function distance(heard: string, want: string): number {
+  let previous = Array.from({ length: heard.length + 1 }, (_, index) => index);
+  let current = new Array<number>(heard.length + 1).fill(0);
+
+  for (let row = 1; row <= want.length; row++) {
+    current[0] = row;
+    for (let column = 1; column <= heard.length; column++) {
+      const substitute =
+        previous[column - 1] + (want[row - 1] === heard[column - 1] ? 0 : 1);
+      current[column] = Math.min(
+        substitute,
+        previous[column] + 1,
+        current[column - 1] + 1,
+      );
+    }
+    [previous, current] = [current, previous];
+  }
+
+  return previous[heard.length];
+}
+
+/** Within `budget` edits; the length check short-circuits most words. */
+const near = (heard: string, want: string, budget: number) =>
+  Math.abs(heard.length - want.length) <= budget &&
+  distance(heard, want) <= budget;
+
+/**
+ * Whether `want[from..]` is spoken contiguously starting at `heard[at]`.
+ * One wanted word may span two heard words ("there's day" for "thursday").
+ */
+function saidAt(
+  heard: readonly string[],
+  at: number,
+  want: readonly string[],
+  from: number,
+  tolerance: number,
+): boolean {
+  if (from === want.length) return true;
+  if (at >= heard.length) return false;
+
+  const word = want[from];
+  // Never 0, or "hay" for "hey" fails.
+  const budget = Math.max(1, Math.round(word.length * tolerance));
+
+  if (
+    near(heard[at], word, budget) &&
+    saidAt(heard, at + 1, want, from + 1, tolerance)
+  ) {
+    return true;
+  }
+  return (
+    at + 1 < heard.length &&
+    near(heard[at] + heard[at + 1], word, budget) &&
+    saidAt(heard, at + 2, want, from + 1, tolerance)
+  );
+}
+
+/**
+ * Whether `heard` contains `want` as one contiguous phrase. Scoring is per
+ * word, not over the joined string, so every word must be present and adjacent.
+ */
+function contains(
+  heard: readonly string[],
+  want: readonly string[],
+  tolerance: number,
+): boolean {
+  for (let at = 0; at < heard.length; at++) {
+    if (saidAt(heard, at, want, 0, tolerance)) return true;
+  }
+  return false;
+}
+
+type WakeOptions = {
+  /** Releases the microphone when false; must be false while a call is open. */
+  enabled: boolean;
+  /** Receives the raw transcript that matched. */
+  onWake: (heard: string) => void;
+  /** Phrases to listen for, matched fuzzily. */
+  phrases: readonly string[];
+  lang?: string;
+  /** See WAKE_TOLERANCE. */
+  tolerance?: number;
+  /** Unrecoverable failures only (denied or missing microphone). */
+  onError?: (reason: string) => void;
+};
+
+export function useWakeWord({
+  enabled,
+  onWake,
+  phrases,
+  lang = WAKE_LANG,
+  tolerance = WAKE_TOLERANCE,
+  onError,
+}: WakeOptions) {
+  // Read at result time so re-renders and phrase changes do not restart the
+  // recognizer; only `lang` must be set before start.
+  const latest = useRef({ onWake, onError, phrases, tolerance });
+  latest.current = { onWake, onError, phrases, tolerance };
+
+  useEffect(() => {
+    if (!enabled) return;
+
+    const global = window as unknown as Record<string, unknown>;
+    const Ctor = (global.SpeechRecognition ?? global.webkitSpeechRecognition) as
+      | SpeechRecognitionCtor
+      | undefined;
+
+    if (!Ctor) {
+      latest.current.onError?.("This browser cannot listen for a wake word");
+      return;
+    }
+
+    const recognition = new Ctor();
+    // Latched off on cleanup so late events cannot revive the recognizer.
+    let alive = true;
+    // Off from a match until the restart that clears the transcript.
+    let armed = true;
+    let restart: ReturnType<typeof setTimeout> | null = null;
+    let delay = RESTART_MS;
+
+    recognition.lang = lang;
+    // Interim results are enough; waiting for isFinal loses a beat.
+    recognition.continuous = true;
+    recognition.interimResults = true;
+    recognition.maxAlternatives = ALTERNATIVES;
+
+    const listen = () => {
+      if (!alive) return;
+      armed = true;
+      try {
+        recognition.start();
+      } catch {
+        // Already running (InvalidStateError).
+      }
+    };
+
+    recognition.onresult = ({ resultIndex, results }) => {
+      if (!alive || !armed) return;
+      delay = RESTART_MS;
+
+      const { phrases, tolerance, onWake } = latest.current;
+      const targets = phrases.map(normalize).filter((words) => words.length);
+
+      // Results before resultIndex are final and already seen.
+      for (let index = resultIndex; index < results.length; index++) {
+        const result = results[index];
+        for (let rank = 0; rank < result.length; rank++) {
+          const heard = result[rank].transcript;
+          const words = normalize(heard);
+          if (!targets.some((target) => contains(words, target, tolerance))) {
+            continue;
+          }
+
+          // Abort before notifying: the transcript keeps growing and would
+          // match again on the next event. `onend` restarts unless the caller
+          // has dropped `enabled` by then.
+          armed = false;
+          recognition.abort();
+          onWake(heard.trim());
+          return;
+        }
+      }
+    };
+
+    recognition.onerror = ({ error }) => {
+      const reason = FATAL[error];
+      if (reason) {
+        alive = false;
+        latest.current.onError?.(reason);
+      } else if (error === "network") {
+        delay = Math.min(delay * 2, RESTART_MAX_MS);
+      }
+      // no-speech and aborted are normal for this API.
+    };
+
+    // Chrome ends a silent session after about a minute; keep restarting.
+    recognition.onend = () => {
+      if (!alive) return;
+      restart = setTimeout(listen, delay);
+    };
+
+    listen();
+
+    return () => {
+      alive = false;
+      if (restart) clearTimeout(restart);
+      recognition.onresult = null;
+      recognition.onerror = null;
+      recognition.onend = null;
+      recognition.abort();
+    };
+  }, [enabled, lang]);
+}

package/lib/limits.ts

@@ -9,6 +9,6 @@ export const COMMON_VALIDATE = {
     max: 100,
   },
   prompt: {
-    max: 2000,
+    max: 4000,
   },
 };

package/lib/oauth-page.ts

@@ -0,0 +1,27 @@
+/**
+ * The page an OAuth redirect lands on: a detour tab that says how it went and closes itself
+ * when it went well. Every sign-in that comes back through a browser draws this one.
+ */
+export function oauthPage(
+  title: string,
+  detail: string,
+  { autoClose = false }: { autoClose?: boolean } = {},
+): string {
+  return `<!doctype html>
+<meta charset="utf-8">
+<title>${escapeHtml(title)}</title>
+<body style="font-family: ui-monospace, monospace; display: grid; place-items: center; min-height: 100dvh; margin: 0">
+  <div style="text-align: center; max-width: 28rem; padding: 1.5rem">
+    <p style="font-size: 1.125rem; margin: 0 0 .5rem">${escapeHtml(title)}</p>
+    <p style="color: #666; font-size: .875rem; margin: 0">${escapeHtml(detail)}</p>
+  </div>
+  ${autoClose ? "<script>setTimeout(() => window.close(), 900)</script>" : ""}
+</body>`;
+}
+
+function escapeHtml(text: string) {
+  return text
+    .replaceAll("&", "&amp;")
+    .replaceAll("<", "&lt;")
+    .replaceAll(">", "&gt;");
+}

package/lib/realtime/openai.ts

@@ -6,8 +6,8 @@ import { createWebRtcTransport } from "./realtime.transport";
 
 const CALLS_URL = "https://api.openai.com/v1/realtime/calls";
 
-/** Streams user transcripts as recognized; whisper-1 reports only after the turn ends. */
-const TRANSCRIPTION_MODEL = "gpt-4o-mini-transcribe";
+/** Streams user transcripts as recognized; the file models report only after the turn ends. */
+const TRANSCRIPTION_MODEL = "gpt-live-transcribe";
 
 export const createOpenAiSession = realtimeSession({
   transport: ({ credential, audio, on }) =>

package/lib/realtime/realtime.driver.ts

@@ -1,3 +1,4 @@
+import { logger } from "@/lib/logger";
 import { errorToString } from "@/lib/utils";
 import type {
   RealtimeFunctionTool,
@@ -85,6 +86,9 @@ export function realtimeSession(
     } | null = null;
     let ackTimer: ReturnType<typeof setTimeout> | null = null;
 
+    /** Event types this session has already noted as unhandled (see `default`). */
+    const seenUnhandled = new Set<string>();
+
     const turns: Turn[] = [];
     const slot = new Map<string, number>();
     const finalized = new Set<string>();
@@ -339,7 +343,25 @@ export function realtimeSession(
 
         case "conversation.item.input_audio_transcription.completed":
           write(event.item_id, "user", event.transcript ?? "", "replace");
-          finish(event.item_id);
+          // Not always the end of the turn. xAI sends this while the user is
+          // still speaking, each time with a longer transcript and `in_progress`
+          // on it; finishing on the first froze the turn at its opening words
+          // and every later transcript was written but never reported (`write`
+          // reports only what is not finalized). A provider that sends it once
+          // omits the field, so anything but `in_progress` ends the turn.
+          if (event.status !== "in_progress") finish(event.item_id);
+          break;
+
+        // A turn nobody could read. Without this the item simply never gets
+        // words, and an item with no words is not reported at all (`report`) —
+        // the turn disappears from the transcript, and every reader downstream
+        // sees a conversation the user did not have.
+        case "conversation.item.input_audio_transcription.failed":
+          on.warn(
+            event.error?.message
+              ? `That turn could not be transcribed: ${event.error.message}`
+              : "That turn could not be transcribed.",
+          );
           break;
 
         case "response.output_audio_transcript.delta":
@@ -388,6 +410,17 @@ export function realtimeSession(
           finish(event.item_id);
           onToolCall(event);
           break;
+
+        default:
+          // Providers speak this protocol with their own additions, and a case
+          // that is missing here is invisible: the event is dropped and the
+          // conversation quietly loses whatever it carried. Once per type per
+          // session, so a busy stream cannot drown the log.
+          if (!seenUnhandled.has(event.type)) {
+            seenUnhandled.add(event.type);
+            logger.debug(`realtime: no case for ${event.type}`);
+          }
+          break;
       }
       settle();
     };

package/lib/realtime/realtime.protocol.ts

@@ -46,7 +46,7 @@ export type OpenAiSessionConfig = {
   audio?: {
     input?: {
       format?: RealtimeAudioFormat;
-      /** Null turns transcription off. `whisper-1` reports only at turn end; `gpt-4o-*-transcribe` streams deltas. */
+      /** Null turns transcription off. A file model reports only at turn end; `gpt-live-transcribe` streams deltas. */
       transcription?: {
         model: string;
         language?: string;
@@ -308,6 +308,12 @@ export type RealtimeServerEvent =
       type: "conversation.item.input_audio_transcription.completed";
       item_id: string;
       transcript: string;
+      /**
+       * xAI sends this event repeatedly while the turn is still being
+       * transcribed, each time with more of it and `in_progress` here; only the
+       * last carries `completed`. OpenAI sends it once and omits the field.
+       */
+      status?: "in_progress" | "completed" | string;
     }
   /** OpenAI only. */
   | {

package/lib/realtime/realtime.tap.ts

@@ -131,3 +131,125 @@ export function createAudioTap(): RealtimeAudio & {
     },
   };
 }
+
+/** What a face moves with on one frame (createVoiceFollower). */
+export type VoiceFrame = {
+  /** How far into its own range the voice is now, 0..1; 0 in silence. */
+  level: number;
+  /** `level` over about half a second: the phrase rather than the syllable. */
+  phrase: number;
+  /** Strength of a syllable that started on this frame; 0 when none did. */
+  onset: number;
+  /** Each band inside its own range, low to high; one the voice never reaches stays 0. */
+  bands: number[];
+};
+
+/** Seconds, except where a line says otherwise. */
+const FOLLOW = {
+  /** A band's ceiling jumps to a new peak and sinks back over `ceilRelease`. */
+  ceilAttack: 0.03,
+  ceilRelease: 2.5,
+  /** Its floor drops to the quiet between syllables and creeps back up over `floorRise`. */
+  floorFall: 0.12,
+  floorRise: 3,
+  /** A band whose range is narrower than this (0..1) carries no shape. */
+  minSpan: 0.12,
+  /** Mean band level (0..1) across which silence turns into voice. */
+  quietBelow: 0.03,
+  voicedAbove: 0.12,
+  fast: 0.05,
+  mid: 0.22,
+  phrase: 0.45,
+  /** A syllable: the fast level rises this far over the mid one, past `onsetMin`, at most once per `onsetGap`. */
+  onsetRise: 0.1,
+  onsetMin: 0.3,
+  onsetGap: 0.12,
+};
+
+/**
+ * Reads the voice's bands the way a face needs them. A voice is loud and narrow:
+ * its bands sit near the top and lean the same way for a whole sentence, so a
+ * face driven by them directly stays swollen and still. Here each band is
+ * followed inside its own recent range — the quiet between syllables is its
+ * floor, the syllables its ceiling — so the face moves with the words whatever
+ * the voice or its volume. One per face; it keeps state between frames.
+ */
+export function createVoiceFollower() {
+  const ceil = new Array<number>(SPECTRUM_BANDS).fill(0);
+  const floor = new Array<number>(SPECTRUM_BANDS).fill(0);
+  const frame: VoiceFrame = {
+    level: 0,
+    phrase: 0,
+    onset: 0,
+    bands: new Array<number>(SPECTRUM_BANDS).fill(0),
+  };
+  let fast = 0;
+  let mid = 0;
+  let sinceOnset = Number.POSITIVE_INFINITY;
+
+  const toward = (from: number, to: number, seconds: number, dt: number) =>
+    from + (to - from) * Math.min(1, dt / seconds);
+
+  return {
+    /** Call once per frame with the seconds since the last call. The frame returned is reused. */
+    read(input: ArrayLike<number>, dt: number): VoiceFrame {
+      let total = 0;
+      let within = 0;
+      let ranged = 0;
+      for (let k = 0; k < SPECTRUM_BANDS; k++) {
+        const x = input[k] ?? 0;
+        total += x;
+        ceil[k] = toward(
+          ceil[k],
+          x,
+          x > ceil[k] ? FOLLOW.ceilAttack : FOLLOW.ceilRelease,
+          dt,
+        );
+        floor[k] = toward(
+          floor[k],
+          x,
+          x < floor[k] ? FOLLOW.floorFall : FOLLOW.floorRise,
+          dt,
+        );
+        const span = ceil[k] - floor[k];
+        const at =
+          span > FOLLOW.minSpan
+            ? Math.min(1, Math.max(0, (x - floor[k]) / span))
+            : 0;
+        frame.bands[k] = at;
+        if (span > FOLLOW.minSpan) {
+          within += at;
+          ranged++;
+        }
+      }
+
+      const mean = total / SPECTRUM_BANDS;
+      const edge = Math.min(
+        1,
+        Math.max(
+          0,
+          (mean - FOLLOW.quietBelow) / (FOLLOW.voicedAbove - FOLLOW.quietBelow),
+        ),
+      );
+      const voiced = edge * edge * (3 - 2 * edge);
+      for (let k = 0; k < SPECTRUM_BANDS; k++) frame.bands[k] *= voiced;
+
+      frame.level = (ranged ? within / ranged : 0) * voiced;
+      fast = toward(fast, frame.level, FOLLOW.fast, dt);
+      mid = toward(mid, frame.level, FOLLOW.mid, dt);
+      frame.phrase = toward(frame.phrase, frame.level, FOLLOW.phrase, dt);
+
+      sinceOnset += dt;
+      frame.onset = 0;
+      if (
+        fast > mid + FOLLOW.onsetRise &&
+        fast > FOLLOW.onsetMin &&
+        sinceOnset > FOLLOW.onsetGap
+      ) {
+        sinceOnset = 0;
+        frame.onset = Math.min(1, fast);
+      }
+      return frame;
+    },
+  };
+}

package/lib/sandbox.ts

@@ -1,6 +1,7 @@
 import { spawn } from "node:child_process";
 import { mkdir, readdir, readFile, writeFile } from "node:fs/promises";
 import { join, resolve as pathResolve, relative, sep } from "node:path";
+import { EXEC_KILL_GRACE_MS, EXEC_TIMEOUT_MS } from "@/config";
 
 export type ExecResult = { stdout: string; stderr: string; exitCode: number };
 
@@ -53,6 +54,25 @@ const IGNORE = new Set([
   "coverage",
 ]);
 
+/**
+ * Every file under `dir`, skipping dot entries and the folders a build or a
+ * package manager fills (IGNORE). A folder that cannot be read yields nothing.
+ */
+export async function* walkFiles(dir: string): AsyncGenerator<string> {
+  let entries;
+  try {
+    entries = await readdir(dir, { withFileTypes: true });
+  } catch {
+    return;
+  }
+  for (const e of entries) {
+    if (IGNORE.has(e.name) || e.name.startsWith(".")) continue;
+    const full = join(dir, e.name);
+    if (e.isDirectory()) yield* walkFiles(full);
+    else if (e.isFile()) yield full;
+  }
+}
+
 /**
  * Environment for every shell the agent runs: nobody watches it, so no pager
  * may stall it, and secrets do not travel to children. playwright-cli runs
@@ -100,21 +120,6 @@ export const createSandBox = ({
   const res = (p: string) =>
     p.startsWith("/") || /^[A-Za-z]:/.test(p) ? p : pathResolve(cwd, p);
 
-  async function* walk(dir: string): AsyncGenerator<string> {
-    let entries;
-    try {
-      entries = await readdir(dir, { withFileTypes: true });
-    } catch {
-      return;
-    }
-    for (const e of entries) {
-      if (IGNORE.has(e.name) || e.name.startsWith(".")) continue;
-      const full = join(dir, e.name);
-      if (e.isDirectory()) yield* walk(full);
-      else if (e.isFile()) yield full;
-    }
-  }
-
   // A very simple glob → regex. Only **/, * and ? are supported
   const globToRe = (g: string) =>
     new RegExp(
@@ -147,7 +152,7 @@ export const createSandBox = ({
       const root = res(path);
       const re = globToRe(pattern);
       const out: string[] = [];
-      for await (const f of walk(root)) {
+      for await (const f of walkFiles(root)) {
         const rel = relative(root, f).split(sep).join("/");
         if (re.test(rel) || re.test(f)) out.push(f);
         if (out.length >= limit) break;
@@ -155,7 +160,7 @@ export const createSandBox = ({
       return out;
     },
 
-    exec(command, { cwd: c, timeoutMs = 180_000, signal, env } = {}) {
+    exec(command, { cwd: c, timeoutMs = EXEC_TIMEOUT_MS, signal, env } = {}) {
       return new Promise((resolve) => {
         const base = shellEnv();
         if (extraPath) base.PATH = `${base.PATH ?? ""}:${extraPath}`;
@@ -172,28 +177,26 @@ export const createSandBox = ({
         child.stdout?.on("data", (d) => (stdout += d));
         child.stderr?.on("data", (d) => (stderr += d));
 
-        const stop = (why: string) => {
-          stderr += `\n[${why}]`;
+        const group = (name: NodeJS.Signals) => {
           try {
             // A negative pid means the group: the shell and everything under it
-            if (child.pid) process.kill(-child.pid, "SIGTERM");
+            if (child.pid) process.kill(-child.pid, name);
           } catch {
             // Already gone, or no process groups on this platform
-            child.kill("SIGTERM");
+            child.kill(name);
           }
         };
 
-        const timer = setTimeout(
-          () => stop(`Timed out after ${timeoutMs}ms`),
-          timeoutMs,
-        );
-
-        const onAbort = () => stop("Aborted");
-        if (signal?.aborted) onAbort();
-        else signal?.addEventListener("abort", onAbort, { once: true });
+        let settled = false;
+        let killing: ReturnType<typeof setTimeout> | undefined;
+        let letGo: ReturnType<typeof setTimeout> | undefined;
 
         const done = async (result: ExecResult) => {
+          if (settled) return;
+          settled = true;
           clearTimeout(timer);
+          clearTimeout(killing);
+          clearTimeout(letGo);
           signal?.removeEventListener("abort", onAbort);
           resolve({
             ...result,
@@ -202,6 +205,34 @@ export const createSandBox = ({
           });
         };
 
+        const stop = (why: string) => {
+          if (killing || settled) return;
+          stderr += `\n[${why}]`;
+          group("SIGTERM");
+          // A command that ignores SIGTERM would hold its step, and everything
+          // waiting on the job, for good (config EXEC_KILL_GRACE_MS)
+          killing = setTimeout(() => {
+            stderr += "\n[Killed: it did not exit on SIGTERM]";
+            group("SIGKILL");
+            // Output held open by a process that left the group never closes;
+            // what arrived by now is the result
+            letGo = setTimeout(() => {
+              child.stdout?.destroy();
+              child.stderr?.destroy();
+              void done({ stdout, stderr, exitCode: -1 });
+            }, 1_000);
+          }, EXEC_KILL_GRACE_MS);
+        };
+
+        const timer = setTimeout(
+          () => stop(`Timed out after ${timeoutMs}ms`),
+          timeoutMs,
+        );
+
+        const onAbort = () => stop("Aborted");
+        if (signal?.aborted) onAbort();
+        else signal?.addEventListener("abort", onAbort, { once: true });
+
         child.on("close", (code) =>
           done({ stdout, stderr, exitCode: code ?? -1 }),
         );

package/lib/tokens.ts

@@ -14,3 +14,22 @@ export function estimateTokens(text: string): number {
   }
   return Math.ceil(cjk / 1.3 + other / 4);
 }
+
+/**
+ * One assembled prompt measured by its own `##` headings, biggest first. Read
+ * off the finished text rather than off what built it: what is counted is then
+ * what is actually sent, and a chapter cannot be added without appearing here.
+ */
+export function sectionTokens(
+  text: string,
+): { name: string; tokens: number }[] {
+  return text
+    .split(/\n(?=## )/)
+    .map((block) => ({
+      name: block.startsWith("## ")
+        ? block.slice(3, block.indexOf("\n"))
+        : "identity",
+      tokens: estimateTokens(block),
+    }))
+    .sort((a, b) => b.tokens - a.tokens);
+}