ucode-agent 1.62.8 → 1.63.0

package/README.md

@@ -368,6 +368,7 @@ Everything after the frontmatter is the instruction.
 | `/undo` | put back every file the last turn changed |
 | `/look [url]` | open the running app and report what is on the page |
 | `/deploy [folder]` | put the app online and get its link |
+| `/mic` | say what you want instead of typing it — same as `ctrl+t` |
 | `/stats` | time, steps and tokens this session |
 | `/doctor` | check that everything ucode needs is working |
 | `/skills` | what it knows how to do, and what is loaded |
@@ -380,6 +381,18 @@ Everything after the frontmatter is the instruction.
 `↑ ↓` scroll the conversation, or walk history once you are typing ·
 `tab` completes a command
 
+### Speak instead of typing
+
+Press `ctrl+t` (or click `mic`, or type `/mic`) and say what you want —
+"make me a quiz app about planets". The chip turns red while it listens.
+`enter` stops and sends it; `ctrl+t` stops and puts the words in the box to
+check first; `esc` throws it away. It stops by itself after two minutes.
+
+Gemini writes down what you said, so nothing extra is needed beyond your key.
+Recording uses what the computer already has: Windows' built-in recorder, `sox`
+or `ffmpeg` on macOS (`brew install sox`), `arecord` or `sox` on Linux. If a
+quiet mic is taken for silence, set `UCODE_MIC_QUIET` lower than 800.
+
 ## Options
 
 ```

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ucode-agent",
-  "version": "1.62.8",
+  "version": "1.63.0",
   "description": "ucode - a terminal coding agent that reads, edits and runs your code, on Google Gemini models.",
   "type": "module",
   "main": "ucode.js",

package/src/core/loop.js

@@ -28,7 +28,7 @@ import { spawn } from 'node:child_process';
 
 import {
   ask, model, setModel, modelName, modelList, contextLimit, rateLimits,
-  estimateConversation, MODELS, DEFAULT_MODEL, PROVIDER, fallbackFor, backupFor,
+  estimateConversation, MODELS, DEFAULT_MODEL, PROVIDER, fallbackFor, backupFor, transcribe,
 } from './provider.js';
 import {
   tools, runTool, describe, setRoot, setConfirm, setRequest,
@@ -54,6 +54,7 @@ import { MAX_FILE_OUTPUT } from '../tools/shared.js';
 import { openInBrowser } from './opener.js';
 import { withScope, isNoise } from './scope.js';
 import { chooseFile, projectFiles, loadAttachments } from './attach.js';
+import { startRecording, isSilent, cleanTranscript, MAX_RECORD_MS } from './voice.js';
 import { runDoctor } from './doctor.js';
 import { JS_LOGIC } from './jslogic.js';
 import { deploy } from '../tools/deploy.js';
@@ -1028,6 +1029,7 @@ export class Agent {
       };
       this.ui.onModeChange = () => this.showHeader({ clear: false });
       this.ui.onAttach = () => { this.attachFile().catch(() => {}); };
+      this.ui.onMic = (action) => { this.mic(action).catch((err) => this.ui.error(err, { debug: this.debug })); };
     }
 
     for (const problem of this.skills.problems ?? []) {
@@ -1827,6 +1829,59 @@ export class Agent {
     }
   }
 
+  /**
+   * The mic button (ctrl+t, a click, or /mic). 'toggle' starts listening, or
+   * stops and puts the words in the box; 'send' stops and sends them; 'cancel'
+   * throws the recording away.
+   */
+  async mic(action = 'toggle') {
+    const state = this.ui.listening;
+    if (state === 'starting' || state === 'writing') return;
+
+    if (state !== 'listening') {
+      if (action !== 'toggle') return;
+      this.ui.setListening('starting');
+      try {
+        this.recording = await startRecording();
+      } catch (err) {
+        this.ui.setListening(null);
+        this.ui.note(`mic: ${err.message}`);
+        return;
+      }
+      this.ui.setListening('listening');
+      this.micTimer = setTimeout(() => this.ui.onMic?.('toggle'), MAX_RECORD_MS);
+      return;
+    }
+
+    clearTimeout(this.micTimer);
+    // Taken and the state moved on before any await, so a second press
+    // while this one finishes finds nothing to stop.
+    const recording = this.recording;
+    this.recording = null;
+    if (!recording) return;
+    if (action === 'cancel') {
+      this.ui.setListening(null);
+      this.ui.flash('stopped listening');
+      await recording.cancel();
+      return;
+    }
+
+    this.ui.setListening('writing');
+    let text = '';
+    try {
+      const wav = await recording.stop();
+      if (!wav || isSilent(wav)) {
+        this.ui.note('mic: heard nothing — check the microphone is on, then speak a little louder');
+        return;
+      }
+      text = cleanTranscript(await transcribe(wav));
+      if (!text) this.ui.note('mic: could not make out any words — try again, closer to the microphone');
+    } finally {
+      this.ui.setListening(null);
+    }
+    if (text) this.ui.insertText(text, { send: action === 'send' });
+  }
+
   /** Show a URL or a page in the project in the desktop browser (see opener.js). UCODE_OPEN=0 turns it off. */
   openInBrowser(target) {
     if (!this.full || process.env.UCODE_OPEN === '0') return false;
@@ -2961,6 +3016,9 @@ export class Agent {
       case '/doctor':   return this.cmdDoctor();
       case '/look':     return this.cmdLook(arg);
       case '/deploy':   return this.cmdDeploy(arg);
+      case '/mic':
+        if (this.ui.onMic) return this.mic('toggle');
+        return this.ui.note('the mic works in the full-screen view — start ucode in a terminal window');
       case '/exit':
       case '/quit':     return 'exit';
 
@@ -3031,6 +3089,7 @@ ${out.content}` });
       ['/doctor', 'check that everything ucode needs is working'],
       ['/look [url]', 'open the running app and report what is on the page'],
       ['/deploy [folder]', 'put the app online and get its link'],
+      ['/mic', 'say what you want instead of typing it (or ctrl+t)'],
       ['/model', 'show the models and switch between them'],
       ['/resume', 'pick up an earlier conversation'],
       ['/new', 'save this one and start fresh'],
@@ -3049,6 +3108,7 @@ ${out.content}` });
     this.ui.blank();
     this.ui.write(dim('  /models, /session and /sessions do the same as /model and /resume.'));
     this.ui.write(dim('  ctrl+b swaps plan and build · + file (or ctrl+o) adds a picture or file · esc stops a running turn · ctrl+d quits'));
+    this.ui.write(dim('  ctrl+t (or the mic button) listens: speak, then enter to send, ctrl+t to check it first, esc to cancel'));
     this.ui.blank();
   }
 

package/src/core/provider.js

@@ -276,6 +276,37 @@ export function resetConnection() {
   client = null;
 }
 
+const TRANSCRIBE_PROMPT =
+  'Transcribe the speech in this recording word for word, in the language spoken. ' +
+  'Reply with only the words spoken - no quotes, no notes. If nothing is said, reply with nothing.';
+
+/**
+ * Write down what was said in a WAV recording (the mic button). Uses the
+ * session's model, and the backup model when that one is overloaded.
+ */
+export async function transcribe(wav, { timeout = 45_000 } = {}) {
+  const request = (id) => connection().chat.completions.create({
+    model: id,
+    messages: [{
+      role: 'user',
+      content: [
+        { type: 'text', text: TRANSCRIBE_PROMPT },
+        { type: 'input_audio', input_audio: { data: wav.toString('base64'), format: 'wav' } },
+      ],
+    }],
+  }, { timeout });
+
+  let res;
+  try {
+    res = await request(current);
+  } catch (err) {
+    const backup = backupFor(current);
+    if (!backup) throw explain(err, current);
+    res = await request(backup).catch((again) => { throw explain(again, backup); });
+  }
+  return res.choices?.[0]?.message?.content ?? '';
+}
+
 // ---------------------------------------------------------------------------
 // Live quota, taken from whatever rate-limit headers come back
 // ---------------------------------------------------------------------------

package/src/core/voice.js

@@ -0,0 +1,172 @@
+/**
+ * voice.js — the mic button: say what you want instead of typing it.
+ *
+ * Recording uses what the computer already has, so there is nothing to
+ * install on Windows: its own sound recorder API (winmm, driven through
+ * PowerShell). macOS uses sox or ffmpeg, Linux arecord or sox, whichever is
+ * there. The recording is sent to the same Gemini model as everything else,
+ * which writes down what was said (see transcribe in provider.js).
+ */
+
+import { spawn } from 'node:child_process';
+import { promises as fs } from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+
+/** Longest a recording runs before it stops by itself. */
+export const MAX_RECORD_MS = 2 * 60_000;
+
+/**
+ * Loudest sample (of 32767) below which a recording counts as silence. A quiet
+ * room measured ~400 on a laptop mic, speech ~3,000 and up. UCODE_MIC_QUIET
+ * tunes it for a mic that reads differently.
+ */
+export const quietBelow = () => Number(process.env.UCODE_MIC_QUIET) || 800;
+
+// winmm's MCI strings refuse long paths (error 304), so the file sits directly
+// in the temp folder with a short name. The script says "ready" once it is
+// recording, and saves when a line arrives on stdin.
+const WINDOWS_SCRIPT = [
+  '$sig = \'[DllImport("winmm.dll", CharSet = CharSet.Unicode)] public static extern int mciSendString(string c, System.Text.StringBuilder r, int l, System.IntPtr h);\'',
+  '$w = Add-Type -MemberDefinition $sig -Name Mci -Namespace UcodeMic -PassThru',
+  'function mci($c) { $r = $w::mciSendString($c, $null, 0, [IntPtr]::Zero); if ($r -ne 0) { [Console]::Out.WriteLine("mci-error $r"); exit 1 } }',
+  'mci "open new type waveaudio alias ucodemic"',
+  'mci "set ucodemic bitspersample 16 channels 1 samplespersec 16000 bytespersec 32000 alignment 2"',
+  'mci "record ucodemic"',
+  '[Console]::Out.WriteLine("ready")',
+  '[void][Console]::In.ReadLine()',
+  'mci "stop ucodemic"',
+  'mci (\'save ucodemic "\' + $env:UCODE_MIC_FILE + \'"\')',
+  'mci "close ucodemic"',
+].join('\n');
+
+/**
+ * The recorders to try on this platform, in order. `stop` is how each is told
+ * to finish and write its file: a line on stdin, "q" on stdin, or SIGINT.
+ */
+export function recordersFor(platform, file) {
+  if (platform === 'win32') {
+    const encoded = Buffer.from(WINDOWS_SCRIPT, 'utf16le').toString('base64');
+    return [{
+      cmd: 'powershell.exe',
+      args: ['-NoProfile', '-NonInteractive', '-EncodedCommand', encoded],
+      stop: 'line',
+      saysReady: true,
+    }];
+  }
+  const sox = { cmd: 'rec', args: ['-q', '-c', '1', '-r', '16000', '-b', '16', file], stop: 'SIGINT' };
+  if (platform === 'darwin') {
+    return [sox, {
+      cmd: 'ffmpeg',
+      args: ['-loglevel', 'error', '-f', 'avfoundation', '-i', ':0', '-ac', '1', '-ar', '16000', '-y', file],
+      stop: 'q',
+    }];
+  }
+  return [{ cmd: 'arecord', args: ['-q', '-f', 'S16_LE', '-r', '16000', '-c', '1', file], stop: 'SIGINT' }, sox];
+}
+
+const NO_RECORDER = {
+  darwin: 'No recorder found. Install one with: brew install sox',
+  linux: 'No recorder found. Install one with: sudo apt install alsa-utils',
+};
+
+/** Start one recorder; resolves once it is listening, rejects with the reason it could not. */
+function launch(rec, file, run) {
+  return new Promise((resolve, reject) => {
+    const child = run(rec.cmd, rec.args, {
+      env: { ...process.env, UCODE_MIC_FILE: file },
+      windowsHide: true,
+      stdio: ['pipe', 'pipe', 'pipe'],
+    });
+    let out = '';
+    let settled = false;
+    const done = (fn, value) => { if (!settled) { settled = true; fn(value); } };
+
+    child.on('error', (err) => done(reject, err));
+    child.stdout?.on('data', (d) => {
+      out += d;
+      if (rec.saysReady && out.includes('ready')) done(resolve, child);
+      const code = /mci-error (\d+)/.exec(out)?.[1];
+      if (code) done(reject, new Error(`Windows could not use the microphone (error ${code}). Check one is plugged in and allowed in Settings › Privacy › Microphone.`));
+    });
+    child.on('exit', (code) => done(reject, new Error(`The recorder stopped before listening (exit ${code}). Check a microphone is connected.`)));
+    if (!rec.saysReady) child.on('spawn', () => done(resolve, child));
+  });
+}
+
+/**
+ * Start recording from the default microphone.
+ * Resolves to { stop() → Promise<Buffer|null>, cancel() → Promise<void> } once listening.
+ */
+export async function startRecording({ platform = process.platform, run = spawn, dir = os.tmpdir() } = {}) {
+  // A private folder of its own, so no other user can guess or pre-place the file.
+  const folder = await fs.mkdtemp(path.join(dir, 'ucm-'));
+  const file = path.join(folder, 'mic.wav');
+  const remove = () => fs.rm(folder, { recursive: true, force: true }).catch(() => {});
+
+  let child = null;
+  let rec = null;
+  try {
+    for (const candidate of recordersFor(platform, file)) {
+      try {
+        child = await launch(candidate, file, run);
+        rec = candidate;
+        break;
+      } catch (err) {
+        if (err.code !== 'ENOENT') throw err;
+      }
+    }
+  } catch (err) {
+    await remove();
+    throw err;
+  }
+  if (!child) {
+    await remove();
+    throw new Error(NO_RECORDER[platform] ?? 'No sound recorder found on this computer.');
+  }
+
+  const closed = new Promise((resolve) => child.on('close', resolve));
+  // A recorder that will not finish is killed rather than waited on forever.
+  const finished = async () => {
+    const timer = setTimeout(() => child.kill('SIGKILL'), 10_000);
+    await closed;
+    clearTimeout(timer);
+  };
+
+  return {
+    async stop() {
+      if (rec.stop === 'line') child.stdin.write('\n');
+      else if (rec.stop === 'q') child.stdin.write('q');
+      else child.kill('SIGINT');
+      await finished();
+      const wav = await fs.readFile(file).catch(() => null);
+      await remove();
+      return wav;
+    },
+    async cancel() {
+      child.kill();
+      await finished();
+      await remove();
+    },
+  };
+}
+
+/** True when a 16-bit PCM WAV holds nothing louder than room noise, or under a third of a second. */
+export function isSilent(wav, threshold = quietBelow()) {
+  const at = wav.indexOf('data');
+  const start = at >= 0 ? at + 8 : 44;
+  if (wav.length - start < 16000 * 2 / 3) return true;
+  for (let i = start; i + 1 < wav.length; i += 2) {
+    if (Math.abs(wav.readInt16LE(i)) >= threshold) return false;
+  }
+  return true;
+}
+
+/** What the model heard, as one line for the input box: no quotes, no line breaks. */
+export function cleanTranscript(text) {
+  return String(text ?? '')
+    .replace(/\s+/g, ' ')
+    .trim()
+    .replace(/^["'“”]+|["'“”]+$/g, '')
+    .trim();
+}

package/src/ui/screen.js

@@ -40,7 +40,7 @@ import {
   theme, blue, sky, deep, dim, edge, ADDED, REMOVED, BANNER, BANNER_WIDTH, SPINNER,
   boxTop, boxBottom, boxRow, visLen, padVis, clip, wrapAnsi,
   shortenPath, asLabel, ensureColour, planLine, bare, narration, narrationMark, groupKind, groupLabel, groupTarget, runLine, planRows, tidyReply, trimAnswer,
-  bannerPaint, RAIL, modeChip, ADD_CHIP, asNarrationLine } from './theme.js';
+  bannerPaint, RAIL, modeChip, ADD_CHIP, micChip, asNarrationLine } from './theme.js';
 import { FRAME_MS, spinnerGlyph, formatDuration, doneLine, workingLine, bannerSweep, SWEEP_MS } from './activity.js';
 import { gitBranch } from '../core/git.js';
 import { renderer, render } from './markdown.js';
@@ -60,7 +60,7 @@ export function isLabel(text) {
 export const COMMANDS = [
   '/help', '/model', '/models', '/session', '/sessions', '/resume',
   '/new', '/remember', '/skills', '/clear', '/search', '/copy', '/exit',
-  '/stats', '/doctor', '/deploy', '/look', '/undo',
+  '/stats', '/doctor', '/deploy', '/look', '/undo', '/mic',
 ];
 
 // ANSI ----------------------------------------------------------------------
@@ -215,6 +215,12 @@ export class Screen {
     this.plusTo = 0;
     this.attachments = [];
     this.onAttach = null;
+    // The mic button: null when idle, else 'starting' | 'listening' | 'writing'.
+    // Ctrl+T starts and stops it; while it listens, enter stops and sends.
+    this.micFrom = 0;
+    this.micTo = 0;
+    this.listening = null;
+    this.onMic = null;
     this.onInterrupt = null;
     this.onModeChange = null;
     this.spinTimer = null;
@@ -808,7 +814,7 @@ export class Screen {
       ['dir', value(shortenPath(this.facts.cwd ?? this.cwd, room - 10))],
       branch && ['branch', value(branch)],
       VERSION && ['version', value(this.facts.update ? `${VERSION} → ${this.facts.update} next start` : VERSION)],
-      ['keys', value('/help · esc interrupts · ctrl+b plan · ctrl+o file')],
+      ['keys', value('/help · ctrl+t mic · ctrl+o file · ctrl+b plan · esc stops')],
     ].filter(Boolean).slice(0, BANNER.length - 1);
 
     while (facts.length < BANNER.length - 1) facts.push(['', '']);
@@ -986,7 +992,8 @@ export class Screen {
   statusRow(width = this.width()) {
     const inner = width - 2;             // the space between the two borders
     const chip = this.modeChip();
-    const left = ` ${chip}  ${ADD_CHIP}  ${chalk.white(this.model || '—')}`;
+    const mic = micChip(this.listening);
+    const buttons = ` ${chip}  ${ADD_CHIP}  ${mic}`;
 
     // How long the turn has taken, back in the box beside the other two facts
     // about the session. It is not on the live line: that line says what is
@@ -996,16 +1003,24 @@ export class Screen {
     const since = this.activity?.start ?? this.status.since ?? 0;
     const running = this.busy() && since && now - since >= 1000;
     const right = `${running ? `${dim(formatDuration(now - since))}   ` : ''}${this.percentChip()} `;
+    // On a narrow terminal the model name goes before any button does.
+    const withModel = `${buttons}  ${chalk.white(this.model || '—')}`;
+    const left = visLen(withModel) + visLen(right) < inner ? withModel : buttons;
 
     // Where a click on the bottom row still counts as hitting the mode chip.
     this.chipTo = 2 + visLen(chip);
     this.plusFrom = this.chipTo + 3;                     // after the two spaces
     this.plusTo = this.plusFrom + visLen(ADD_CHIP) - 1;
+    this.micFrom = this.plusTo + 3;
+    this.micTo = this.micFrom + visLen(mic) - 1;
 
     const between = Math.max(1, inner - visLen(left) - visLen(right));
     const files = this.attachments.map((f) => path.basename(f)).join(', ');
-    const middle = this.flashText ? dim(clip(this.flashText, between - 2))
-      : files ? sky(clip(`+ ${files}`, between - 2)) : '';
+    // Room for the three spaces after it and at least one before.
+    const room = between - 4;
+    const middle = this.listening === 'listening' ? sky(clip('speak now · enter sends · ctrl+t stops · esc cancels', room))
+      : this.flashText ? dim(clip(this.flashText, room))
+        : files ? sky(clip(`+ ${files}`, room)) : '';
 
     const tail = middle ? `${middle}   ` : '';
     const pad = Math.max(1, inner - visLen(left) - visLen(tail) - visLen(right));
@@ -1430,12 +1445,30 @@ export class Screen {
       if (row !== statusRow) return;
       if (col > g.left + 1 && col <= g.left + this.chipTo) this.toggleMode();
       else if (col >= g.left + this.plusFrom && col <= g.left + this.plusTo) this.onAttach?.();
+      else if (col >= g.left + this.micFrom && col <= g.left + this.micTo) this.onMic?.('toggle');
       return;
     }
-    // The mode chip and the "+ file" button, at the left of the bottom row.
+    // The mode chip, "+ file" and mic buttons, at the left of the bottom row.
     if (row !== this.rows - 1) return;
     if (col >= 2 && col <= this.chipTo) this.toggleMode();
     else if (col >= this.plusFrom && col <= this.plusTo) this.onAttach?.();
+    else if (col >= this.micFrom && col <= this.micTo) this.onMic?.('toggle');
+  }
+
+  /** The mic's state, for the chip: null, 'starting', 'listening' or 'writing'. */
+  setListening(state) {
+    this.listening = state;
+    this.render();
+  }
+
+  /** Words from the mic, into the box at the cursor; `send` presses enter for them. */
+  insertText(text, { send = false } = {}) {
+    const before = this.buffer.slice(0, this.cursor);
+    const joined = before && !before.endsWith(' ') ? `${before} ${text}` : before + text;
+    this.buffer = joined + this.buffer.slice(this.cursor);
+    this.cursor = joined.length;
+    if (send) this.onKey('\r');
+    else this.render();
   }
 
   /** A file chosen with "+ file", to go with the next message. */
@@ -1470,6 +1503,12 @@ export class Screen {
       return;
     }
 
+    // While the mic listens, enter means "done, send it" and esc means "never mind".
+    if (this.listening === 'listening') {
+      if (key === '\r' || key === '\n') { this.onMic?.('send'); return; }
+      if (key === ESC || key === '\x03') { this.onMic?.('cancel'); return; }
+    }
+
     switch (key) {
       case '\r':
       case '\n': {
@@ -1516,6 +1555,10 @@ export class Screen {
         this.onAttach?.();
         return;
 
+      case '\x14':  // ctrl+t — the mic button: start listening, or stop and put the words in the box
+        this.onMic?.('toggle');
+        return;
+
       case '\x15':  // ctrl+u — clear the line; on an empty line, the attached files
         if (!this.buffer) this.attachments = [];
         this.buffer = this.buffer.slice(this.cursor);

package/src/ui/theme.js

@@ -130,6 +130,13 @@ export const modeChip = (mode) =>
 /** The "+ file" button beside the mode chip, in the quieter of the two chip styles. */
 export const ADD_CHIP = PLAN_CHIP(' + file ');
 
+/** The mic button beside it: quiet when idle, red while it is listening. */
+const LISTEN_CHIP = chalk.bgHex('#e5484d').hex('#ffffff').bold;
+export const micChip = (state) =>
+  state === 'listening' ? LISTEN_CHIP(' ● listening ')
+    : state ? PLAN_CHIP(` ${state === 'starting' ? 'starting' : 'writing'}… `)
+      : PLAN_CHIP(' mic ');
+
 /** The spinner. Braille dots, because they animate in place without jitter. */
 export const SPINNER = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏'];