tryharness 0.7.83 → 0.7.86

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (215) hide show
  1. package/README.md +21 -19
  2. package/assets/dsh-plugin.mjs +75 -4
  3. package/assets/herdr-mod/harness/main.js +367 -0
  4. package/assets/herdr-mod/harness/mod.json +6 -0
  5. package/assets/opencode-plugin.js +87 -22
  6. package/assets/pi-extension.ts +81 -2
  7. package/cli.mjs +544 -208
  8. package/lib/adapters.mjs +62 -27
  9. package/lib/agentBinaries.mjs +6 -8
  10. package/lib/agentClaims.mjs +168 -0
  11. package/lib/agentReadTools.json +415 -0
  12. package/lib/agentSignIn.mjs +17 -18
  13. package/lib/agentWiring.mjs +114 -0
  14. package/lib/archetypes.js +10 -11
  15. package/lib/attention.mjs +100 -0
  16. package/lib/audit.mjs +1 -1
  17. package/lib/backupSetup.mjs +6 -28
  18. package/lib/bankrApprovals.mjs +86 -0
  19. package/lib/boardAccess.mjs +40 -66
  20. package/lib/boardBackup.mjs +86 -85
  21. package/lib/boardCommands.mjs +28 -0
  22. package/lib/boardFileCommands.mjs +29 -36
  23. package/lib/boardFiles.mjs +83 -15
  24. package/lib/boardMirror.mjs +78 -11
  25. package/lib/boardProtocol.json +191 -123
  26. package/lib/boardThreads.mjs +19 -37
  27. package/lib/boardTools.mjs +22 -15
  28. package/lib/boardWatches.mjs +43 -15
  29. package/lib/bridge.mjs +7 -3
  30. package/lib/browserAgentTools.mjs +59 -0
  31. package/lib/browserWake.mjs +0 -6
  32. package/lib/cards.mjs +52 -0
  33. package/lib/circling.mjs +189 -0
  34. package/lib/cleanup.mjs +234 -0
  35. package/lib/cloudBoot.mjs +21 -4
  36. package/lib/cloudSave.mjs +46 -5
  37. package/lib/compaction.mjs +8 -1
  38. package/lib/completion.mjs +358 -0
  39. package/lib/computerDevices.mjs +853 -0
  40. package/lib/connectionStatus.mjs +34 -0
  41. package/lib/connectors.mjs +139 -1
  42. package/lib/coordinator.mjs +108 -14
  43. package/lib/coreModels.mjs +36 -12
  44. package/lib/daemon.mjs +2426 -338
  45. package/lib/daemonLock.mjs +103 -0
  46. package/lib/deviceDoor.mjs +308 -0
  47. package/lib/deviceIdentity.mjs +114 -0
  48. package/lib/deviceList.mjs +172 -0
  49. package/lib/deviceSeal.mjs +74 -0
  50. package/lib/dispatcher.mjs +497 -26
  51. package/lib/dispatches.mjs +3 -3
  52. package/lib/environment.mjs +2 -1
  53. package/lib/environments.mjs +67 -41
  54. package/lib/events.mjs +318 -0
  55. package/lib/facts.mjs +170 -0
  56. package/lib/fileSource.mjs +540 -0
  57. package/lib/files.mjs +47 -2
  58. package/lib/gitHost.mjs +39 -1
  59. package/lib/goalCommands.mjs +12 -80
  60. package/lib/handover.mjs +21 -16
  61. package/lib/harnessOffers.mjs +140 -15
  62. package/lib/herdr.mjs +100 -15
  63. package/lib/herdrAppCall.mjs +19 -0
  64. package/lib/herdrCore.mjs +59 -0
  65. package/lib/herdrRuns.mjs +1126 -0
  66. package/lib/hookDialects.mjs +235 -50
  67. package/lib/identityLink.mjs +81 -0
  68. package/lib/inference.mjs +31 -2
  69. package/lib/jamAccess.mjs +82 -0
  70. package/lib/{projectBoard.mjs → jamBoard.mjs} +5 -4
  71. package/lib/{projectGit.mjs → jamGit.mjs} +53 -15
  72. package/lib/jamKey.mjs +274 -0
  73. package/lib/{projectPieces.mjs → jamPieces.mjs} +35 -29
  74. package/lib/{projectSettings.mjs → jamSettings.mjs} +12 -22
  75. package/lib/jamStore.mjs +973 -0
  76. package/lib/jamUpgrade.mjs +112 -0
  77. package/lib/jams.mjs +3079 -0
  78. package/lib/kanban.mjs +13 -7
  79. package/lib/landed.mjs +5 -1
  80. package/lib/library.mjs +41 -22
  81. package/lib/limits.mjs +91 -0
  82. package/lib/lineDiff.mjs +97 -0
  83. package/lib/lineup.mjs +18 -13
  84. package/lib/link.mjs +584 -0
  85. package/lib/localStack.mjs +75 -16
  86. package/lib/localmodels.mjs +58 -78
  87. package/lib/loopBackstop.mjs +2 -1
  88. package/lib/machine.mjs +544 -0
  89. package/lib/machineLink.mjs +446 -0
  90. package/lib/machineRoutes.mjs +68 -19
  91. package/lib/machineRuns.mjs +85 -4
  92. package/lib/machineSeal.mjs +6 -31
  93. package/lib/manager.mjs +134 -12
  94. package/lib/mcp-stdio.mjs +492 -215
  95. package/lib/mcpRelay.mjs +34 -35
  96. package/lib/members.json +1 -1
  97. package/lib/modelRoute.mjs +162 -0
  98. package/lib/nativeHost.mjs +81 -90
  99. package/lib/nativeSenses.mjs +258 -0
  100. package/lib/nerves.mjs +523 -0
  101. package/lib/network.mjs +117 -278
  102. package/lib/networkPlaces.mjs +578 -0
  103. package/lib/nextStep.mjs +95 -0
  104. package/lib/notify.mjs +2 -2
  105. package/lib/nudge.mjs +116 -10
  106. package/lib/originals.mjs +216 -0
  107. package/lib/outsideEvents.mjs +383 -0
  108. package/lib/outsideRuns.mjs +230 -0
  109. package/lib/ownMachines.mjs +674 -256
  110. package/lib/pain.mjs +215 -0
  111. package/lib/pairing.mjs +91 -11
  112. package/lib/paths.mjs +42 -3
  113. package/lib/peerProcess.mjs +21 -7
  114. package/lib/peopleLink.mjs +268 -0
  115. package/lib/personNotices.mjs +3 -2
  116. package/lib/phones.mjs +252 -542
  117. package/lib/placeCards.mjs +282 -0
  118. package/lib/placeChecks.mjs +95 -0
  119. package/lib/placeCommands.mjs +8 -17
  120. package/lib/placeRequests.mjs +152 -0
  121. package/lib/placeStore.mjs +295 -0
  122. package/lib/placeTool.d.ts +12 -0
  123. package/lib/placeTool.js +40 -0
  124. package/lib/places.mjs +63 -14
  125. package/lib/plans.mjs +53 -8
  126. package/lib/presence.mjs +270 -0
  127. package/lib/privateNetwork.d.ts +2 -0
  128. package/lib/privateNetwork.js +93 -0
  129. package/lib/proposals.mjs +300 -0
  130. package/lib/publicFetch.d.ts +32 -0
  131. package/lib/publicFetch.js +248 -0
  132. package/lib/reach.mjs +128 -0
  133. package/lib/rebuildable.mjs +72 -0
  134. package/lib/recall.mjs +216 -0
  135. package/lib/record.mjs +13 -0
  136. package/lib/reflexes.mjs +392 -0
  137. package/lib/relayLimits.mjs +31 -2
  138. package/lib/releaseNotice.mjs +198 -0
  139. package/lib/removeHarness.mjs +15 -4
  140. package/lib/repoLinks.mjs +204 -35
  141. package/lib/runAuthority.mjs +102 -39
  142. package/lib/runBrief.mjs +93 -40
  143. package/lib/runRequests.mjs +52 -14
  144. package/lib/runUsage.mjs +31 -0
  145. package/lib/runWorker.mjs +74 -25
  146. package/lib/runner.mjs +289 -45
  147. package/lib/runtime-install.mjs +37 -8
  148. package/lib/schedules.mjs +171 -47
  149. package/lib/screenWatches.mjs +221 -0
  150. package/lib/sealed.mjs +12 -30
  151. package/lib/sealedThreads.mjs +198 -392
  152. package/lib/secretStore.mjs +178 -0
  153. package/lib/secrets.mjs +11 -0
  154. package/lib/seen.mjs +3 -9
  155. package/lib/selfUpdate.mjs +51 -15
  156. package/lib/senseCall.mjs +1514 -0
  157. package/lib/senseHook.mjs +114 -0
  158. package/lib/senseTools.mjs +1298 -0
  159. package/lib/sessionBriefs.mjs +87 -0
  160. package/lib/sessionControl.mjs +63 -13
  161. package/lib/sessionFiles.mjs +85 -0
  162. package/lib/setupChoices.mjs +12 -3
  163. package/lib/setupMode.mjs +70 -25
  164. package/lib/sharedKeys.mjs +166 -0
  165. package/lib/sleep.mjs +396 -0
  166. package/lib/sources.mjs +306 -0
  167. package/lib/speech.mjs +166 -411
  168. package/lib/spoken.mjs +210 -0
  169. package/lib/steerHook.mjs +3 -1
  170. package/lib/steerQueue.mjs +40 -11
  171. package/lib/store.mjs +323 -130
  172. package/lib/stuck.mjs +85 -7
  173. package/lib/swarm.mjs +157 -49
  174. package/lib/swarmDevices.mjs +25 -85
  175. package/lib/swarmMemory.mjs +93 -9
  176. package/lib/t3.mjs +73 -3
  177. package/lib/taskCommands.mjs +51 -60
  178. package/lib/taskRooms.mjs +30 -4
  179. package/lib/taskRules.json +29 -0
  180. package/lib/taskRules.mjs +21 -0
  181. package/lib/taskRuns.mjs +74 -11
  182. package/lib/team.mjs +2 -1
  183. package/lib/threadActions.mjs +37 -9
  184. package/lib/threadAuthority.mjs +4 -4
  185. package/lib/threadBriefs.mjs +173 -43
  186. package/lib/threadNotes.mjs +320 -0
  187. package/lib/threadRecord.mjs +491 -0
  188. package/lib/threadRuns.mjs +216 -108
  189. package/lib/threadView.mjs +27 -30
  190. package/lib/threadWatches.mjs +1962 -0
  191. package/lib/timeSense.mjs +839 -0
  192. package/lib/tripwires.mjs +454 -0
  193. package/lib/ui.mjs +1 -1
  194. package/lib/vault.mjs +560 -45
  195. package/lib/vaultCommands.mjs +5 -3
  196. package/lib/vaultPeers.mjs +165 -0
  197. package/lib/vaultTools.d.ts +4 -1
  198. package/lib/vaultTools.js +22 -1
  199. package/lib/verdicts.mjs +12 -26
  200. package/lib/versions.mjs +117 -0
  201. package/lib/waitTool.mjs +278 -0
  202. package/lib/wakePicture.mjs +268 -0
  203. package/lib/wakeQueue.mjs +108 -0
  204. package/lib/watchResponses.mjs +533 -0
  205. package/lib/watches.mjs +290 -145
  206. package/lib/windowAsks.mjs +80 -17
  207. package/lib/worktrees.mjs +159 -9
  208. package/npm-shrinkwrap.json +138 -631
  209. package/package.json +7 -6
  210. package/release.json +211 -129
  211. package/lib/callSessions.mjs +0 -51
  212. package/lib/directLink.mjs +0 -349
  213. package/lib/peopleBoard.mjs +0 -131
  214. package/lib/projects.mjs +0 -2917
  215. package/lib/roomChannels.mjs +0 -667
package/lib/speech.mjs CHANGED
@@ -2,32 +2,36 @@ import { spawn } from "node:child_process";
2
2
  import fs from "node:fs";
3
3
  import os from "node:os";
4
4
  import path from "node:path";
5
+ import { SPEAKING_JOB } from "./dispatcher.mjs";
5
6
  import {
6
7
  MODELS,
8
+ ensureLlama,
7
9
  ensureModels,
8
10
  modelById,
9
- modelDir,
10
11
  modelFile,
11
12
  modelsPresent,
12
13
  } from "./localmodels.mjs";
13
14
 
14
- // Talk in the buddy, heard on this computer only. Silero VAD finds where someone spoke, so a held key over a fan, a
15
- // keyboard or silence writes nothing. On a Mac with macOS 26, Apple's own transcriber writes the words (the buddy's
16
- // HarnessSpeech helper), steered toward the names the relay knows; elsewhere Parakeet does (its feature extractor,
17
- // encoder and token-and-duration decode in plain JS on ONNX Runtime), and the names the relay knows (its agents, the
18
- // person's folders and files) replace what it made of them when they sound alike. Moonshine stays as the baseline it
19
- // was measured against. The models are "extended": downloaded the first time the person holds Talk, never at setup.
20
- // Choices measured on synthetic requests and real people reading.
15
+ // What the person says while holding to talk, and the words in recordings for agents here, as text. On a Mac with macOS 26
16
+ // Apple's own transcriber writes the words (the buddy's HarnessSpeech helper), steered toward the names the relay knows; it
17
+ // needs nothing downloaded and was the most accurate measured (2.7% of words wrong over 30 minutes, eval long-listen,
18
+ // November 6). Elsewhere LFM3-Omni hears the audio itself, a piece at a time, asked first whether anyone is speaking: told
19
+ // to transcribe outright it writes words over noise. It writes nothing over noise this way, and about 10% wrong over 30
20
+ // minutes. The names the relay knows replace what was heard where they sound alike. Omni comes down the first time it's
21
+ // needed on a computer that doesn't have it, never in the way of the person.
21
22
 
22
23
  export const RATE = 16_000;
23
- export const SPEECH_IDS = ["vad", "speech"];
24
- const FRAME = 512;
25
- const CONTEXT = 64;
26
- // Moonshine was trained on clips up to about 30 s, and Parakeet is used as measured: speech is recognised in pieces of
27
- // at most 25 s, cut between stretches of speech where it can be.
28
- const MAX_PIECE_SECONDS = 25;
29
- // The most tokens a second of speech may produce: well above anyone's pace, so a decoder stuck repeating itself stops.
30
- const TOKENS_PER_SECOND = 8;
24
+ const OMNI = "dispatcher";
25
+ // Omni hears a recording in pieces of about this long, each cut at the quietest moment of its last few seconds.
26
+ const PIECE_SECONDS = 25;
27
+ const CUT_WINDOW_SECONDS = 5;
28
+ // Well above anyone's pace, so a model that loops stops soon.
29
+ const TOKENS_PER_SECOND = 12;
30
+ const WRITE_WORDS =
31
+ "Write out every word spoken in this clip, exactly as said, with nothing added.";
32
+ // Under this a stretch of 30 ms frames counts as quiet, about -50 dBFS.
33
+ const SOUND_LEVEL = 0.003;
34
+ const SOUND_FRAME = Math.round(RATE * 0.03);
31
35
  // The most of the relay's names (agents first, then folders and files) Apple's transcriber is steered toward.
32
36
  const APPLE_NAMES = 300;
33
37
  // The Mac's own word list: on the Apple path, a name of the person's never replaces a word in it.
@@ -169,102 +173,87 @@ export function resample(samples, from, to = RATE) {
169
173
  return out;
170
174
  }
171
175
 
172
- /**
173
- * Where speech is, from the VAD's per-frame probabilities: [start, end) in samples. A stretch starts at 0.5 and ends
174
- * after 500 ms under 0.35 (a pause between words keeps it going); one shorter than 250 ms is a click, not a word.
175
- */
176
- export function speechSpans(
177
- probs,
178
- {
179
- total = probs.length * FRAME,
180
- threshold = 0.5,
181
- minSpeechMs = 250,
182
- minSilenceMs = 500,
183
- padMs = 250,
184
- } = {}
176
+ /** Where there is any sound: [start, end) sample pairs, stretches under half a second apart joined. No model. */
177
+ export function soundSpans(
178
+ samples,
179
+ { minMs = 250, gapMs = 500, padMs = 200 } = {}
185
180
  ) {
186
- const frameMs = (FRAME / RATE) * 1000;
187
- const off = threshold - 0.15;
188
- const found = [];
189
- let start = -1,
190
- quiet = -1;
191
- const close = (end) => {
192
- if ((end - start) * frameMs >= minSpeechMs)
193
- found.push([start * FRAME, end * FRAME]);
194
- start = -1;
195
- quiet = -1;
196
- };
197
- for (let i = 0; i < probs.length; i += 1) {
198
- if (start < 0) {
199
- if (probs[i] >= threshold) start = i;
200
- continue;
201
- }
202
- if (probs[i] >= threshold) quiet = -1;
203
- else if (probs[i] < off) {
204
- if (quiet < 0) quiet = i;
205
- if ((i - quiet) * frameMs >= minSilenceMs) close(quiet);
206
- }
207
- }
208
- if (start >= 0) close(quiet >= 0 ? quiet : probs.length);
181
+ const gap = Math.round((gapMs / 1000) * RATE);
209
182
  const pad = Math.round((padMs / 1000) * RATE);
210
183
  const spans = [];
211
- for (const [s, e] of found) {
212
- const span = [Math.max(0, s - pad), Math.min(total, e + pad)];
213
- if (spans.length && span[0] <= spans.at(-1)[1]) spans.at(-1)[1] = span[1];
214
- else spans.push(span);
184
+ for (let at = 0; at < samples.length; at += SOUND_FRAME) {
185
+ let energy = 0;
186
+ const end = Math.min(samples.length, at + SOUND_FRAME);
187
+ for (let i = at; i < end; i += 1) energy += samples[i] ** 2;
188
+ if (Math.sqrt(energy / Math.max(1, end - at)) < SOUND_LEVEL) continue;
189
+ const last = spans.at(-1);
190
+ if (last && at - last[1] <= gap) last[1] = end;
191
+ else spans.push([at, end]);
215
192
  }
216
- return spans;
193
+ return spans
194
+ .filter(([start, end]) => ((end - start) / RATE) * 1000 >= minMs)
195
+ .map(([start, end]) => [
196
+ Math.max(0, start - pad),
197
+ Math.min(samples.length, end + pad),
198
+ ]);
217
199
  }
218
200
 
219
- /** A Llama-style SentencePiece vocabulary as a decoder: ids to text, special tokens skipped, byte fallback kept. */
220
- function tokenText(tokenizer) {
221
- const pieces = [];
222
- for (const [piece, id] of Object.entries(tokenizer.model.vocab))
223
- pieces[id] = piece;
224
- const special = new Set(
225
- tokenizer.added_tokens
226
- .filter((token) => token.special)
227
- .map((token) => token.id)
228
- );
229
- const decoder = new TextDecoder();
230
- const decode = (ids) => {
231
- const bytes = [];
232
- for (const id of ids) {
233
- if (special.has(id) || pieces[id] === undefined) continue;
234
- const byte = /^<0x([0-9A-F]{2})>$/.exec(pieces[id]);
235
- if (byte) bytes.push(parseInt(byte[1], 16));
236
- else bytes.push(...Buffer.from(pieces[id].replaceAll("▁", " "), "utf8"));
201
+ /** Sound cut into pieces Omni can hear: at most 25 s, cut at a quiet moment. */
202
+ export function quietPieces(samples, spans = soundSpans(samples)) {
203
+ const frame = Math.round(RATE / 10);
204
+ const out = [];
205
+ for (const [from, to] of spans) {
206
+ let start = from;
207
+ while (start < to) {
208
+ let end = Math.min(to, start + PIECE_SECONDS * RATE);
209
+ if (end < to) {
210
+ let quietest = Infinity;
211
+ let cut = end;
212
+ for (
213
+ let at = end - CUT_WINDOW_SECONDS * RATE;
214
+ at + frame < end;
215
+ at += frame
216
+ ) {
217
+ let energy = 0;
218
+ for (let i = at; i < at + frame; i += 4) energy += samples[i] ** 2;
219
+ if (energy < quietest) {
220
+ quietest = energy;
221
+ cut = at + Math.floor(frame / 2);
222
+ }
223
+ }
224
+ end = cut;
225
+ }
226
+ out.push(samples.subarray(start, end));
227
+ start = end;
237
228
  }
238
- return decoder.decode(Uint8Array.from(bytes)).replace(/^ /, "");
239
- };
240
- // Words the recogniser writes as one piece: the common words a name made of them must not be snapped from.
241
- const common = new Set(
242
- pieces
243
- .filter(
244
- (piece) => piece?.startsWith("▁") && /^[a-z]{2,}$/i.test(piece.slice(1))
245
- )
246
- .map((piece) => piece.slice(1).toLowerCase())
247
- );
248
- return { decode, common };
229
+ }
230
+ return out;
249
231
  }
250
232
 
251
- /** NeMo's vocabulary file ("piece id" a line) as a decoder, spaced as NeMo's own detokenizer does. */
252
- function vocabText(text) {
253
- const pieces = text
254
- .split("\n")
255
- .filter(Boolean)
256
- .map((line) => line.slice(0, line.lastIndexOf(" ")).replaceAll("▁", " "));
257
- const decode = (ids) =>
258
- ids
259
- .map((id) => pieces[id])
260
- .join("")
261
- .replace(/^\s|\s\B|(\s)\b/g, (_, space) => (space ? " " : ""));
262
- const common = new Set(
263
- pieces
264
- .filter((piece) => /^ [a-z]{2,}$/i.test(piece))
265
- .map((piece) => piece.slice(1).toLowerCase())
266
- );
267
- return { pieces, decode, common, blank: pieces.indexOf("<blk>") };
233
+ /**
234
+ * What a model wrote, as words someone said: a note about who is speaking comes off ("A man says:"), and a loop or a line
235
+ * of no letters counts as nothing heard (base Omni wrote "click click click" over keyboard noise).
236
+ */
237
+ export function spokenWords(text) {
238
+ const words = String(text)
239
+ .replace(/<think>[\s\S]*?<\/think>/g, "")
240
+ .trim()
241
+ .replace(
242
+ /^(?:a|an|the)\s+(?:male|female|man|woman|person|speaker|voice|narrator)(?:\s+(?:voice|speaker|narrator))?\s+(?:says|said|speaks|reads)\s*(?:[,:]\s*|(?=["“']))/i,
243
+ ""
244
+ )
245
+ .trim();
246
+ const unquoted = /^["“'].*["”']$/s.test(words) ? words.slice(1, -1) : words;
247
+ if (!/\p{L}{2,}|^I$/u.test(unquoted) || /^(?:I|mhm)\.?$/i.test(unquoted))
248
+ return "";
249
+ const seen = new Map();
250
+ const list = unquoted.toLowerCase().split(/\s+/);
251
+ for (let at = 0; at + 3 <= list.length; at += 1) {
252
+ const three = list.slice(at, at + 3).join(" ");
253
+ seen.set(three, (seen.get(three) ?? 0) + 1);
254
+ if (seen.get(three) >= 6) return "";
255
+ }
256
+ return unquoted;
268
257
  }
269
258
 
270
259
  /**
@@ -344,167 +333,6 @@ export function createAppleSpeech({
344
333
  };
345
334
  }
346
335
 
347
- // The models' own work, run on the model thread (onnxWorker.mjs) beside the sessions; each returns what the daemon's
348
- // side turns into words. Samples are sent as a copy of just their piece: a piece of a longer recording sent as is
349
- // would copy the whole recording to the other thread.
350
- /** The VAD's speech probability for each 32 ms frame: a job on the model thread (onnxWorker.mjs). */
351
- export async function vadProbabilities({ ort, graph }, file, samples) {
352
- const vad = await graph(file);
353
- const rate = new ort.Tensor("int64", BigInt64Array.from([BigInt(RATE)]), []);
354
- let state = new ort.Tensor("float32", new Float32Array(2 * 128), [2, 1, 128]);
355
- const input = new Float32Array(CONTEXT + FRAME);
356
- const probs = new Float32Array(Math.floor(samples.length / FRAME));
357
- for (let i = 0; i < probs.length; i += 1) {
358
- // Each frame carries the last 64 samples of the one before it, as the model was trained.
359
- input.copyWithin(0, FRAME, FRAME + CONTEXT);
360
- input.set(samples.subarray(i * FRAME, (i + 1) * FRAME), CONTEXT);
361
- const out = await vad.run({
362
- input: new ort.Tensor("float32", input.slice(), [1, CONTEXT + FRAME]),
363
- state,
364
- sr: rate,
365
- });
366
- probs[i] = out.output.data[0];
367
- state = out.stateN;
368
- }
369
- return probs;
370
- }
371
-
372
- /** Moonshine's token ids for one piece of speech: a job on the model thread. */
373
- export async function moonshineIds({ ort, graph }, files, config, samples) {
374
- const [encoder, decoder] = await Promise.all([
375
- graph(files.encoder),
376
- graph(files.decoder),
377
- ]);
378
- const { last_hidden_state: states } = await encoder.run({
379
- input_values: new ort.Tensor("float32", samples, [1, samples.length]),
380
- });
381
- const layers = config.decoder_num_hidden_layers,
382
- heads = config.decoder_num_key_value_heads,
383
- width = config.hidden_size / config.decoder_num_attention_heads;
384
- const empty = new ort.Tensor("float32", new Float32Array(0), [
385
- 1,
386
- heads,
387
- 0,
388
- width,
389
- ]);
390
- const past = {};
391
- for (let i = 0; i < layers; i += 1)
392
- for (const part of ["decoder", "encoder"])
393
- for (const kv of ["key", "value"])
394
- past[`past_key_values.${i}.${part}.${kv}`] = empty;
395
- const limit = Math.ceil((samples.length / RATE) * TOKENS_PER_SECOND) + 4;
396
- const ids = [];
397
- let token = config.decoder_start_token_id;
398
- for (let step = 0; step < limit; step += 1) {
399
- const out = await decoder.run({
400
- input_ids: new ort.Tensor(
401
- "int64",
402
- BigInt64Array.from([BigInt(token)]),
403
- [1, 1]
404
- ),
405
- encoder_hidden_states: states,
406
- use_cache_branch: new ort.Tensor(
407
- "bool",
408
- Uint8Array.from([step > 0 ? 1 : 0]),
409
- [1]
410
- ),
411
- ...past,
412
- });
413
- const logits = out.logits.data;
414
- let best = 0;
415
- for (let v = 1; v < logits.length; v += 1)
416
- if (logits[v] > logits[best]) best = v;
417
- // The encoder's keys and values come from the first step and stay; the decoder's grow by one each step.
418
- for (let i = 0; i < layers; i += 1)
419
- for (const part of step === 0 ? ["decoder", "encoder"] : ["decoder"])
420
- for (const kv of ["key", "value"])
421
- past[`past_key_values.${i}.${part}.${kv}`] =
422
- out[`present.${i}.${part}.${kv}`];
423
- if (best === config.eos_token_id) break;
424
- ids.push(best);
425
- token = best;
426
- }
427
- return ids;
428
- }
429
-
430
- /**
431
- * Parakeet's token ids for one piece of speech, a job on the model thread: its feature extractor and encoder, then a
432
- * greedy token-and-duration decode over the encoder's frames (as NeMo and onnx-asr do): each step names a token (or
433
- * blank) and how many frames to move. `vocabulary` is how many tokens there are; `blank` the blank's id.
434
- */
435
- export async function parakeetIds(
436
- { ort, graph },
437
- files,
438
- { vocabulary, blank },
439
- samples
440
- ) {
441
- const [features, encoder, joint] = await Promise.all(
442
- [files.features, files.encoder, files.joint].map((file) => graph(file))
443
- );
444
- const count = new ort.Tensor(
445
- "int64",
446
- BigInt64Array.from([BigInt(samples.length)]),
447
- [1]
448
- );
449
- const heard = await features.run({
450
- waveforms: new ort.Tensor("float32", samples, [1, samples.length]),
451
- waveforms_lens: count,
452
- });
453
- const { outputs, encoded_lengths } = await encoder.run({
454
- audio_signal: heard.features,
455
- length: heard.features_lens,
456
- });
457
- const [, width, frames] = outputs.dims,
458
- length = Number(encoded_lengths.data[0]);
459
- const frame = (t) => {
460
- const v = new Float32Array(width);
461
- for (let d = 0; d < width; d += 1) v[d] = outputs.data[d * frames + t];
462
- return v;
463
- };
464
- let states = [1, 2].map(
465
- () => new ort.Tensor("float32", new Float32Array(2 * 640), [2, 1, 640])
466
- );
467
- const ids = [];
468
- for (let t = 0, emitted = 0; t < length;) {
469
- const out = await joint.run({
470
- encoder_outputs: new ort.Tensor("float32", frame(t), [1, width, 1]),
471
- targets: new ort.Tensor(
472
- "int32",
473
- Int32Array.from([ids.at(-1) ?? blank]),
474
- [1, 1]
475
- ),
476
- target_length: new ort.Tensor("int32", Int32Array.from([1]), [1]),
477
- input_states_1: states[0],
478
- input_states_2: states[1],
479
- });
480
- const scores = out.outputs.data;
481
- let token = 0,
482
- step = 0;
483
- for (let v = 1; v < vocabulary; v += 1)
484
- if (scores[v] > scores[token]) token = v;
485
- for (let k = 1; k < scores.length - vocabulary; k += 1)
486
- if (scores[vocabulary + k] > scores[vocabulary + step]) step = k;
487
- if (token !== blank) {
488
- states = [out.output_states_1, out.output_states_2];
489
- ids.push(token);
490
- emitted += 1;
491
- }
492
- // At most ten tokens on one frame, as the model's decoding allows.
493
- if (step > 0) {
494
- t += step;
495
- emitted = 0;
496
- } else if (token === blank || emitted === 10) {
497
- t += 1;
498
- emitted = 0;
499
- }
500
- }
501
- return ids;
502
- }
503
-
504
- const VAD_JOB = { module: import.meta.url, name: "vadProbabilities" };
505
- const MOONSHINE_JOB = { module: import.meta.url, name: "moonshineIds" };
506
- const PARAKEET_JOB = { module: import.meta.url, name: "parakeetIds" };
507
-
508
336
  export function createSpeech({
509
337
  inference,
510
338
  root,
@@ -512,45 +340,13 @@ export function createSpeech({
512
340
  vocabulary = () => [],
513
341
  apple = null,
514
342
  fetchImpl,
343
+ getLlama = ensureLlama,
515
344
  log = () => {},
516
345
  }) {
517
- const vadEntry = modelById("vad", list),
518
- speechEntry = modelById("speech", list);
519
- const fileOf = (entry, pattern) =>
520
- entry.files.find((file) => pattern.test(file.path))?.path;
521
- const engine = speechEntry?.engine ?? "moonshine";
522
- let tokens = null,
523
- config = null;
524
- const load = () => {
525
- if (tokens) return;
526
- const dir = modelDir(root, speechEntry);
527
- if (engine === "parakeet") {
528
- tokens = vocabText(
529
- fs.readFileSync(
530
- path.join(dir, fileOf(speechEntry, /(^|\/)vocab\.txt$/)),
531
- "utf8"
532
- )
533
- );
534
- return;
535
- }
536
- tokens = tokenText(
537
- JSON.parse(
538
- fs.readFileSync(
539
- path.join(dir, fileOf(speechEntry, /(^|\/)tokenizer\.json$/)),
540
- "utf8"
541
- )
542
- )
543
- );
544
- config = JSON.parse(
545
- fs.readFileSync(
546
- path.join(dir, fileOf(speechEntry, /(^|\/)config\.json$/)),
547
- "utf8"
548
- )
549
- );
550
- };
346
+ const omniEntry = modelById(OMNI, list);
551
347
 
552
348
  // ── Apple's transcriber, where this Mac has it ──────────────────────────────
553
- // Asked once what it can do; while its model comes down, each status asks again. Without it, Parakeet.
349
+ // Asked once what it can do; while its model comes down, each status asks again. Without it, Omni.
554
350
  let macOS = apple ? null : { available: false };
555
351
  const probe = () =>
556
352
  apple.ask({ op: "status" }).then((answer) => {
@@ -558,7 +354,7 @@ export function createSpeech({
558
354
  });
559
355
  if (apple) void probe();
560
356
  const onMac = () => Boolean(macOS?.available);
561
- const ids = () => (onMac() ? ["vad"] : SPEECH_IDS);
357
+ const ids = () => (onMac() ? [] : [OMNI]);
562
358
 
563
359
  // ── Downloading on first use ────────────────────────────────────────────────
564
360
  let downloading = null,
@@ -578,7 +374,7 @@ export function createSpeech({
578
374
  if (ready())
579
375
  return {
580
376
  state: "ready",
581
- model: onMac() ? "Apple's transcriber" : speechEntry.name,
377
+ model: onMac() ? "Apple's transcriber" : omniEntry.name,
582
378
  };
583
379
  if (onMac() && !macOS.installed) {
584
380
  void probe();
@@ -605,14 +401,14 @@ export function createSpeech({
605
401
  ? { state: "failed", error: failure, percent, bytes: total }
606
402
  : { state: "absent", percent, bytes: total };
607
403
  };
608
- /** Starts the download if the models are missing; answers where it stands. */
404
+ /** Starts the download if a model is missing; answers where it stands. */
609
405
  const ensure = () => {
610
406
  if (!ready() && !downloading && macOS !== null) {
611
407
  failure = null;
612
408
  log(
613
409
  onMac()
614
410
  ? "speech: getting Apple's transcriber ready"
615
- : "speech: downloading the speech models"
411
+ : "speech: downloading the model that hears"
616
412
  );
617
413
  const mac =
618
414
  onMac() && !macOS.installed
@@ -621,16 +417,19 @@ export function createSpeech({
621
417
  return probe();
622
418
  })
623
419
  : null;
624
- downloading = Promise.all([
625
- ensureModels({
626
- root,
627
- ids: ids(),
628
- list,
629
- ...(fetchImpl ? { fetchImpl } : {}),
630
- log,
631
- }),
632
- mac,
633
- ])
420
+ const omni = onMac()
421
+ ? null
422
+ : Promise.all([
423
+ getLlama({ root, log, ...(fetchImpl ? { fetchImpl } : {}) }),
424
+ ensureModels({
425
+ root,
426
+ ids: ids(),
427
+ list,
428
+ ...(fetchImpl ? { fetchImpl } : {}),
429
+ log,
430
+ }),
431
+ ]);
432
+ downloading = Promise.all([omni, mac])
634
433
  .then(
635
434
  () => log("speech: ready"),
636
435
  (error) => {
@@ -646,67 +445,46 @@ export function createSpeech({
646
445
  };
647
446
 
648
447
  // ── Hearing ─────────────────────────────────────────────────────────────────
649
- const vadFile = fileOf(vadEntry, /\.onnx$/);
650
- /** The VAD's speech probability for each 32 ms frame. */
651
- const probabilities = (samples, lane) =>
652
- inference.onnx("vad", VAD_JOB, [vadFile, samples.slice()], { lane });
653
-
654
- /** Where in the samples someone spoke: [start, end) pairs, none for noise or silence. */
655
- const voiced = async (samples, { lane = "interactive" } = {}) =>
656
- speechSpans(await probabilities(samples, lane), { total: samples.length });
657
-
658
- /** Moonshine's words for one piece of speech. */
659
- const moonshine = async (samples, { lane = "interactive" } = {}) => {
660
- load();
661
- const files = {
662
- encoder: fileOf(speechEntry, /encoder[^/]*\.onnx$/),
663
- decoder: fileOf(speechEntry, /decoder[^/]*\.onnx$/),
664
- };
665
- const ids = await inference.onnx(
666
- "speech",
667
- MOONSHINE_JOB,
668
- [files, config, samples.slice()],
669
- { lane }
670
- );
671
- return tokens.decode(ids).trim();
672
- };
448
+ /** Where there is sound. Omni and Apple's transcriber decide whether any of it is words. */
449
+ const voiced = async (samples) => soundSpans(samples);
673
450
 
674
- /** Parakeet's words for one piece of speech. */
675
- const parakeet = async (samples, { lane = "interactive" } = {}) => {
676
- load();
677
- const files = {
678
- features: fileOf(speechEntry, /nemo\d+\.onnx$/),
679
- encoder: fileOf(speechEntry, /encoder[^/]*\.onnx$/),
680
- joint: fileOf(speechEntry, /decoder_joint[^/]*\.onnx$/),
681
- };
682
- const ids = await inference.onnx(
683
- "speech",
684
- PARAKEET_JOB,
685
- [
686
- files,
687
- { vocabulary: tokens.pieces.length, blank: tokens.blank },
688
- samples.slice(),
689
- ],
451
+ const askOmni = async (wav, question, maxTokens, lane) => {
452
+ const reply = await inference.chat(
453
+ OMNI,
454
+ {
455
+ temperature: 0.1,
456
+ top_k: 50,
457
+ repeat_penalty: 1.1,
458
+ max_tokens: maxTokens,
459
+ messages: [
460
+ { role: "system", content: SPEAKING_JOB.system },
461
+ {
462
+ role: "user",
463
+ content: [
464
+ {
465
+ type: "input_audio",
466
+ input_audio: { data: wav, format: "wav" },
467
+ },
468
+ { type: "text", text: question },
469
+ ],
470
+ },
471
+ ],
472
+ },
690
473
  { lane }
691
474
  );
692
- return tokens.decode(ids).trim();
475
+ return String(reply?.choices?.[0]?.message?.content ?? "");
693
476
  };
694
- const recognise = engine === "parakeet" ? parakeet : moonshine;
695
477
 
696
- /** Speech stretches joined into pieces Moonshine can take, cut between stretches. */
697
- const pieces = (samples, spans) => {
698
- const out = [];
699
- for (const [start, end] of spans) {
700
- const last = out.at(-1);
701
- if (last && end - last[0] <= MAX_PIECE_SECONDS * RATE) last[1] = end;
702
- else
703
- for (let s = start; s < end; s += MAX_PIECE_SECONDS * RATE)
704
- out.push([s, Math.min(end, s + MAX_PIECE_SECONDS * RATE)]);
705
- }
706
- return out.map(([start, end]) => samples.subarray(start, end));
478
+ /** Omni's words for one piece, nothing when no one is speaking in it. */
479
+ const hearPiece = async (piece, lane) => {
480
+ const wav = Buffer.from(encodeWav(piece, RATE)).toString("base64");
481
+ const speaking = await askOmni(wav, SPEAKING_JOB.question, 4, lane);
482
+ if (!/\byes\b/i.test(speaking)) return "";
483
+ const tokens = Math.ceil((piece.length / RATE) * TOKENS_PER_SECOND) + 20;
484
+ return spokenWords(await askOmni(wav, WRITE_WORDS, tokens, lane));
707
485
  };
708
486
 
709
- /** Audio (WAV bytes, or { sampleRate, samples }) to text: { text, heard, ms, model, raw }; nothing heard is { text: "", heard: false }. */
487
+ /** Audio (WAV bytes, or { sampleRate, samples }) to text: { text, heard, ms, model, raw }; nothing heard is no text. */
710
488
  const transcribe = async (input, { lane = "interactive" } = {}) => {
711
489
  const started = performance.now();
712
490
  if (!ready()) throw new Error("the speech models are not downloaded yet");
@@ -718,19 +496,20 @@ export function createSpeech({
718
496
  audio.sampleRate === RATE
719
497
  ? Float32Array.from(audio.samples)
720
498
  : resample(audio.samples, audio.sampleRate);
721
- const spans = await voiced(samples, { lane });
499
+ const spans = soundSpans(samples);
500
+ const model = onMac() ? "Apple's transcriber" : omniEntry.name;
501
+ const result = (raw, text = raw) => ({
502
+ text,
503
+ heard: Boolean(text),
504
+ ms: Math.round(performance.now() - started),
505
+ model,
506
+ raw,
507
+ });
508
+ if (!spans.length) return result("");
722
509
  const entries = vocabularyEntries(await vocabulary());
723
510
  if (onMac()) {
724
- // Apple's model hears the whole recording, steered toward the names the relay knows; then names are put right as
725
- // for Parakeet, with the Mac's own dictionary as the words a name never replaces.
726
- if (!spans.length)
727
- return {
728
- text: "",
729
- heard: false,
730
- ms: Math.round(performance.now() - started),
731
- model: "Apple's transcriber",
732
- raw: "",
733
- };
511
+ // Apple's model hears the whole recording, steered toward the names the relay knows; then names are put right, with
512
+ // the Mac's own dictionary as the words a name never replaces.
734
513
  const file = path.join(
735
514
  fs.mkdtempSync(path.join(os.tmpdir(), "harness-talk-")),
736
515
  "talk.wav"
@@ -743,49 +522,25 @@ export function createSpeech({
743
522
  names: entries.map((entry) => entry.display).slice(0, APPLE_NAMES),
744
523
  });
745
524
  if (!answer.ok) throw new Error(answer.error);
746
- const text = answer.text
747
- ? snapNames(answer.text, entries, macWords()).text
748
- : "";
749
- return {
750
- text,
751
- heard: Boolean(text),
752
- ms: Math.round(performance.now() - started),
753
- model: "Apple's transcriber",
754
- raw: answer.text,
755
- };
525
+ const raw = spokenWords(answer.text);
526
+ return result(
527
+ answer.text,
528
+ raw ? snapNames(raw, entries, macWords()).text : ""
529
+ );
756
530
  } finally {
757
531
  fs.rmSync(path.dirname(file), { recursive: true, force: true });
758
532
  }
759
533
  }
760
534
  const said = [];
761
- for (const piece of pieces(samples, spans)) {
762
- const words = await recognise(piece, { lane });
535
+ for (const piece of quietPieces(samples, spans)) {
536
+ const words = await hearPiece(piece, lane);
763
537
  if (words) said.push(words);
764
538
  }
765
539
  const raw = said.join(" ").trim();
766
- load();
767
- const text = raw ? snapNames(raw, entries, tokens.common).text : "";
768
- return {
769
- text,
770
- heard: Boolean(text),
771
- ms: Math.round(performance.now() - started),
772
- model: speechEntry.name,
773
- raw,
774
- };
540
+ return result(raw, raw ? snapNames(raw, entries, macWords()).text : "");
775
541
  };
776
542
 
777
- return {
778
- ready,
779
- status,
780
- ensure,
781
- transcribe,
782
- voiced,
783
- recognise,
784
- common: () => {
785
- load();
786
- return tokens.common;
787
- },
788
- };
543
+ return { ready, status, ensure, transcribe, voiced };
789
544
  }
790
545
 
791
546
  // ── Names ─────────────────────────────────────────────────────────────────────