tryharness 0.7.83 → 0.7.86
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -19
- package/assets/dsh-plugin.mjs +75 -4
- package/assets/herdr-mod/harness/main.js +367 -0
- package/assets/herdr-mod/harness/mod.json +6 -0
- package/assets/opencode-plugin.js +87 -22
- package/assets/pi-extension.ts +81 -2
- package/cli.mjs +544 -208
- package/lib/adapters.mjs +62 -27
- package/lib/agentBinaries.mjs +6 -8
- package/lib/agentClaims.mjs +168 -0
- package/lib/agentReadTools.json +415 -0
- package/lib/agentSignIn.mjs +17 -18
- package/lib/agentWiring.mjs +114 -0
- package/lib/archetypes.js +10 -11
- package/lib/attention.mjs +100 -0
- package/lib/audit.mjs +1 -1
- package/lib/backupSetup.mjs +6 -28
- package/lib/bankrApprovals.mjs +86 -0
- package/lib/boardAccess.mjs +40 -66
- package/lib/boardBackup.mjs +86 -85
- package/lib/boardCommands.mjs +28 -0
- package/lib/boardFileCommands.mjs +29 -36
- package/lib/boardFiles.mjs +83 -15
- package/lib/boardMirror.mjs +78 -11
- package/lib/boardProtocol.json +191 -123
- package/lib/boardThreads.mjs +19 -37
- package/lib/boardTools.mjs +22 -15
- package/lib/boardWatches.mjs +43 -15
- package/lib/bridge.mjs +7 -3
- package/lib/browserAgentTools.mjs +59 -0
- package/lib/browserWake.mjs +0 -6
- package/lib/cards.mjs +52 -0
- package/lib/circling.mjs +189 -0
- package/lib/cleanup.mjs +234 -0
- package/lib/cloudBoot.mjs +21 -4
- package/lib/cloudSave.mjs +46 -5
- package/lib/compaction.mjs +8 -1
- package/lib/completion.mjs +358 -0
- package/lib/computerDevices.mjs +853 -0
- package/lib/connectionStatus.mjs +34 -0
- package/lib/connectors.mjs +139 -1
- package/lib/coordinator.mjs +108 -14
- package/lib/coreModels.mjs +36 -12
- package/lib/daemon.mjs +2426 -338
- package/lib/daemonLock.mjs +103 -0
- package/lib/deviceDoor.mjs +308 -0
- package/lib/deviceIdentity.mjs +114 -0
- package/lib/deviceList.mjs +172 -0
- package/lib/deviceSeal.mjs +74 -0
- package/lib/dispatcher.mjs +497 -26
- package/lib/dispatches.mjs +3 -3
- package/lib/environment.mjs +2 -1
- package/lib/environments.mjs +67 -41
- package/lib/events.mjs +318 -0
- package/lib/facts.mjs +170 -0
- package/lib/fileSource.mjs +540 -0
- package/lib/files.mjs +47 -2
- package/lib/gitHost.mjs +39 -1
- package/lib/goalCommands.mjs +12 -80
- package/lib/handover.mjs +21 -16
- package/lib/harnessOffers.mjs +140 -15
- package/lib/herdr.mjs +100 -15
- package/lib/herdrAppCall.mjs +19 -0
- package/lib/herdrCore.mjs +59 -0
- package/lib/herdrRuns.mjs +1126 -0
- package/lib/hookDialects.mjs +235 -50
- package/lib/identityLink.mjs +81 -0
- package/lib/inference.mjs +31 -2
- package/lib/jamAccess.mjs +82 -0
- package/lib/{projectBoard.mjs → jamBoard.mjs} +5 -4
- package/lib/{projectGit.mjs → jamGit.mjs} +53 -15
- package/lib/jamKey.mjs +274 -0
- package/lib/{projectPieces.mjs → jamPieces.mjs} +35 -29
- package/lib/{projectSettings.mjs → jamSettings.mjs} +12 -22
- package/lib/jamStore.mjs +973 -0
- package/lib/jamUpgrade.mjs +112 -0
- package/lib/jams.mjs +3079 -0
- package/lib/kanban.mjs +13 -7
- package/lib/landed.mjs +5 -1
- package/lib/library.mjs +41 -22
- package/lib/limits.mjs +91 -0
- package/lib/lineDiff.mjs +97 -0
- package/lib/lineup.mjs +18 -13
- package/lib/link.mjs +584 -0
- package/lib/localStack.mjs +75 -16
- package/lib/localmodels.mjs +58 -78
- package/lib/loopBackstop.mjs +2 -1
- package/lib/machine.mjs +544 -0
- package/lib/machineLink.mjs +446 -0
- package/lib/machineRoutes.mjs +68 -19
- package/lib/machineRuns.mjs +85 -4
- package/lib/machineSeal.mjs +6 -31
- package/lib/manager.mjs +134 -12
- package/lib/mcp-stdio.mjs +492 -215
- package/lib/mcpRelay.mjs +34 -35
- package/lib/members.json +1 -1
- package/lib/modelRoute.mjs +162 -0
- package/lib/nativeHost.mjs +81 -90
- package/lib/nativeSenses.mjs +258 -0
- package/lib/nerves.mjs +523 -0
- package/lib/network.mjs +117 -278
- package/lib/networkPlaces.mjs +578 -0
- package/lib/nextStep.mjs +95 -0
- package/lib/notify.mjs +2 -2
- package/lib/nudge.mjs +116 -10
- package/lib/originals.mjs +216 -0
- package/lib/outsideEvents.mjs +383 -0
- package/lib/outsideRuns.mjs +230 -0
- package/lib/ownMachines.mjs +674 -256
- package/lib/pain.mjs +215 -0
- package/lib/pairing.mjs +91 -11
- package/lib/paths.mjs +42 -3
- package/lib/peerProcess.mjs +21 -7
- package/lib/peopleLink.mjs +268 -0
- package/lib/personNotices.mjs +3 -2
- package/lib/phones.mjs +252 -542
- package/lib/placeCards.mjs +282 -0
- package/lib/placeChecks.mjs +95 -0
- package/lib/placeCommands.mjs +8 -17
- package/lib/placeRequests.mjs +152 -0
- package/lib/placeStore.mjs +295 -0
- package/lib/placeTool.d.ts +12 -0
- package/lib/placeTool.js +40 -0
- package/lib/places.mjs +63 -14
- package/lib/plans.mjs +53 -8
- package/lib/presence.mjs +270 -0
- package/lib/privateNetwork.d.ts +2 -0
- package/lib/privateNetwork.js +93 -0
- package/lib/proposals.mjs +300 -0
- package/lib/publicFetch.d.ts +32 -0
- package/lib/publicFetch.js +248 -0
- package/lib/reach.mjs +128 -0
- package/lib/rebuildable.mjs +72 -0
- package/lib/recall.mjs +216 -0
- package/lib/record.mjs +13 -0
- package/lib/reflexes.mjs +392 -0
- package/lib/relayLimits.mjs +31 -2
- package/lib/releaseNotice.mjs +198 -0
- package/lib/removeHarness.mjs +15 -4
- package/lib/repoLinks.mjs +204 -35
- package/lib/runAuthority.mjs +102 -39
- package/lib/runBrief.mjs +93 -40
- package/lib/runRequests.mjs +52 -14
- package/lib/runUsage.mjs +31 -0
- package/lib/runWorker.mjs +74 -25
- package/lib/runner.mjs +289 -45
- package/lib/runtime-install.mjs +37 -8
- package/lib/schedules.mjs +171 -47
- package/lib/screenWatches.mjs +221 -0
- package/lib/sealed.mjs +12 -30
- package/lib/sealedThreads.mjs +198 -392
- package/lib/secretStore.mjs +178 -0
- package/lib/secrets.mjs +11 -0
- package/lib/seen.mjs +3 -9
- package/lib/selfUpdate.mjs +51 -15
- package/lib/senseCall.mjs +1514 -0
- package/lib/senseHook.mjs +114 -0
- package/lib/senseTools.mjs +1298 -0
- package/lib/sessionBriefs.mjs +87 -0
- package/lib/sessionControl.mjs +63 -13
- package/lib/sessionFiles.mjs +85 -0
- package/lib/setupChoices.mjs +12 -3
- package/lib/setupMode.mjs +70 -25
- package/lib/sharedKeys.mjs +166 -0
- package/lib/sleep.mjs +396 -0
- package/lib/sources.mjs +306 -0
- package/lib/speech.mjs +166 -411
- package/lib/spoken.mjs +210 -0
- package/lib/steerHook.mjs +3 -1
- package/lib/steerQueue.mjs +40 -11
- package/lib/store.mjs +323 -130
- package/lib/stuck.mjs +85 -7
- package/lib/swarm.mjs +157 -49
- package/lib/swarmDevices.mjs +25 -85
- package/lib/swarmMemory.mjs +93 -9
- package/lib/t3.mjs +73 -3
- package/lib/taskCommands.mjs +51 -60
- package/lib/taskRooms.mjs +30 -4
- package/lib/taskRules.json +29 -0
- package/lib/taskRules.mjs +21 -0
- package/lib/taskRuns.mjs +74 -11
- package/lib/team.mjs +2 -1
- package/lib/threadActions.mjs +37 -9
- package/lib/threadAuthority.mjs +4 -4
- package/lib/threadBriefs.mjs +173 -43
- package/lib/threadNotes.mjs +320 -0
- package/lib/threadRecord.mjs +491 -0
- package/lib/threadRuns.mjs +216 -108
- package/lib/threadView.mjs +27 -30
- package/lib/threadWatches.mjs +1962 -0
- package/lib/timeSense.mjs +839 -0
- package/lib/tripwires.mjs +454 -0
- package/lib/ui.mjs +1 -1
- package/lib/vault.mjs +560 -45
- package/lib/vaultCommands.mjs +5 -3
- package/lib/vaultPeers.mjs +165 -0
- package/lib/vaultTools.d.ts +4 -1
- package/lib/vaultTools.js +22 -1
- package/lib/verdicts.mjs +12 -26
- package/lib/versions.mjs +117 -0
- package/lib/waitTool.mjs +278 -0
- package/lib/wakePicture.mjs +268 -0
- package/lib/wakeQueue.mjs +108 -0
- package/lib/watchResponses.mjs +533 -0
- package/lib/watches.mjs +290 -145
- package/lib/windowAsks.mjs +80 -17
- package/lib/worktrees.mjs +159 -9
- package/npm-shrinkwrap.json +138 -631
- package/package.json +7 -6
- package/release.json +211 -129
- package/lib/callSessions.mjs +0 -51
- package/lib/directLink.mjs +0 -349
- package/lib/peopleBoard.mjs +0 -131
- package/lib/projects.mjs +0 -2917
- package/lib/roomChannels.mjs +0 -667
package/lib/speech.mjs
CHANGED
|
@@ -2,32 +2,36 @@ import { spawn } from "node:child_process";
|
|
|
2
2
|
import fs from "node:fs";
|
|
3
3
|
import os from "node:os";
|
|
4
4
|
import path from "node:path";
|
|
5
|
+
import { SPEAKING_JOB } from "./dispatcher.mjs";
|
|
5
6
|
import {
|
|
6
7
|
MODELS,
|
|
8
|
+
ensureLlama,
|
|
7
9
|
ensureModels,
|
|
8
10
|
modelById,
|
|
9
|
-
modelDir,
|
|
10
11
|
modelFile,
|
|
11
12
|
modelsPresent,
|
|
12
13
|
} from "./localmodels.mjs";
|
|
13
14
|
|
|
14
|
-
//
|
|
15
|
-
//
|
|
16
|
-
//
|
|
17
|
-
//
|
|
18
|
-
//
|
|
19
|
-
//
|
|
20
|
-
//
|
|
15
|
+
// What the person says while holding to talk, and the words in recordings for agents here, as text. On a Mac with macOS 26
|
|
16
|
+
// Apple's own transcriber writes the words (the buddy's HarnessSpeech helper), steered toward the names the relay knows; it
|
|
17
|
+
// needs nothing downloaded and was the most accurate measured (2.7% of words wrong over 30 minutes, eval long-listen,
|
|
18
|
+
// November 6). Elsewhere LFM3-Omni hears the audio itself, a piece at a time, asked first whether anyone is speaking: told
|
|
19
|
+
// to transcribe outright it writes words over noise. It writes nothing over noise this way, and about 10% wrong over 30
|
|
20
|
+
// minutes. The names the relay knows replace what was heard where they sound alike. Omni comes down the first time it's
|
|
21
|
+
// needed on a computer that doesn't have it, never in the way of the person.
|
|
21
22
|
|
|
22
23
|
export const RATE = 16_000;
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
const
|
|
26
|
-
|
|
27
|
-
//
|
|
28
|
-
const
|
|
29
|
-
|
|
30
|
-
|
|
24
|
+
const OMNI = "dispatcher";
|
|
25
|
+
// Omni hears a recording in pieces of about this long, each cut at the quietest moment of its last few seconds.
|
|
26
|
+
const PIECE_SECONDS = 25;
|
|
27
|
+
const CUT_WINDOW_SECONDS = 5;
|
|
28
|
+
// Well above anyone's pace, so a model that loops stops soon.
|
|
29
|
+
const TOKENS_PER_SECOND = 12;
|
|
30
|
+
const WRITE_WORDS =
|
|
31
|
+
"Write out every word spoken in this clip, exactly as said, with nothing added.";
|
|
32
|
+
// Under this a stretch of 30 ms frames counts as quiet, about -50 dBFS.
|
|
33
|
+
const SOUND_LEVEL = 0.003;
|
|
34
|
+
const SOUND_FRAME = Math.round(RATE * 0.03);
|
|
31
35
|
// The most of the relay's names (agents first, then folders and files) Apple's transcriber is steered toward.
|
|
32
36
|
const APPLE_NAMES = 300;
|
|
33
37
|
// The Mac's own word list: on the Apple path, a name of the person's never replaces a word in it.
|
|
@@ -169,102 +173,87 @@ export function resample(samples, from, to = RATE) {
|
|
|
169
173
|
return out;
|
|
170
174
|
}
|
|
171
175
|
|
|
172
|
-
/**
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
export function speechSpans(
|
|
177
|
-
probs,
|
|
178
|
-
{
|
|
179
|
-
total = probs.length * FRAME,
|
|
180
|
-
threshold = 0.5,
|
|
181
|
-
minSpeechMs = 250,
|
|
182
|
-
minSilenceMs = 500,
|
|
183
|
-
padMs = 250,
|
|
184
|
-
} = {}
|
|
176
|
+
/** Where there is any sound: [start, end) sample pairs, stretches under half a second apart joined. No model. */
|
|
177
|
+
export function soundSpans(
|
|
178
|
+
samples,
|
|
179
|
+
{ minMs = 250, gapMs = 500, padMs = 200 } = {}
|
|
185
180
|
) {
|
|
186
|
-
const
|
|
187
|
-
const off = threshold - 0.15;
|
|
188
|
-
const found = [];
|
|
189
|
-
let start = -1,
|
|
190
|
-
quiet = -1;
|
|
191
|
-
const close = (end) => {
|
|
192
|
-
if ((end - start) * frameMs >= minSpeechMs)
|
|
193
|
-
found.push([start * FRAME, end * FRAME]);
|
|
194
|
-
start = -1;
|
|
195
|
-
quiet = -1;
|
|
196
|
-
};
|
|
197
|
-
for (let i = 0; i < probs.length; i += 1) {
|
|
198
|
-
if (start < 0) {
|
|
199
|
-
if (probs[i] >= threshold) start = i;
|
|
200
|
-
continue;
|
|
201
|
-
}
|
|
202
|
-
if (probs[i] >= threshold) quiet = -1;
|
|
203
|
-
else if (probs[i] < off) {
|
|
204
|
-
if (quiet < 0) quiet = i;
|
|
205
|
-
if ((i - quiet) * frameMs >= minSilenceMs) close(quiet);
|
|
206
|
-
}
|
|
207
|
-
}
|
|
208
|
-
if (start >= 0) close(quiet >= 0 ? quiet : probs.length);
|
|
181
|
+
const gap = Math.round((gapMs / 1000) * RATE);
|
|
209
182
|
const pad = Math.round((padMs / 1000) * RATE);
|
|
210
183
|
const spans = [];
|
|
211
|
-
for (
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
184
|
+
for (let at = 0; at < samples.length; at += SOUND_FRAME) {
|
|
185
|
+
let energy = 0;
|
|
186
|
+
const end = Math.min(samples.length, at + SOUND_FRAME);
|
|
187
|
+
for (let i = at; i < end; i += 1) energy += samples[i] ** 2;
|
|
188
|
+
if (Math.sqrt(energy / Math.max(1, end - at)) < SOUND_LEVEL) continue;
|
|
189
|
+
const last = spans.at(-1);
|
|
190
|
+
if (last && at - last[1] <= gap) last[1] = end;
|
|
191
|
+
else spans.push([at, end]);
|
|
215
192
|
}
|
|
216
|
-
return spans
|
|
193
|
+
return spans
|
|
194
|
+
.filter(([start, end]) => ((end - start) / RATE) * 1000 >= minMs)
|
|
195
|
+
.map(([start, end]) => [
|
|
196
|
+
Math.max(0, start - pad),
|
|
197
|
+
Math.min(samples.length, end + pad),
|
|
198
|
+
]);
|
|
217
199
|
}
|
|
218
200
|
|
|
219
|
-
/**
|
|
220
|
-
function
|
|
221
|
-
const
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
.
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
201
|
+
/** Sound cut into pieces Omni can hear: at most 25 s, cut at a quiet moment. */
|
|
202
|
+
export function quietPieces(samples, spans = soundSpans(samples)) {
|
|
203
|
+
const frame = Math.round(RATE / 10);
|
|
204
|
+
const out = [];
|
|
205
|
+
for (const [from, to] of spans) {
|
|
206
|
+
let start = from;
|
|
207
|
+
while (start < to) {
|
|
208
|
+
let end = Math.min(to, start + PIECE_SECONDS * RATE);
|
|
209
|
+
if (end < to) {
|
|
210
|
+
let quietest = Infinity;
|
|
211
|
+
let cut = end;
|
|
212
|
+
for (
|
|
213
|
+
let at = end - CUT_WINDOW_SECONDS * RATE;
|
|
214
|
+
at + frame < end;
|
|
215
|
+
at += frame
|
|
216
|
+
) {
|
|
217
|
+
let energy = 0;
|
|
218
|
+
for (let i = at; i < at + frame; i += 4) energy += samples[i] ** 2;
|
|
219
|
+
if (energy < quietest) {
|
|
220
|
+
quietest = energy;
|
|
221
|
+
cut = at + Math.floor(frame / 2);
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
end = cut;
|
|
225
|
+
}
|
|
226
|
+
out.push(samples.subarray(start, end));
|
|
227
|
+
start = end;
|
|
237
228
|
}
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
// Words the recogniser writes as one piece: the common words a name made of them must not be snapped from.
|
|
241
|
-
const common = new Set(
|
|
242
|
-
pieces
|
|
243
|
-
.filter(
|
|
244
|
-
(piece) => piece?.startsWith("▁") && /^[a-z]{2,}$/i.test(piece.slice(1))
|
|
245
|
-
)
|
|
246
|
-
.map((piece) => piece.slice(1).toLowerCase())
|
|
247
|
-
);
|
|
248
|
-
return { decode, common };
|
|
229
|
+
}
|
|
230
|
+
return out;
|
|
249
231
|
}
|
|
250
232
|
|
|
251
|
-
/**
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
233
|
+
/**
|
|
234
|
+
* What a model wrote, as words someone said: a note about who is speaking comes off ("A man says:"), and a loop or a line
|
|
235
|
+
* of no letters counts as nothing heard (base Omni wrote "click click click" over keyboard noise).
|
|
236
|
+
*/
|
|
237
|
+
export function spokenWords(text) {
|
|
238
|
+
const words = String(text)
|
|
239
|
+
.replace(/<think>[\s\S]*?<\/think>/g, "")
|
|
240
|
+
.trim()
|
|
241
|
+
.replace(
|
|
242
|
+
/^(?:a|an|the)\s+(?:male|female|man|woman|person|speaker|voice|narrator)(?:\s+(?:voice|speaker|narrator))?\s+(?:says|said|speaks|reads)\s*(?:[,:]\s*|(?=["“']))/i,
|
|
243
|
+
""
|
|
244
|
+
)
|
|
245
|
+
.trim();
|
|
246
|
+
const unquoted = /^["“'].*["”']$/s.test(words) ? words.slice(1, -1) : words;
|
|
247
|
+
if (!/\p{L}{2,}|^I$/u.test(unquoted) || /^(?:I|mhm)\.?$/i.test(unquoted))
|
|
248
|
+
return "";
|
|
249
|
+
const seen = new Map();
|
|
250
|
+
const list = unquoted.toLowerCase().split(/\s+/);
|
|
251
|
+
for (let at = 0; at + 3 <= list.length; at += 1) {
|
|
252
|
+
const three = list.slice(at, at + 3).join(" ");
|
|
253
|
+
seen.set(three, (seen.get(three) ?? 0) + 1);
|
|
254
|
+
if (seen.get(three) >= 6) return "";
|
|
255
|
+
}
|
|
256
|
+
return unquoted;
|
|
268
257
|
}
|
|
269
258
|
|
|
270
259
|
/**
|
|
@@ -344,167 +333,6 @@ export function createAppleSpeech({
|
|
|
344
333
|
};
|
|
345
334
|
}
|
|
346
335
|
|
|
347
|
-
// The models' own work, run on the model thread (onnxWorker.mjs) beside the sessions; each returns what the daemon's
|
|
348
|
-
// side turns into words. Samples are sent as a copy of just their piece: a piece of a longer recording sent as is
|
|
349
|
-
// would copy the whole recording to the other thread.
|
|
350
|
-
/** The VAD's speech probability for each 32 ms frame: a job on the model thread (onnxWorker.mjs). */
|
|
351
|
-
export async function vadProbabilities({ ort, graph }, file, samples) {
|
|
352
|
-
const vad = await graph(file);
|
|
353
|
-
const rate = new ort.Tensor("int64", BigInt64Array.from([BigInt(RATE)]), []);
|
|
354
|
-
let state = new ort.Tensor("float32", new Float32Array(2 * 128), [2, 1, 128]);
|
|
355
|
-
const input = new Float32Array(CONTEXT + FRAME);
|
|
356
|
-
const probs = new Float32Array(Math.floor(samples.length / FRAME));
|
|
357
|
-
for (let i = 0; i < probs.length; i += 1) {
|
|
358
|
-
// Each frame carries the last 64 samples of the one before it, as the model was trained.
|
|
359
|
-
input.copyWithin(0, FRAME, FRAME + CONTEXT);
|
|
360
|
-
input.set(samples.subarray(i * FRAME, (i + 1) * FRAME), CONTEXT);
|
|
361
|
-
const out = await vad.run({
|
|
362
|
-
input: new ort.Tensor("float32", input.slice(), [1, CONTEXT + FRAME]),
|
|
363
|
-
state,
|
|
364
|
-
sr: rate,
|
|
365
|
-
});
|
|
366
|
-
probs[i] = out.output.data[0];
|
|
367
|
-
state = out.stateN;
|
|
368
|
-
}
|
|
369
|
-
return probs;
|
|
370
|
-
}
|
|
371
|
-
|
|
372
|
-
/** Moonshine's token ids for one piece of speech: a job on the model thread. */
|
|
373
|
-
export async function moonshineIds({ ort, graph }, files, config, samples) {
|
|
374
|
-
const [encoder, decoder] = await Promise.all([
|
|
375
|
-
graph(files.encoder),
|
|
376
|
-
graph(files.decoder),
|
|
377
|
-
]);
|
|
378
|
-
const { last_hidden_state: states } = await encoder.run({
|
|
379
|
-
input_values: new ort.Tensor("float32", samples, [1, samples.length]),
|
|
380
|
-
});
|
|
381
|
-
const layers = config.decoder_num_hidden_layers,
|
|
382
|
-
heads = config.decoder_num_key_value_heads,
|
|
383
|
-
width = config.hidden_size / config.decoder_num_attention_heads;
|
|
384
|
-
const empty = new ort.Tensor("float32", new Float32Array(0), [
|
|
385
|
-
1,
|
|
386
|
-
heads,
|
|
387
|
-
0,
|
|
388
|
-
width,
|
|
389
|
-
]);
|
|
390
|
-
const past = {};
|
|
391
|
-
for (let i = 0; i < layers; i += 1)
|
|
392
|
-
for (const part of ["decoder", "encoder"])
|
|
393
|
-
for (const kv of ["key", "value"])
|
|
394
|
-
past[`past_key_values.${i}.${part}.${kv}`] = empty;
|
|
395
|
-
const limit = Math.ceil((samples.length / RATE) * TOKENS_PER_SECOND) + 4;
|
|
396
|
-
const ids = [];
|
|
397
|
-
let token = config.decoder_start_token_id;
|
|
398
|
-
for (let step = 0; step < limit; step += 1) {
|
|
399
|
-
const out = await decoder.run({
|
|
400
|
-
input_ids: new ort.Tensor(
|
|
401
|
-
"int64",
|
|
402
|
-
BigInt64Array.from([BigInt(token)]),
|
|
403
|
-
[1, 1]
|
|
404
|
-
),
|
|
405
|
-
encoder_hidden_states: states,
|
|
406
|
-
use_cache_branch: new ort.Tensor(
|
|
407
|
-
"bool",
|
|
408
|
-
Uint8Array.from([step > 0 ? 1 : 0]),
|
|
409
|
-
[1]
|
|
410
|
-
),
|
|
411
|
-
...past,
|
|
412
|
-
});
|
|
413
|
-
const logits = out.logits.data;
|
|
414
|
-
let best = 0;
|
|
415
|
-
for (let v = 1; v < logits.length; v += 1)
|
|
416
|
-
if (logits[v] > logits[best]) best = v;
|
|
417
|
-
// The encoder's keys and values come from the first step and stay; the decoder's grow by one each step.
|
|
418
|
-
for (let i = 0; i < layers; i += 1)
|
|
419
|
-
for (const part of step === 0 ? ["decoder", "encoder"] : ["decoder"])
|
|
420
|
-
for (const kv of ["key", "value"])
|
|
421
|
-
past[`past_key_values.${i}.${part}.${kv}`] =
|
|
422
|
-
out[`present.${i}.${part}.${kv}`];
|
|
423
|
-
if (best === config.eos_token_id) break;
|
|
424
|
-
ids.push(best);
|
|
425
|
-
token = best;
|
|
426
|
-
}
|
|
427
|
-
return ids;
|
|
428
|
-
}
|
|
429
|
-
|
|
430
|
-
/**
|
|
431
|
-
* Parakeet's token ids for one piece of speech, a job on the model thread: its feature extractor and encoder, then a
|
|
432
|
-
* greedy token-and-duration decode over the encoder's frames (as NeMo and onnx-asr do): each step names a token (or
|
|
433
|
-
* blank) and how many frames to move. `vocabulary` is how many tokens there are; `blank` the blank's id.
|
|
434
|
-
*/
|
|
435
|
-
export async function parakeetIds(
|
|
436
|
-
{ ort, graph },
|
|
437
|
-
files,
|
|
438
|
-
{ vocabulary, blank },
|
|
439
|
-
samples
|
|
440
|
-
) {
|
|
441
|
-
const [features, encoder, joint] = await Promise.all(
|
|
442
|
-
[files.features, files.encoder, files.joint].map((file) => graph(file))
|
|
443
|
-
);
|
|
444
|
-
const count = new ort.Tensor(
|
|
445
|
-
"int64",
|
|
446
|
-
BigInt64Array.from([BigInt(samples.length)]),
|
|
447
|
-
[1]
|
|
448
|
-
);
|
|
449
|
-
const heard = await features.run({
|
|
450
|
-
waveforms: new ort.Tensor("float32", samples, [1, samples.length]),
|
|
451
|
-
waveforms_lens: count,
|
|
452
|
-
});
|
|
453
|
-
const { outputs, encoded_lengths } = await encoder.run({
|
|
454
|
-
audio_signal: heard.features,
|
|
455
|
-
length: heard.features_lens,
|
|
456
|
-
});
|
|
457
|
-
const [, width, frames] = outputs.dims,
|
|
458
|
-
length = Number(encoded_lengths.data[0]);
|
|
459
|
-
const frame = (t) => {
|
|
460
|
-
const v = new Float32Array(width);
|
|
461
|
-
for (let d = 0; d < width; d += 1) v[d] = outputs.data[d * frames + t];
|
|
462
|
-
return v;
|
|
463
|
-
};
|
|
464
|
-
let states = [1, 2].map(
|
|
465
|
-
() => new ort.Tensor("float32", new Float32Array(2 * 640), [2, 1, 640])
|
|
466
|
-
);
|
|
467
|
-
const ids = [];
|
|
468
|
-
for (let t = 0, emitted = 0; t < length;) {
|
|
469
|
-
const out = await joint.run({
|
|
470
|
-
encoder_outputs: new ort.Tensor("float32", frame(t), [1, width, 1]),
|
|
471
|
-
targets: new ort.Tensor(
|
|
472
|
-
"int32",
|
|
473
|
-
Int32Array.from([ids.at(-1) ?? blank]),
|
|
474
|
-
[1, 1]
|
|
475
|
-
),
|
|
476
|
-
target_length: new ort.Tensor("int32", Int32Array.from([1]), [1]),
|
|
477
|
-
input_states_1: states[0],
|
|
478
|
-
input_states_2: states[1],
|
|
479
|
-
});
|
|
480
|
-
const scores = out.outputs.data;
|
|
481
|
-
let token = 0,
|
|
482
|
-
step = 0;
|
|
483
|
-
for (let v = 1; v < vocabulary; v += 1)
|
|
484
|
-
if (scores[v] > scores[token]) token = v;
|
|
485
|
-
for (let k = 1; k < scores.length - vocabulary; k += 1)
|
|
486
|
-
if (scores[vocabulary + k] > scores[vocabulary + step]) step = k;
|
|
487
|
-
if (token !== blank) {
|
|
488
|
-
states = [out.output_states_1, out.output_states_2];
|
|
489
|
-
ids.push(token);
|
|
490
|
-
emitted += 1;
|
|
491
|
-
}
|
|
492
|
-
// At most ten tokens on one frame, as the model's decoding allows.
|
|
493
|
-
if (step > 0) {
|
|
494
|
-
t += step;
|
|
495
|
-
emitted = 0;
|
|
496
|
-
} else if (token === blank || emitted === 10) {
|
|
497
|
-
t += 1;
|
|
498
|
-
emitted = 0;
|
|
499
|
-
}
|
|
500
|
-
}
|
|
501
|
-
return ids;
|
|
502
|
-
}
|
|
503
|
-
|
|
504
|
-
const VAD_JOB = { module: import.meta.url, name: "vadProbabilities" };
|
|
505
|
-
const MOONSHINE_JOB = { module: import.meta.url, name: "moonshineIds" };
|
|
506
|
-
const PARAKEET_JOB = { module: import.meta.url, name: "parakeetIds" };
|
|
507
|
-
|
|
508
336
|
export function createSpeech({
|
|
509
337
|
inference,
|
|
510
338
|
root,
|
|
@@ -512,45 +340,13 @@ export function createSpeech({
|
|
|
512
340
|
vocabulary = () => [],
|
|
513
341
|
apple = null,
|
|
514
342
|
fetchImpl,
|
|
343
|
+
getLlama = ensureLlama,
|
|
515
344
|
log = () => {},
|
|
516
345
|
}) {
|
|
517
|
-
const
|
|
518
|
-
speechEntry = modelById("speech", list);
|
|
519
|
-
const fileOf = (entry, pattern) =>
|
|
520
|
-
entry.files.find((file) => pattern.test(file.path))?.path;
|
|
521
|
-
const engine = speechEntry?.engine ?? "moonshine";
|
|
522
|
-
let tokens = null,
|
|
523
|
-
config = null;
|
|
524
|
-
const load = () => {
|
|
525
|
-
if (tokens) return;
|
|
526
|
-
const dir = modelDir(root, speechEntry);
|
|
527
|
-
if (engine === "parakeet") {
|
|
528
|
-
tokens = vocabText(
|
|
529
|
-
fs.readFileSync(
|
|
530
|
-
path.join(dir, fileOf(speechEntry, /(^|\/)vocab\.txt$/)),
|
|
531
|
-
"utf8"
|
|
532
|
-
)
|
|
533
|
-
);
|
|
534
|
-
return;
|
|
535
|
-
}
|
|
536
|
-
tokens = tokenText(
|
|
537
|
-
JSON.parse(
|
|
538
|
-
fs.readFileSync(
|
|
539
|
-
path.join(dir, fileOf(speechEntry, /(^|\/)tokenizer\.json$/)),
|
|
540
|
-
"utf8"
|
|
541
|
-
)
|
|
542
|
-
)
|
|
543
|
-
);
|
|
544
|
-
config = JSON.parse(
|
|
545
|
-
fs.readFileSync(
|
|
546
|
-
path.join(dir, fileOf(speechEntry, /(^|\/)config\.json$/)),
|
|
547
|
-
"utf8"
|
|
548
|
-
)
|
|
549
|
-
);
|
|
550
|
-
};
|
|
346
|
+
const omniEntry = modelById(OMNI, list);
|
|
551
347
|
|
|
552
348
|
// ── Apple's transcriber, where this Mac has it ──────────────────────────────
|
|
553
|
-
// Asked once what it can do; while its model comes down, each status asks again. Without it,
|
|
349
|
+
// Asked once what it can do; while its model comes down, each status asks again. Without it, Omni.
|
|
554
350
|
let macOS = apple ? null : { available: false };
|
|
555
351
|
const probe = () =>
|
|
556
352
|
apple.ask({ op: "status" }).then((answer) => {
|
|
@@ -558,7 +354,7 @@ export function createSpeech({
|
|
|
558
354
|
});
|
|
559
355
|
if (apple) void probe();
|
|
560
356
|
const onMac = () => Boolean(macOS?.available);
|
|
561
|
-
const ids = () => (onMac() ? [
|
|
357
|
+
const ids = () => (onMac() ? [] : [OMNI]);
|
|
562
358
|
|
|
563
359
|
// ── Downloading on first use ────────────────────────────────────────────────
|
|
564
360
|
let downloading = null,
|
|
@@ -578,7 +374,7 @@ export function createSpeech({
|
|
|
578
374
|
if (ready())
|
|
579
375
|
return {
|
|
580
376
|
state: "ready",
|
|
581
|
-
model: onMac() ? "Apple's transcriber" :
|
|
377
|
+
model: onMac() ? "Apple's transcriber" : omniEntry.name,
|
|
582
378
|
};
|
|
583
379
|
if (onMac() && !macOS.installed) {
|
|
584
380
|
void probe();
|
|
@@ -605,14 +401,14 @@ export function createSpeech({
|
|
|
605
401
|
? { state: "failed", error: failure, percent, bytes: total }
|
|
606
402
|
: { state: "absent", percent, bytes: total };
|
|
607
403
|
};
|
|
608
|
-
/** Starts the download if
|
|
404
|
+
/** Starts the download if a model is missing; answers where it stands. */
|
|
609
405
|
const ensure = () => {
|
|
610
406
|
if (!ready() && !downloading && macOS !== null) {
|
|
611
407
|
failure = null;
|
|
612
408
|
log(
|
|
613
409
|
onMac()
|
|
614
410
|
? "speech: getting Apple's transcriber ready"
|
|
615
|
-
: "speech: downloading the
|
|
411
|
+
: "speech: downloading the model that hears"
|
|
616
412
|
);
|
|
617
413
|
const mac =
|
|
618
414
|
onMac() && !macOS.installed
|
|
@@ -621,16 +417,19 @@ export function createSpeech({
|
|
|
621
417
|
return probe();
|
|
622
418
|
})
|
|
623
419
|
: null;
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
420
|
+
const omni = onMac()
|
|
421
|
+
? null
|
|
422
|
+
: Promise.all([
|
|
423
|
+
getLlama({ root, log, ...(fetchImpl ? { fetchImpl } : {}) }),
|
|
424
|
+
ensureModels({
|
|
425
|
+
root,
|
|
426
|
+
ids: ids(),
|
|
427
|
+
list,
|
|
428
|
+
...(fetchImpl ? { fetchImpl } : {}),
|
|
429
|
+
log,
|
|
430
|
+
}),
|
|
431
|
+
]);
|
|
432
|
+
downloading = Promise.all([omni, mac])
|
|
634
433
|
.then(
|
|
635
434
|
() => log("speech: ready"),
|
|
636
435
|
(error) => {
|
|
@@ -646,67 +445,46 @@ export function createSpeech({
|
|
|
646
445
|
};
|
|
647
446
|
|
|
648
447
|
// ── Hearing ─────────────────────────────────────────────────────────────────
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
const probabilities = (samples, lane) =>
|
|
652
|
-
inference.onnx("vad", VAD_JOB, [vadFile, samples.slice()], { lane });
|
|
653
|
-
|
|
654
|
-
/** Where in the samples someone spoke: [start, end) pairs, none for noise or silence. */
|
|
655
|
-
const voiced = async (samples, { lane = "interactive" } = {}) =>
|
|
656
|
-
speechSpans(await probabilities(samples, lane), { total: samples.length });
|
|
657
|
-
|
|
658
|
-
/** Moonshine's words for one piece of speech. */
|
|
659
|
-
const moonshine = async (samples, { lane = "interactive" } = {}) => {
|
|
660
|
-
load();
|
|
661
|
-
const files = {
|
|
662
|
-
encoder: fileOf(speechEntry, /encoder[^/]*\.onnx$/),
|
|
663
|
-
decoder: fileOf(speechEntry, /decoder[^/]*\.onnx$/),
|
|
664
|
-
};
|
|
665
|
-
const ids = await inference.onnx(
|
|
666
|
-
"speech",
|
|
667
|
-
MOONSHINE_JOB,
|
|
668
|
-
[files, config, samples.slice()],
|
|
669
|
-
{ lane }
|
|
670
|
-
);
|
|
671
|
-
return tokens.decode(ids).trim();
|
|
672
|
-
};
|
|
448
|
+
/** Where there is sound. Omni and Apple's transcriber decide whether any of it is words. */
|
|
449
|
+
const voiced = async (samples) => soundSpans(samples);
|
|
673
450
|
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
451
|
+
const askOmni = async (wav, question, maxTokens, lane) => {
|
|
452
|
+
const reply = await inference.chat(
|
|
453
|
+
OMNI,
|
|
454
|
+
{
|
|
455
|
+
temperature: 0.1,
|
|
456
|
+
top_k: 50,
|
|
457
|
+
repeat_penalty: 1.1,
|
|
458
|
+
max_tokens: maxTokens,
|
|
459
|
+
messages: [
|
|
460
|
+
{ role: "system", content: SPEAKING_JOB.system },
|
|
461
|
+
{
|
|
462
|
+
role: "user",
|
|
463
|
+
content: [
|
|
464
|
+
{
|
|
465
|
+
type: "input_audio",
|
|
466
|
+
input_audio: { data: wav, format: "wav" },
|
|
467
|
+
},
|
|
468
|
+
{ type: "text", text: question },
|
|
469
|
+
],
|
|
470
|
+
},
|
|
471
|
+
],
|
|
472
|
+
},
|
|
690
473
|
{ lane }
|
|
691
474
|
);
|
|
692
|
-
return
|
|
475
|
+
return String(reply?.choices?.[0]?.message?.content ?? "");
|
|
693
476
|
};
|
|
694
|
-
const recognise = engine === "parakeet" ? parakeet : moonshine;
|
|
695
477
|
|
|
696
|
-
/**
|
|
697
|
-
const
|
|
698
|
-
const
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
for (let s = start; s < end; s += MAX_PIECE_SECONDS * RATE)
|
|
704
|
-
out.push([s, Math.min(end, s + MAX_PIECE_SECONDS * RATE)]);
|
|
705
|
-
}
|
|
706
|
-
return out.map(([start, end]) => samples.subarray(start, end));
|
|
478
|
+
/** Omni's words for one piece, nothing when no one is speaking in it. */
|
|
479
|
+
const hearPiece = async (piece, lane) => {
|
|
480
|
+
const wav = Buffer.from(encodeWav(piece, RATE)).toString("base64");
|
|
481
|
+
const speaking = await askOmni(wav, SPEAKING_JOB.question, 4, lane);
|
|
482
|
+
if (!/\byes\b/i.test(speaking)) return "";
|
|
483
|
+
const tokens = Math.ceil((piece.length / RATE) * TOKENS_PER_SECOND) + 20;
|
|
484
|
+
return spokenWords(await askOmni(wav, WRITE_WORDS, tokens, lane));
|
|
707
485
|
};
|
|
708
486
|
|
|
709
|
-
/** Audio (WAV bytes, or { sampleRate, samples }) to text: { text, heard, ms, model, raw }; nothing heard is
|
|
487
|
+
/** Audio (WAV bytes, or { sampleRate, samples }) to text: { text, heard, ms, model, raw }; nothing heard is no text. */
|
|
710
488
|
const transcribe = async (input, { lane = "interactive" } = {}) => {
|
|
711
489
|
const started = performance.now();
|
|
712
490
|
if (!ready()) throw new Error("the speech models are not downloaded yet");
|
|
@@ -718,19 +496,20 @@ export function createSpeech({
|
|
|
718
496
|
audio.sampleRate === RATE
|
|
719
497
|
? Float32Array.from(audio.samples)
|
|
720
498
|
: resample(audio.samples, audio.sampleRate);
|
|
721
|
-
const spans =
|
|
499
|
+
const spans = soundSpans(samples);
|
|
500
|
+
const model = onMac() ? "Apple's transcriber" : omniEntry.name;
|
|
501
|
+
const result = (raw, text = raw) => ({
|
|
502
|
+
text,
|
|
503
|
+
heard: Boolean(text),
|
|
504
|
+
ms: Math.round(performance.now() - started),
|
|
505
|
+
model,
|
|
506
|
+
raw,
|
|
507
|
+
});
|
|
508
|
+
if (!spans.length) return result("");
|
|
722
509
|
const entries = vocabularyEntries(await vocabulary());
|
|
723
510
|
if (onMac()) {
|
|
724
|
-
// Apple's model hears the whole recording, steered toward the names the relay knows; then names are put right
|
|
725
|
-
//
|
|
726
|
-
if (!spans.length)
|
|
727
|
-
return {
|
|
728
|
-
text: "",
|
|
729
|
-
heard: false,
|
|
730
|
-
ms: Math.round(performance.now() - started),
|
|
731
|
-
model: "Apple's transcriber",
|
|
732
|
-
raw: "",
|
|
733
|
-
};
|
|
511
|
+
// Apple's model hears the whole recording, steered toward the names the relay knows; then names are put right, with
|
|
512
|
+
// the Mac's own dictionary as the words a name never replaces.
|
|
734
513
|
const file = path.join(
|
|
735
514
|
fs.mkdtempSync(path.join(os.tmpdir(), "harness-talk-")),
|
|
736
515
|
"talk.wav"
|
|
@@ -743,49 +522,25 @@ export function createSpeech({
|
|
|
743
522
|
names: entries.map((entry) => entry.display).slice(0, APPLE_NAMES),
|
|
744
523
|
});
|
|
745
524
|
if (!answer.ok) throw new Error(answer.error);
|
|
746
|
-
const
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
heard: Boolean(text),
|
|
752
|
-
ms: Math.round(performance.now() - started),
|
|
753
|
-
model: "Apple's transcriber",
|
|
754
|
-
raw: answer.text,
|
|
755
|
-
};
|
|
525
|
+
const raw = spokenWords(answer.text);
|
|
526
|
+
return result(
|
|
527
|
+
answer.text,
|
|
528
|
+
raw ? snapNames(raw, entries, macWords()).text : ""
|
|
529
|
+
);
|
|
756
530
|
} finally {
|
|
757
531
|
fs.rmSync(path.dirname(file), { recursive: true, force: true });
|
|
758
532
|
}
|
|
759
533
|
}
|
|
760
534
|
const said = [];
|
|
761
|
-
for (const piece of
|
|
762
|
-
const words = await
|
|
535
|
+
for (const piece of quietPieces(samples, spans)) {
|
|
536
|
+
const words = await hearPiece(piece, lane);
|
|
763
537
|
if (words) said.push(words);
|
|
764
538
|
}
|
|
765
539
|
const raw = said.join(" ").trim();
|
|
766
|
-
|
|
767
|
-
const text = raw ? snapNames(raw, entries, tokens.common).text : "";
|
|
768
|
-
return {
|
|
769
|
-
text,
|
|
770
|
-
heard: Boolean(text),
|
|
771
|
-
ms: Math.round(performance.now() - started),
|
|
772
|
-
model: speechEntry.name,
|
|
773
|
-
raw,
|
|
774
|
-
};
|
|
540
|
+
return result(raw, raw ? snapNames(raw, entries, macWords()).text : "");
|
|
775
541
|
};
|
|
776
542
|
|
|
777
|
-
return {
|
|
778
|
-
ready,
|
|
779
|
-
status,
|
|
780
|
-
ensure,
|
|
781
|
-
transcribe,
|
|
782
|
-
voiced,
|
|
783
|
-
recognise,
|
|
784
|
-
common: () => {
|
|
785
|
-
load();
|
|
786
|
-
return tokens.common;
|
|
787
|
-
},
|
|
788
|
-
};
|
|
543
|
+
return { ready, status, ensure, transcribe, voiced };
|
|
789
544
|
}
|
|
790
545
|
|
|
791
546
|
// ── Names ─────────────────────────────────────────────────────────────────────
|