tryharness 0.7.23 → 0.7.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/opencode-plugin.js +48 -11
- package/assets/pi-extension.ts +105 -26
- package/cli.mjs +1238 -322
- package/lib/activity.mjs +197 -40
- package/lib/adapters.mjs +337 -54
- package/lib/agentnode.mjs +175 -35
- package/lib/atlas.mjs +383 -83
- package/lib/cards.mjs +174 -40
- package/lib/claudemem.mjs +74 -18
- package/lib/commands.mjs +13 -5
- package/lib/context.mjs +409 -71
- package/lib/control-client.mjs +5 -2
- package/lib/coordinator.mjs +274 -57
- package/lib/daemon.mjs +5248 -1024
- package/lib/desktop.mjs +149 -35
- package/lib/digest.mjs +54 -13
- package/lib/environment.mjs +103 -21
- package/lib/files.mjs +246 -13
- package/lib/firewall.mjs +169 -35
- package/lib/folders.mjs +179 -36
- package/lib/frames.mjs +214 -40
- package/lib/inference.mjs +463 -126
- package/lib/learn.mjs +78 -16
- package/lib/live.mjs +286 -61
- package/lib/localmodels.mjs +424 -83
- package/lib/mcp-stdio.mjs +868 -151
- package/lib/media.mjs +138 -28
- package/lib/members.json +1 -1
- package/lib/members.mjs +35 -9
- package/lib/models.mjs +342 -80
- package/lib/network.mjs +201 -42
- package/lib/notify.mjs +49 -12
- package/lib/nudge.mjs +184 -33
- package/lib/ocr.mjs +223 -60
- package/lib/onboard.mjs +321 -70
- package/lib/parsers.mjs +132 -41
- package/lib/paths.mjs +33 -10
- package/lib/personal.mjs +51 -13
- package/lib/phones.mjs +382 -60
- package/lib/places.mjs +555 -107
- package/lib/provenance.mjs +19 -4
- package/lib/receipts.mjs +51 -11
- package/lib/redact.mjs +512 -116
- package/lib/runner.mjs +345 -55
- package/lib/runtime-install.mjs +55 -14
- package/lib/screens.mjs +154 -38
- package/lib/sealed.mjs +157 -28
- package/lib/secrets.mjs +18 -5
- package/lib/service.mjs +170 -34
- package/lib/snippets.mjs +94 -18
- package/lib/speech.mjs +688 -175
- package/lib/store.mjs +1654 -307
- package/lib/swarm.mjs +596 -117
- package/lib/t3.mjs +400 -79
- package/lib/trails.mjs +285 -61
- package/lib/ui.mjs +95 -23
- package/lib/vectors.mjs +5 -2
- package/lib/wakes.mjs +284 -68
- package/lib/watches.mjs +247 -48
- package/lib/worktrees.mjs +186 -38
- package/package.json +3 -3
package/lib/speech.mjs
CHANGED
|
@@ -2,7 +2,14 @@ import { spawn } from "node:child_process";
|
|
|
2
2
|
import fs from "node:fs";
|
|
3
3
|
import os from "node:os";
|
|
4
4
|
import path from "node:path";
|
|
5
|
-
import {
|
|
5
|
+
import {
|
|
6
|
+
MODELS,
|
|
7
|
+
ensureModels,
|
|
8
|
+
modelById,
|
|
9
|
+
modelDir,
|
|
10
|
+
modelFile,
|
|
11
|
+
modelsPresent,
|
|
12
|
+
} from "./localmodels.mjs";
|
|
6
13
|
|
|
7
14
|
// Talk in the buddy, heard on this computer only. Silero VAD finds where someone spoke, so a held key over a fan, a
|
|
8
15
|
// keyboard or silence writes nothing. On a Mac with macOS 26, Apple's own transcriber writes the words (the buddy's
|
|
@@ -25,37 +32,70 @@ const TOKENS_PER_SECOND = 8;
|
|
|
25
32
|
const APPLE_NAMES = 300;
|
|
26
33
|
// The Mac's own word list: on the Apple path, a name of the person's never replaces a word in it.
|
|
27
34
|
let dictionary = null;
|
|
28
|
-
const macWords = () =>
|
|
35
|
+
const macWords = () =>
|
|
36
|
+
(dictionary ??= new Set(
|
|
37
|
+
fs.existsSync("/usr/share/dict/words")
|
|
38
|
+
? fs
|
|
39
|
+
.readFileSync("/usr/share/dict/words", "utf8")
|
|
40
|
+
.split("\n")
|
|
41
|
+
.filter((word) => /^[a-z]{2,}$/.test(word))
|
|
42
|
+
: []
|
|
43
|
+
));
|
|
29
44
|
|
|
30
45
|
/** A PCM or float WAV (any channel count) as mono samples at its own rate. */
|
|
31
46
|
export function decodeWav(bytes) {
|
|
32
47
|
const data = bytes instanceof Uint8Array ? bytes : new Uint8Array(bytes);
|
|
33
48
|
const view = new DataView(data.buffer, data.byteOffset, data.byteLength);
|
|
34
|
-
const tag = (at) =>
|
|
35
|
-
|
|
36
|
-
|
|
49
|
+
const tag = (at) =>
|
|
50
|
+
String.fromCharCode(data[at], data[at + 1], data[at + 2], data[at + 3]);
|
|
51
|
+
if (data.length < 12 || tag(0) !== "RIFF" || tag(8) !== "WAVE")
|
|
52
|
+
throw new Error("the recording is not a WAV file");
|
|
53
|
+
let format = null,
|
|
54
|
+
body = null;
|
|
37
55
|
for (let at = 12; at + 8 <= data.length;) {
|
|
38
56
|
const size = view.getUint32(at + 4, true);
|
|
39
57
|
if (tag(at) === "fmt ") {
|
|
40
58
|
const code = view.getUint16(at + 8, true);
|
|
41
|
-
format = {
|
|
42
|
-
|
|
59
|
+
format = {
|
|
60
|
+
code: code === 0xfffe ? view.getUint16(at + 32, true) : code,
|
|
61
|
+
channels: view.getUint16(at + 10, true),
|
|
62
|
+
rate: view.getUint32(at + 12, true),
|
|
63
|
+
bits: view.getUint16(at + 22, true),
|
|
64
|
+
};
|
|
65
|
+
} else if (tag(at) === "data")
|
|
66
|
+
body = { at: at + 8, size: Math.min(size, data.length - at - 8) };
|
|
43
67
|
at += 8 + size + (size & 1);
|
|
44
68
|
}
|
|
45
|
-
if (!format || !body || !format.channels || !format.rate)
|
|
69
|
+
if (!format || !body || !format.channels || !format.rate)
|
|
70
|
+
throw new Error("the WAV file has no audio in it");
|
|
46
71
|
const { code, channels, bits } = format;
|
|
47
72
|
const width = bits / 8;
|
|
48
|
-
const read =
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
73
|
+
const read =
|
|
74
|
+
code === 3
|
|
75
|
+
? bits === 64
|
|
76
|
+
? (at) => view.getFloat64(at, true)
|
|
77
|
+
: (at) => view.getFloat32(at, true)
|
|
78
|
+
: bits === 8
|
|
79
|
+
? (at) => (view.getUint8(at) - 128) / 128
|
|
80
|
+
: bits === 16
|
|
81
|
+
? (at) => view.getInt16(at, true) / 32_768
|
|
82
|
+
: bits === 24
|
|
83
|
+
? (at) =>
|
|
84
|
+
(view.getUint8(at) |
|
|
85
|
+
(view.getUint8(at + 1) << 8) |
|
|
86
|
+
(view.getInt8(at + 2) << 16)) /
|
|
87
|
+
8_388_608
|
|
88
|
+
: bits === 32
|
|
89
|
+
? (at) => view.getInt32(at, true) / 2_147_483_648
|
|
90
|
+
: null;
|
|
91
|
+
if (!read || (code !== 1 && code !== 3))
|
|
92
|
+
throw new Error(`a ${bits}-bit WAV of format ${code} can't be read`);
|
|
54
93
|
const frames = Math.floor(body.size / (width * channels));
|
|
55
94
|
const samples = new Float32Array(frames);
|
|
56
95
|
for (let i = 0; i < frames; i += 1) {
|
|
57
96
|
let sum = 0;
|
|
58
|
-
for (let c = 0; c < channels; c += 1)
|
|
97
|
+
for (let c = 0; c < channels; c += 1)
|
|
98
|
+
sum += read(body.at + (i * channels + c) * width);
|
|
59
99
|
samples[i] = sum / channels;
|
|
60
100
|
}
|
|
61
101
|
return { sampleRate: format.rate, samples };
|
|
@@ -64,11 +104,24 @@ export function decodeWav(bytes) {
|
|
|
64
104
|
/** 16-bit mono WAV bytes for samples at a rate (the buddy's recordings, the eval's clips). */
|
|
65
105
|
export function encodeWav(samples, sampleRate = RATE) {
|
|
66
106
|
const out = Buffer.alloc(44 + samples.length * 2);
|
|
67
|
-
out.write("RIFF", 0);
|
|
68
|
-
out.
|
|
69
|
-
out.
|
|
70
|
-
out.write("
|
|
71
|
-
|
|
107
|
+
out.write("RIFF", 0);
|
|
108
|
+
out.writeUInt32LE(36 + samples.length * 2, 4);
|
|
109
|
+
out.write("WAVE", 8);
|
|
110
|
+
out.write("fmt ", 12);
|
|
111
|
+
out.writeUInt32LE(16, 16);
|
|
112
|
+
out.writeUInt16LE(1, 20);
|
|
113
|
+
out.writeUInt16LE(1, 22);
|
|
114
|
+
out.writeUInt32LE(sampleRate, 24);
|
|
115
|
+
out.writeUInt32LE(sampleRate * 2, 28);
|
|
116
|
+
out.writeUInt16LE(2, 32);
|
|
117
|
+
out.writeUInt16LE(16, 34);
|
|
118
|
+
out.write("data", 36);
|
|
119
|
+
out.writeUInt32LE(samples.length * 2, 40);
|
|
120
|
+
for (let i = 0; i < samples.length; i += 1)
|
|
121
|
+
out.writeInt16LE(
|
|
122
|
+
Math.max(-32_768, Math.min(32_767, Math.round(samples[i] * 32_767))),
|
|
123
|
+
44 + i * 2
|
|
124
|
+
);
|
|
72
125
|
return out;
|
|
73
126
|
}
|
|
74
127
|
|
|
@@ -80,7 +133,10 @@ const KERNEL = (() => {
|
|
|
80
133
|
for (let i = 0; i <= ZEROS * STEPS; i += 1) {
|
|
81
134
|
const x = i / STEPS;
|
|
82
135
|
const sinc = x === 0 ? 1 : Math.sin(Math.PI * x) / (Math.PI * x);
|
|
83
|
-
const w =
|
|
136
|
+
const w =
|
|
137
|
+
0.42 +
|
|
138
|
+
0.5 * Math.cos((Math.PI * x) / ZEROS) +
|
|
139
|
+
0.08 * Math.cos((2 * Math.PI * x) / ZEROS);
|
|
84
140
|
table[i] = sinc * w;
|
|
85
141
|
}
|
|
86
142
|
return table;
|
|
@@ -96,10 +152,17 @@ export function resample(samples, from, to = RATE) {
|
|
|
96
152
|
for (let n = 0; n < out.length; n += 1) {
|
|
97
153
|
const centre = n / ratio;
|
|
98
154
|
let sum = 0;
|
|
99
|
-
for (
|
|
155
|
+
for (
|
|
156
|
+
let k = Math.max(0, Math.ceil(centre - reach)),
|
|
157
|
+
last = Math.min(samples.length - 1, Math.floor(centre + reach));
|
|
158
|
+
k <= last;
|
|
159
|
+
k += 1
|
|
160
|
+
) {
|
|
100
161
|
const at = Math.abs(centre - k) * scale * STEPS;
|
|
101
162
|
const i = at | 0;
|
|
102
|
-
if (i < ZEROS * STEPS)
|
|
163
|
+
if (i < ZEROS * STEPS)
|
|
164
|
+
sum +=
|
|
165
|
+
samples[k] * (KERNEL[i] + (KERNEL[i + 1] - KERNEL[i]) * (at - i));
|
|
103
166
|
}
|
|
104
167
|
out[n] = sum * scale;
|
|
105
168
|
}
|
|
@@ -110,14 +173,32 @@ export function resample(samples, from, to = RATE) {
|
|
|
110
173
|
* Where speech is, from the VAD's per-frame probabilities: [start, end) in samples. A stretch starts at 0.5 and ends
|
|
111
174
|
* after 500 ms under 0.35 (a pause between words keeps it going); one shorter than 250 ms is a click, not a word.
|
|
112
175
|
*/
|
|
113
|
-
export function speechSpans(
|
|
176
|
+
export function speechSpans(
|
|
177
|
+
probs,
|
|
178
|
+
{
|
|
179
|
+
total = probs.length * FRAME,
|
|
180
|
+
threshold = 0.5,
|
|
181
|
+
minSpeechMs = 250,
|
|
182
|
+
minSilenceMs = 500,
|
|
183
|
+
padMs = 250,
|
|
184
|
+
} = {}
|
|
185
|
+
) {
|
|
114
186
|
const frameMs = (FRAME / RATE) * 1000;
|
|
115
187
|
const off = threshold - 0.15;
|
|
116
188
|
const found = [];
|
|
117
|
-
let start = -1,
|
|
118
|
-
|
|
189
|
+
let start = -1,
|
|
190
|
+
quiet = -1;
|
|
191
|
+
const close = (end) => {
|
|
192
|
+
if ((end - start) * frameMs >= minSpeechMs)
|
|
193
|
+
found.push([start * FRAME, end * FRAME]);
|
|
194
|
+
start = -1;
|
|
195
|
+
quiet = -1;
|
|
196
|
+
};
|
|
119
197
|
for (let i = 0; i < probs.length; i += 1) {
|
|
120
|
-
if (start < 0) {
|
|
198
|
+
if (start < 0) {
|
|
199
|
+
if (probs[i] >= threshold) start = i;
|
|
200
|
+
continue;
|
|
201
|
+
}
|
|
121
202
|
if (probs[i] >= threshold) quiet = -1;
|
|
122
203
|
else if (probs[i] < off) {
|
|
123
204
|
if (quiet < 0) quiet = i;
|
|
@@ -138,8 +219,13 @@ export function speechSpans(probs, { total = probs.length * FRAME, threshold = 0
|
|
|
138
219
|
/** A Llama-style SentencePiece vocabulary as a decoder: ids to text, special tokens skipped, byte fallback kept. */
|
|
139
220
|
function tokenText(tokenizer) {
|
|
140
221
|
const pieces = [];
|
|
141
|
-
for (const [piece, id] of Object.entries(tokenizer.model.vocab))
|
|
142
|
-
|
|
222
|
+
for (const [piece, id] of Object.entries(tokenizer.model.vocab))
|
|
223
|
+
pieces[id] = piece;
|
|
224
|
+
const special = new Set(
|
|
225
|
+
tokenizer.added_tokens
|
|
226
|
+
.filter((token) => token.special)
|
|
227
|
+
.map((token) => token.id)
|
|
228
|
+
);
|
|
143
229
|
const decoder = new TextDecoder();
|
|
144
230
|
const decode = (ids) => {
|
|
145
231
|
const bytes = [];
|
|
@@ -152,15 +238,32 @@ function tokenText(tokenizer) {
|
|
|
152
238
|
return decoder.decode(Uint8Array.from(bytes)).replace(/^ /, "");
|
|
153
239
|
};
|
|
154
240
|
// Words the recogniser writes as one piece: the common words a name made of them must not be snapped from.
|
|
155
|
-
const common = new Set(
|
|
241
|
+
const common = new Set(
|
|
242
|
+
pieces
|
|
243
|
+
.filter(
|
|
244
|
+
(piece) => piece?.startsWith("▁") && /^[a-z]{2,}$/i.test(piece.slice(1))
|
|
245
|
+
)
|
|
246
|
+
.map((piece) => piece.slice(1).toLowerCase())
|
|
247
|
+
);
|
|
156
248
|
return { decode, common };
|
|
157
249
|
}
|
|
158
250
|
|
|
159
251
|
/** NeMo's vocabulary file ("piece id" a line) as a decoder, spaced as NeMo's own detokenizer does. */
|
|
160
252
|
function vocabText(text) {
|
|
161
|
-
const pieces = text
|
|
162
|
-
|
|
163
|
-
|
|
253
|
+
const pieces = text
|
|
254
|
+
.split("\n")
|
|
255
|
+
.filter(Boolean)
|
|
256
|
+
.map((line) => line.slice(0, line.lastIndexOf(" ")).replaceAll("▁", " "));
|
|
257
|
+
const decode = (ids) =>
|
|
258
|
+
ids
|
|
259
|
+
.map((id) => pieces[id])
|
|
260
|
+
.join("")
|
|
261
|
+
.replace(/^\s|\s\B|(\s)\b/g, (_, space) => (space ? " " : ""));
|
|
262
|
+
const common = new Set(
|
|
263
|
+
pieces
|
|
264
|
+
.filter((piece) => /^ [a-z]{2,}$/i.test(piece))
|
|
265
|
+
.map((piece) => piece.slice(1).toLowerCase())
|
|
266
|
+
);
|
|
164
267
|
return { pieces, decode, common, blank: pieces.indexOf("<blk>") };
|
|
165
268
|
}
|
|
166
269
|
|
|
@@ -168,79 +271,215 @@ function vocabText(text) {
|
|
|
168
271
|
* Apple's on-device transcriber through the buddy's HarnessSpeech helper (packages/desktop/helper, macOS 26): one process
|
|
169
272
|
* kept while Talk is in use, a JSON request a line, closed after five idle minutes. Null where there is no helper.
|
|
170
273
|
*/
|
|
171
|
-
export function createAppleSpeech({
|
|
274
|
+
export function createAppleSpeech({
|
|
275
|
+
helper,
|
|
276
|
+
locale = Intl.DateTimeFormat().resolvedOptions().locale || "en-US",
|
|
277
|
+
idleMs = 5 * 60_000,
|
|
278
|
+
log = () => {},
|
|
279
|
+
}) {
|
|
172
280
|
if (!helper || !fs.existsSync(helper)) return null;
|
|
173
|
-
let child = null,
|
|
281
|
+
let child = null,
|
|
282
|
+
buffer = "",
|
|
283
|
+
timer = null,
|
|
284
|
+
chain = Promise.resolve();
|
|
174
285
|
const waiting = [];
|
|
175
286
|
const start = () => {
|
|
176
|
-
const own = (child = spawn(helper, [], {
|
|
287
|
+
const own = (child = spawn(helper, [], {
|
|
288
|
+
stdio: ["pipe", "pipe", "ignore"],
|
|
289
|
+
}));
|
|
177
290
|
own.stdout.setEncoding("utf8");
|
|
178
291
|
own.stdout.on("data", (chunk) => {
|
|
179
292
|
buffer += chunk;
|
|
180
|
-
for (
|
|
293
|
+
for (
|
|
294
|
+
let at;
|
|
295
|
+
(at = buffer.indexOf("\n")) >= 0;
|
|
296
|
+
buffer = buffer.slice(at + 1)
|
|
297
|
+
)
|
|
298
|
+
waiting.shift()?.(buffer.slice(0, at));
|
|
299
|
+
});
|
|
300
|
+
own.on("error", (error) =>
|
|
301
|
+
log(`speech: Apple's transcriber could not start (${error.message})`)
|
|
302
|
+
);
|
|
303
|
+
own.on("exit", () => {
|
|
304
|
+
if (child === own) child = null;
|
|
305
|
+
buffer = "";
|
|
306
|
+
for (const answer of waiting.splice(0)) answer(null);
|
|
181
307
|
});
|
|
182
|
-
own.on("error", (error) => log(`speech: Apple's transcriber could not start (${error.message})`));
|
|
183
|
-
own.on("exit", () => { if (child === own) child = null; buffer = ""; for (const answer of waiting.splice(0)) answer(null); });
|
|
184
308
|
};
|
|
185
309
|
/** One request, answered in turn: the helper's answer, or { ok: false } when it stopped. */
|
|
186
|
-
const ask = (request) =>
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
310
|
+
const ask = (request) =>
|
|
311
|
+
(chain = chain
|
|
312
|
+
.then(
|
|
313
|
+
() =>
|
|
314
|
+
new Promise((resolve) => {
|
|
315
|
+
clearTimeout(timer);
|
|
316
|
+
if (!child) start();
|
|
317
|
+
waiting.push((line) => {
|
|
318
|
+
try {
|
|
319
|
+
resolve(
|
|
320
|
+
line === null
|
|
321
|
+
? { ok: false, error: "Apple's transcriber stopped" }
|
|
322
|
+
: JSON.parse(line)
|
|
323
|
+
);
|
|
324
|
+
} catch {
|
|
325
|
+
resolve({
|
|
326
|
+
ok: false,
|
|
327
|
+
error: "Apple's transcriber answered something unreadable",
|
|
328
|
+
});
|
|
329
|
+
}
|
|
330
|
+
});
|
|
331
|
+
child.stdin.write(`${JSON.stringify({ ...request, locale })}\n`);
|
|
332
|
+
})
|
|
333
|
+
)
|
|
334
|
+
.finally(() => {
|
|
335
|
+
timer = setTimeout(() => child?.stdin.end(), idleMs);
|
|
336
|
+
timer.unref?.();
|
|
337
|
+
}));
|
|
338
|
+
return {
|
|
339
|
+
ask,
|
|
340
|
+
stop: () => {
|
|
341
|
+
clearTimeout(timer);
|
|
342
|
+
child?.stdin.end();
|
|
343
|
+
},
|
|
344
|
+
};
|
|
193
345
|
}
|
|
194
346
|
|
|
195
|
-
export function createSpeech({
|
|
196
|
-
|
|
197
|
-
|
|
347
|
+
export function createSpeech({
|
|
348
|
+
inference,
|
|
349
|
+
root,
|
|
350
|
+
list = MODELS,
|
|
351
|
+
vocabulary = () => [],
|
|
352
|
+
apple = null,
|
|
353
|
+
fetchImpl,
|
|
354
|
+
log = () => {},
|
|
355
|
+
}) {
|
|
356
|
+
const vadEntry = modelById("vad", list),
|
|
357
|
+
speechEntry = modelById("speech", list);
|
|
358
|
+
const fileOf = (entry, pattern) =>
|
|
359
|
+
entry.files.find((file) => pattern.test(file.path))?.path;
|
|
198
360
|
const engine = speechEntry?.engine ?? "moonshine";
|
|
199
|
-
let tokens = null,
|
|
361
|
+
let tokens = null,
|
|
362
|
+
config = null;
|
|
200
363
|
const load = () => {
|
|
201
364
|
if (tokens) return;
|
|
202
365
|
const dir = modelDir(root, speechEntry);
|
|
203
|
-
if (engine === "parakeet") {
|
|
204
|
-
|
|
205
|
-
|
|
366
|
+
if (engine === "parakeet") {
|
|
367
|
+
tokens = vocabText(
|
|
368
|
+
fs.readFileSync(
|
|
369
|
+
path.join(dir, fileOf(speechEntry, /(^|\/)vocab\.txt$/)),
|
|
370
|
+
"utf8"
|
|
371
|
+
)
|
|
372
|
+
);
|
|
373
|
+
return;
|
|
374
|
+
}
|
|
375
|
+
tokens = tokenText(
|
|
376
|
+
JSON.parse(
|
|
377
|
+
fs.readFileSync(
|
|
378
|
+
path.join(dir, fileOf(speechEntry, /(^|\/)tokenizer\.json$/)),
|
|
379
|
+
"utf8"
|
|
380
|
+
)
|
|
381
|
+
)
|
|
382
|
+
);
|
|
383
|
+
config = JSON.parse(
|
|
384
|
+
fs.readFileSync(
|
|
385
|
+
path.join(dir, fileOf(speechEntry, /(^|\/)config\.json$/)),
|
|
386
|
+
"utf8"
|
|
387
|
+
)
|
|
388
|
+
);
|
|
206
389
|
};
|
|
207
390
|
|
|
208
391
|
// ── Apple's transcriber, where this Mac has it ──────────────────────────────
|
|
209
392
|
// Asked once what it can do; while its model comes down, each status asks again. Without it, Parakeet.
|
|
210
393
|
let macOS = apple ? null : { available: false };
|
|
211
|
-
const probe = () =>
|
|
394
|
+
const probe = () =>
|
|
395
|
+
apple.ask({ op: "status" }).then((answer) => {
|
|
396
|
+
macOS = answer.ok ? answer : (macOS ?? { available: false });
|
|
397
|
+
});
|
|
212
398
|
if (apple) void probe();
|
|
213
399
|
const onMac = () => Boolean(macOS?.available);
|
|
214
400
|
const ids = () => (onMac() ? ["vad"] : SPEECH_IDS);
|
|
215
401
|
|
|
216
402
|
// ── Downloading on first use ────────────────────────────────────────────────
|
|
217
|
-
let downloading = null,
|
|
218
|
-
|
|
219
|
-
const
|
|
403
|
+
let downloading = null,
|
|
404
|
+
failure = null;
|
|
405
|
+
const ready = () =>
|
|
406
|
+
modelsPresent(root, ids(), list) && (!onMac() || macOS.installed);
|
|
407
|
+
const files = () =>
|
|
408
|
+
ids().flatMap((id) => {
|
|
409
|
+
const entry = modelById(id, list);
|
|
410
|
+
return entry.files.map((file, index) => ({
|
|
411
|
+
file: modelFile(root, entry, index),
|
|
412
|
+
size: file.size,
|
|
413
|
+
}));
|
|
414
|
+
});
|
|
220
415
|
const status = () => {
|
|
221
416
|
if (apple && macOS === null) return { state: "downloading", percent: 0 };
|
|
222
|
-
if (ready())
|
|
417
|
+
if (ready())
|
|
418
|
+
return {
|
|
419
|
+
state: "ready",
|
|
420
|
+
model: onMac() ? "Apple's transcriber" : speechEntry.name,
|
|
421
|
+
};
|
|
223
422
|
if (onMac() && !macOS.installed) {
|
|
224
423
|
void probe();
|
|
225
|
-
if (macOS.failed || failure)
|
|
226
|
-
|
|
424
|
+
if (macOS.failed || failure)
|
|
425
|
+
return { state: "failed", error: macOS.failed ?? failure };
|
|
426
|
+
return {
|
|
427
|
+
state: macOS.downloading || downloading ? "downloading" : "absent",
|
|
428
|
+
percent: macOS.percent ?? 0,
|
|
429
|
+
};
|
|
227
430
|
}
|
|
228
431
|
const all = files();
|
|
229
432
|
const total = all.reduce((sum, f) => sum + f.size, 0);
|
|
230
|
-
const have = all.reduce((sum, f) => {
|
|
433
|
+
const have = all.reduce((sum, f) => {
|
|
434
|
+
for (const candidate of [f.file, `${f.file}.partial`]) {
|
|
435
|
+
try {
|
|
436
|
+
return sum + Math.min(f.size, fs.statSync(candidate).size);
|
|
437
|
+
} catch {}
|
|
438
|
+
}
|
|
439
|
+
return sum;
|
|
440
|
+
}, 0);
|
|
231
441
|
const percent = Math.min(99, Math.floor((have / total) * 100));
|
|
232
442
|
if (downloading) return { state: "downloading", percent, bytes: total };
|
|
233
|
-
return failure
|
|
443
|
+
return failure
|
|
444
|
+
? { state: "failed", error: failure, percent, bytes: total }
|
|
445
|
+
: { state: "absent", percent, bytes: total };
|
|
234
446
|
};
|
|
235
447
|
/** Starts the download if the models are missing; answers where it stands. */
|
|
236
448
|
const ensure = () => {
|
|
237
449
|
if (!ready() && !downloading && macOS !== null) {
|
|
238
450
|
failure = null;
|
|
239
|
-
log(
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
451
|
+
log(
|
|
452
|
+
onMac()
|
|
453
|
+
? "speech: getting Apple's transcriber ready"
|
|
454
|
+
: "speech: downloading the speech models"
|
|
455
|
+
);
|
|
456
|
+
const mac =
|
|
457
|
+
onMac() && !macOS.installed
|
|
458
|
+
? apple.ask({ op: "install" }).then((answer) => {
|
|
459
|
+
if (!answer.ok) throw new Error(answer.error);
|
|
460
|
+
return probe();
|
|
461
|
+
})
|
|
462
|
+
: null;
|
|
463
|
+
downloading = Promise.all([
|
|
464
|
+
ensureModels({
|
|
465
|
+
root,
|
|
466
|
+
ids: ids(),
|
|
467
|
+
list,
|
|
468
|
+
...(fetchImpl ? { fetchImpl } : {}),
|
|
469
|
+
log,
|
|
470
|
+
}),
|
|
471
|
+
mac,
|
|
472
|
+
])
|
|
473
|
+
.then(
|
|
474
|
+
() => log("speech: ready"),
|
|
475
|
+
(error) => {
|
|
476
|
+
failure = error.message;
|
|
477
|
+
log(`speech: download failed: ${error.message}`);
|
|
478
|
+
}
|
|
479
|
+
)
|
|
480
|
+
.finally(() => {
|
|
481
|
+
downloading = null;
|
|
482
|
+
});
|
|
244
483
|
}
|
|
245
484
|
return status();
|
|
246
485
|
};
|
|
@@ -248,81 +487,195 @@ export function createSpeech({ inference, root, list = MODELS, vocabulary = () =
|
|
|
248
487
|
// ── Hearing ─────────────────────────────────────────────────────────────────
|
|
249
488
|
const vadFile = fileOf(vadEntry, /\.onnx$/);
|
|
250
489
|
/** The VAD's speech probability for each 32 ms frame. */
|
|
251
|
-
const probabilities = (samples, lane) =>
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
490
|
+
const probabilities = (samples, lane) =>
|
|
491
|
+
inference.onnx(
|
|
492
|
+
"vad",
|
|
493
|
+
async ({ ort, graph }) => {
|
|
494
|
+
const vad = await graph(vadFile);
|
|
495
|
+
const rate = new ort.Tensor(
|
|
496
|
+
"int64",
|
|
497
|
+
BigInt64Array.from([BigInt(RATE)]),
|
|
498
|
+
[]
|
|
499
|
+
);
|
|
500
|
+
let state = new ort.Tensor(
|
|
501
|
+
"float32",
|
|
502
|
+
new Float32Array(2 * 128),
|
|
503
|
+
[2, 1, 128]
|
|
504
|
+
);
|
|
505
|
+
const input = new Float32Array(CONTEXT + FRAME);
|
|
506
|
+
const probs = new Float32Array(Math.floor(samples.length / FRAME));
|
|
507
|
+
for (let i = 0; i < probs.length; i += 1) {
|
|
508
|
+
// Each frame carries the last 64 samples of the one before it, as the model was trained.
|
|
509
|
+
input.copyWithin(0, FRAME, FRAME + CONTEXT);
|
|
510
|
+
input.set(samples.subarray(i * FRAME, (i + 1) * FRAME), CONTEXT);
|
|
511
|
+
const out = await vad.run({
|
|
512
|
+
input: new ort.Tensor("float32", input.slice(), [
|
|
513
|
+
1,
|
|
514
|
+
CONTEXT + FRAME,
|
|
515
|
+
]),
|
|
516
|
+
state,
|
|
517
|
+
sr: rate,
|
|
518
|
+
});
|
|
519
|
+
probs[i] = out.output.data[0];
|
|
520
|
+
state = out.stateN;
|
|
521
|
+
}
|
|
522
|
+
return probs;
|
|
523
|
+
},
|
|
524
|
+
{ lane }
|
|
525
|
+
);
|
|
267
526
|
|
|
268
527
|
/** Where in the samples someone spoke: [start, end) pairs, none for noise or silence. */
|
|
269
|
-
const voiced = async (samples, { lane = "interactive" } = {}) =>
|
|
528
|
+
const voiced = async (samples, { lane = "interactive" } = {}) =>
|
|
529
|
+
speechSpans(await probabilities(samples, lane), { total: samples.length });
|
|
270
530
|
|
|
271
531
|
/** Moonshine's words for one piece of speech. */
|
|
272
|
-
const moonshine = (samples, { lane = "interactive" } = {}) =>
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
532
|
+
const moonshine = (samples, { lane = "interactive" } = {}) =>
|
|
533
|
+
inference.onnx(
|
|
534
|
+
"speech",
|
|
535
|
+
async ({ ort, graph }) => {
|
|
536
|
+
load();
|
|
537
|
+
const [encoder, decoder] = await Promise.all([
|
|
538
|
+
graph(fileOf(speechEntry, /encoder[^/]*\.onnx$/)),
|
|
539
|
+
graph(fileOf(speechEntry, /decoder[^/]*\.onnx$/)),
|
|
540
|
+
]);
|
|
541
|
+
const { last_hidden_state: states } = await encoder.run({
|
|
542
|
+
input_values: new ort.Tensor("float32", samples, [1, samples.length]),
|
|
543
|
+
});
|
|
544
|
+
const layers = config.decoder_num_hidden_layers,
|
|
545
|
+
heads = config.decoder_num_key_value_heads,
|
|
546
|
+
width = config.hidden_size / config.decoder_num_attention_heads;
|
|
547
|
+
const empty = new ort.Tensor("float32", new Float32Array(0), [
|
|
548
|
+
1,
|
|
549
|
+
heads,
|
|
550
|
+
0,
|
|
551
|
+
width,
|
|
552
|
+
]);
|
|
553
|
+
const past = {};
|
|
554
|
+
for (let i = 0; i < layers; i += 1)
|
|
555
|
+
for (const part of ["decoder", "encoder"])
|
|
556
|
+
for (const kv of ["key", "value"])
|
|
557
|
+
past[`past_key_values.${i}.${part}.${kv}`] = empty;
|
|
558
|
+
const limit =
|
|
559
|
+
Math.ceil((samples.length / RATE) * TOKENS_PER_SECOND) + 4;
|
|
560
|
+
const ids = [];
|
|
561
|
+
let token = config.decoder_start_token_id;
|
|
562
|
+
for (let step = 0; step < limit; step += 1) {
|
|
563
|
+
const out = await decoder.run({
|
|
564
|
+
input_ids: new ort.Tensor(
|
|
565
|
+
"int64",
|
|
566
|
+
BigInt64Array.from([BigInt(token)]),
|
|
567
|
+
[1, 1]
|
|
568
|
+
),
|
|
569
|
+
encoder_hidden_states: states,
|
|
570
|
+
use_cache_branch: new ort.Tensor(
|
|
571
|
+
"bool",
|
|
572
|
+
Uint8Array.from([step > 0 ? 1 : 0]),
|
|
573
|
+
[1]
|
|
574
|
+
),
|
|
575
|
+
...past,
|
|
576
|
+
});
|
|
577
|
+
const logits = out.logits.data;
|
|
578
|
+
let best = 0;
|
|
579
|
+
for (let v = 1; v < logits.length; v += 1)
|
|
580
|
+
if (logits[v] > logits[best]) best = v;
|
|
581
|
+
// The encoder's keys and values come from the first step and stay; the decoder's grow by one each step.
|
|
582
|
+
for (let i = 0; i < layers; i += 1)
|
|
583
|
+
for (const part of step === 0
|
|
584
|
+
? ["decoder", "encoder"]
|
|
585
|
+
: ["decoder"])
|
|
586
|
+
for (const kv of ["key", "value"])
|
|
587
|
+
past[`past_key_values.${i}.${part}.${kv}`] =
|
|
588
|
+
out[`present.${i}.${part}.${kv}`];
|
|
589
|
+
if (best === config.eos_token_id) break;
|
|
590
|
+
ids.push(best);
|
|
591
|
+
token = best;
|
|
592
|
+
}
|
|
593
|
+
return tokens.decode(ids).trim();
|
|
594
|
+
},
|
|
595
|
+
{ lane }
|
|
596
|
+
);
|
|
297
597
|
|
|
298
598
|
/**
|
|
299
599
|
* Parakeet's words for one piece of speech: its feature extractor and encoder, then a greedy token-and-duration decode
|
|
300
600
|
* over the encoder's frames (as NeMo and onnx-asr do): each step names a token (or blank) and how many frames to move.
|
|
301
601
|
*/
|
|
302
|
-
const parakeet = (samples, { lane = "interactive" } = {}) =>
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
602
|
+
const parakeet = (samples, { lane = "interactive" } = {}) =>
|
|
603
|
+
inference.onnx(
|
|
604
|
+
"speech",
|
|
605
|
+
async ({ ort, graph }) => {
|
|
606
|
+
load();
|
|
607
|
+
const [features, encoder, joint] = await Promise.all(
|
|
608
|
+
[
|
|
609
|
+
/nemo\d+\.onnx$/,
|
|
610
|
+
/encoder[^/]*\.onnx$/,
|
|
611
|
+
/decoder_joint[^/]*\.onnx$/,
|
|
612
|
+
].map((pattern) => graph(fileOf(speechEntry, pattern)))
|
|
613
|
+
);
|
|
614
|
+
const count = new ort.Tensor(
|
|
615
|
+
"int64",
|
|
616
|
+
BigInt64Array.from([BigInt(samples.length)]),
|
|
617
|
+
[1]
|
|
618
|
+
);
|
|
619
|
+
const heard = await features.run({
|
|
620
|
+
waveforms: new ort.Tensor("float32", samples, [1, samples.length]),
|
|
621
|
+
waveforms_lens: count,
|
|
622
|
+
});
|
|
623
|
+
const { outputs, encoded_lengths } = await encoder.run({
|
|
624
|
+
audio_signal: heard.features,
|
|
625
|
+
length: heard.features_lens,
|
|
626
|
+
});
|
|
627
|
+
const [, width, frames] = outputs.dims,
|
|
628
|
+
length = Number(encoded_lengths.data[0]);
|
|
629
|
+
const frame = (t) => {
|
|
630
|
+
const v = new Float32Array(width);
|
|
631
|
+
for (let d = 0; d < width; d += 1)
|
|
632
|
+
v[d] = outputs.data[d * frames + t];
|
|
633
|
+
return v;
|
|
634
|
+
};
|
|
635
|
+
const { pieces, blank } = tokens;
|
|
636
|
+
let states = [1, 2].map(
|
|
637
|
+
() =>
|
|
638
|
+
new ort.Tensor("float32", new Float32Array(2 * 640), [2, 1, 640])
|
|
639
|
+
);
|
|
640
|
+
const ids = [];
|
|
641
|
+
for (let t = 0, emitted = 0; t < length;) {
|
|
642
|
+
const out = await joint.run({
|
|
643
|
+
encoder_outputs: new ort.Tensor("float32", frame(t), [1, width, 1]),
|
|
644
|
+
targets: new ort.Tensor(
|
|
645
|
+
"int32",
|
|
646
|
+
Int32Array.from([ids.at(-1) ?? blank]),
|
|
647
|
+
[1, 1]
|
|
648
|
+
),
|
|
649
|
+
target_length: new ort.Tensor("int32", Int32Array.from([1]), [1]),
|
|
650
|
+
input_states_1: states[0],
|
|
651
|
+
input_states_2: states[1],
|
|
652
|
+
});
|
|
653
|
+
const scores = out.outputs.data;
|
|
654
|
+
let token = 0,
|
|
655
|
+
step = 0;
|
|
656
|
+
for (let v = 1; v < pieces.length; v += 1)
|
|
657
|
+
if (scores[v] > scores[token]) token = v;
|
|
658
|
+
for (let k = 1; k < scores.length - pieces.length; k += 1)
|
|
659
|
+
if (scores[pieces.length + k] > scores[pieces.length + step])
|
|
660
|
+
step = k;
|
|
661
|
+
if (token !== blank) {
|
|
662
|
+
states = [out.output_states_1, out.output_states_2];
|
|
663
|
+
ids.push(token);
|
|
664
|
+
emitted += 1;
|
|
665
|
+
}
|
|
666
|
+
// At most ten tokens on one frame, as the model's decoding allows.
|
|
667
|
+
if (step > 0) {
|
|
668
|
+
t += step;
|
|
669
|
+
emitted = 0;
|
|
670
|
+
} else if (token === blank || emitted === 10) {
|
|
671
|
+
t += 1;
|
|
672
|
+
emitted = 0;
|
|
673
|
+
}
|
|
674
|
+
}
|
|
675
|
+
return tokens.decode(ids).trim();
|
|
676
|
+
},
|
|
677
|
+
{ lane }
|
|
678
|
+
);
|
|
326
679
|
const recognise = engine === "parakeet" ? parakeet : moonshine;
|
|
327
680
|
|
|
328
681
|
/** Speech stretches joined into pieces Moonshine can take, cut between stretches. */
|
|
@@ -331,7 +684,9 @@ export function createSpeech({ inference, root, list = MODELS, vocabulary = () =
|
|
|
331
684
|
for (const [start, end] of spans) {
|
|
332
685
|
const last = out.at(-1);
|
|
333
686
|
if (last && end - last[0] <= MAX_PIECE_SECONDS * RATE) last[1] = end;
|
|
334
|
-
else
|
|
687
|
+
else
|
|
688
|
+
for (let s = start; s < end; s += MAX_PIECE_SECONDS * RATE)
|
|
689
|
+
out.push([s, Math.min(end, s + MAX_PIECE_SECONDS * RATE)]);
|
|
335
690
|
}
|
|
336
691
|
return out.map(([start, end]) => samples.subarray(start, end));
|
|
337
692
|
};
|
|
@@ -340,32 +695,82 @@ export function createSpeech({ inference, root, list = MODELS, vocabulary = () =
|
|
|
340
695
|
const transcribe = async (input, { lane = "interactive" } = {}) => {
|
|
341
696
|
const started = performance.now();
|
|
342
697
|
if (!ready()) throw new Error("the speech models are not downloaded yet");
|
|
343
|
-
const audio =
|
|
344
|
-
|
|
698
|
+
const audio =
|
|
699
|
+
input instanceof Uint8Array || input instanceof ArrayBuffer
|
|
700
|
+
? decodeWav(input)
|
|
701
|
+
: input;
|
|
702
|
+
const samples =
|
|
703
|
+
audio.sampleRate === RATE
|
|
704
|
+
? Float32Array.from(audio.samples)
|
|
705
|
+
: resample(audio.samples, audio.sampleRate);
|
|
345
706
|
const spans = await voiced(samples, { lane });
|
|
346
707
|
const entries = vocabularyEntries(await vocabulary());
|
|
347
708
|
if (onMac()) {
|
|
348
709
|
// Apple's model hears the whole recording, steered toward the names the relay knows; then names are put right as
|
|
349
710
|
// for Parakeet, with the Mac's own dictionary as the words a name never replaces.
|
|
350
|
-
if (!spans.length)
|
|
351
|
-
|
|
711
|
+
if (!spans.length)
|
|
712
|
+
return {
|
|
713
|
+
text: "",
|
|
714
|
+
heard: false,
|
|
715
|
+
ms: Math.round(performance.now() - started),
|
|
716
|
+
model: "Apple's transcriber",
|
|
717
|
+
raw: "",
|
|
718
|
+
};
|
|
719
|
+
const file = path.join(
|
|
720
|
+
fs.mkdtempSync(path.join(os.tmpdir(), "harness-talk-")),
|
|
721
|
+
"talk.wav"
|
|
722
|
+
);
|
|
352
723
|
try {
|
|
353
724
|
fs.writeFileSync(file, encodeWav(samples), { mode: 0o600 });
|
|
354
|
-
const answer = await apple.ask({
|
|
725
|
+
const answer = await apple.ask({
|
|
726
|
+
op: "transcribe",
|
|
727
|
+
file,
|
|
728
|
+
names: entries.map((entry) => entry.display).slice(0, APPLE_NAMES),
|
|
729
|
+
});
|
|
355
730
|
if (!answer.ok) throw new Error(answer.error);
|
|
356
|
-
const text = answer.text
|
|
357
|
-
|
|
358
|
-
|
|
731
|
+
const text = answer.text
|
|
732
|
+
? snapNames(answer.text, entries, macWords()).text
|
|
733
|
+
: "";
|
|
734
|
+
return {
|
|
735
|
+
text,
|
|
736
|
+
heard: Boolean(text),
|
|
737
|
+
ms: Math.round(performance.now() - started),
|
|
738
|
+
model: "Apple's transcriber",
|
|
739
|
+
raw: answer.text,
|
|
740
|
+
};
|
|
741
|
+
} finally {
|
|
742
|
+
fs.rmSync(path.dirname(file), { recursive: true, force: true });
|
|
743
|
+
}
|
|
359
744
|
}
|
|
360
745
|
const said = [];
|
|
361
|
-
for (const piece of pieces(samples, spans)) {
|
|
746
|
+
for (const piece of pieces(samples, spans)) {
|
|
747
|
+
const words = await recognise(piece, { lane });
|
|
748
|
+
if (words) said.push(words);
|
|
749
|
+
}
|
|
362
750
|
const raw = said.join(" ").trim();
|
|
363
751
|
load();
|
|
364
752
|
const text = raw ? snapNames(raw, entries, tokens.common).text : "";
|
|
365
|
-
return {
|
|
753
|
+
return {
|
|
754
|
+
text,
|
|
755
|
+
heard: Boolean(text),
|
|
756
|
+
ms: Math.round(performance.now() - started),
|
|
757
|
+
model: speechEntry.name,
|
|
758
|
+
raw,
|
|
759
|
+
};
|
|
366
760
|
};
|
|
367
761
|
|
|
368
|
-
return {
|
|
762
|
+
return {
|
|
763
|
+
ready,
|
|
764
|
+
status,
|
|
765
|
+
ensure,
|
|
766
|
+
transcribe,
|
|
767
|
+
voiced,
|
|
768
|
+
recognise,
|
|
769
|
+
common: () => {
|
|
770
|
+
load();
|
|
771
|
+
return tokens.common;
|
|
772
|
+
},
|
|
773
|
+
};
|
|
369
774
|
}
|
|
370
775
|
|
|
371
776
|
// ── Names ─────────────────────────────────────────────────────────────────────
|
|
@@ -373,60 +778,127 @@ export function createSpeech({ inference, root, list = MODELS, vocabulary = () =
|
|
|
373
778
|
// name (as written or as people say it), a file name where a dot or "dot" was heard, or one word no dictionary has.
|
|
374
779
|
// Words that are the name's own words in plain English ("pick folder", "payout for") are left as said.
|
|
375
780
|
|
|
376
|
-
const ROSTER = JSON.parse(
|
|
781
|
+
const ROSTER = JSON.parse(
|
|
782
|
+
fs.readFileSync(new URL("./members.json", import.meta.url), "utf8")
|
|
783
|
+
).members;
|
|
377
784
|
// How people say the names that are not spelled as said.
|
|
378
|
-
const SAID = {
|
|
785
|
+
const SAID = {
|
|
786
|
+
bankr: ["banker"],
|
|
787
|
+
pi: ["pie", "pye"],
|
|
788
|
+
codex: ["code x", "codecs"],
|
|
789
|
+
"claude-code": ["cloud code", "clawed code"],
|
|
790
|
+
"grok-bot": ["grock bot", "grog bot"],
|
|
791
|
+
};
|
|
379
792
|
const SNAP = 0.8;
|
|
380
793
|
|
|
381
|
-
const squash = (text) =>
|
|
794
|
+
const squash = (text) =>
|
|
795
|
+
String(text)
|
|
796
|
+
.toLowerCase()
|
|
797
|
+
.replace(/[^a-z0-9]/g, "");
|
|
382
798
|
function distance(a, b) {
|
|
383
799
|
let row = Array.from({ length: b.length + 1 }, (_, j) => j);
|
|
384
800
|
for (let i = 1; i <= a.length; i += 1) {
|
|
385
801
|
const next = [i];
|
|
386
|
-
for (let j = 1; j <= b.length; j += 1)
|
|
802
|
+
for (let j = 1; j <= b.length; j += 1)
|
|
803
|
+
next[j] = Math.min(
|
|
804
|
+
row[j] + 1,
|
|
805
|
+
next[j - 1] + 1,
|
|
806
|
+
row[j - 1] + (a[i - 1] === b[j - 1] ? 0 : 1)
|
|
807
|
+
);
|
|
387
808
|
row = next;
|
|
388
809
|
}
|
|
389
810
|
return row[b.length];
|
|
390
811
|
}
|
|
391
812
|
/** Metaphone, enough of it to hear "cloud" as "claude" and "demon" as "daemon". */
|
|
392
813
|
export function sound(word) {
|
|
393
|
-
let w = squash(word)
|
|
814
|
+
let w = squash(word)
|
|
815
|
+
.replace(/[0-9]/g, "")
|
|
816
|
+
.replace(/^(kn|gn|pn|ae|wr)/, (m) => m[1])
|
|
817
|
+
.replace(/^x/, "s")
|
|
818
|
+
.replace(/^wh/, "w")
|
|
819
|
+
.replace(/mb$/, "m");
|
|
394
820
|
const vowel = (c) => "aeiou".includes(c ?? "");
|
|
395
821
|
let out = "";
|
|
396
822
|
for (let i = 0; i < w.length; i += 1) {
|
|
397
|
-
const c = w[i],
|
|
823
|
+
const c = w[i],
|
|
824
|
+
next = w[i + 1] ?? "",
|
|
825
|
+
prev = w[i - 1] ?? "";
|
|
398
826
|
if (c === prev && c !== "c") continue;
|
|
399
|
-
if (vowel(c)) {
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
827
|
+
if (vowel(c)) {
|
|
828
|
+
if (i === 0) out += "A";
|
|
829
|
+
continue;
|
|
830
|
+
}
|
|
831
|
+
if (c === "c")
|
|
832
|
+
out +=
|
|
833
|
+
next === "h" || (next === "i" && w[i + 2] === "a")
|
|
834
|
+
? "X"
|
|
835
|
+
: "iey".includes(next)
|
|
836
|
+
? "S"
|
|
837
|
+
: "K";
|
|
838
|
+
else if (c === "d")
|
|
839
|
+
out += next === "g" && "eiy".includes(w[i + 2] ?? "-") ? "J" : "T";
|
|
840
|
+
else if (c === "g") {
|
|
841
|
+
if (next === "h" && !vowel(w[i + 2])) i += 1;
|
|
842
|
+
else out += "iey".includes(next) ? "J" : "K";
|
|
843
|
+
} else if (c === "h") {
|
|
844
|
+
if (vowel(next) && !"csptg".includes(prev)) out += "H";
|
|
845
|
+
} else if (c === "k") {
|
|
846
|
+
if (prev !== "c") out += "K";
|
|
847
|
+
} else if (c === "p") out += next === "h" ? "F" : "P";
|
|
406
848
|
else if (c === "q") out += "K";
|
|
407
|
-
else if (c === "s")
|
|
408
|
-
|
|
849
|
+
else if (c === "s")
|
|
850
|
+
out +=
|
|
851
|
+
next === "h" || (next === "i" && "oa".includes(w[i + 2] ?? "-"))
|
|
852
|
+
? "X"
|
|
853
|
+
: "S";
|
|
854
|
+
else if (c === "t")
|
|
855
|
+
out +=
|
|
856
|
+
next === "h"
|
|
857
|
+
? "0"
|
|
858
|
+
: next === "i" && "oa".includes(w[i + 2] ?? "-")
|
|
859
|
+
? "X"
|
|
860
|
+
: "T";
|
|
409
861
|
else if (c === "v") out += "F";
|
|
410
|
-
else if (c === "w" || c === "y") {
|
|
411
|
-
|
|
862
|
+
else if (c === "w" || c === "y") {
|
|
863
|
+
if (vowel(next)) out += c.toUpperCase();
|
|
864
|
+
} else if (c === "x") out += "KS";
|
|
412
865
|
else if (c === "z") out += "S";
|
|
413
866
|
else out += c.toUpperCase();
|
|
414
867
|
if ((c === "c" || c === "s" || c === "p") && next === "h") i += 1;
|
|
415
868
|
}
|
|
416
869
|
return out;
|
|
417
870
|
}
|
|
418
|
-
const similar = (a, b) =>
|
|
871
|
+
const similar = (a, b) =>
|
|
872
|
+
(2 -
|
|
873
|
+
distance(a, b) / Math.max(a.length, b.length) -
|
|
874
|
+
distance(sound(a), sound(b)) /
|
|
875
|
+
Math.max(1, sound(a).length, sound(b).length)) /
|
|
876
|
+
2;
|
|
419
877
|
|
|
420
878
|
/** What the names are matched by: the agents' names (as written and said), then the person's words, files and folders. */
|
|
421
879
|
export function vocabularyEntries(names = []) {
|
|
422
|
-
const entries = ROSTER.map((member) => ({
|
|
880
|
+
const entries = ROSTER.map((member) => ({
|
|
881
|
+
display: member.name,
|
|
882
|
+
kind: "agent",
|
|
883
|
+
keys: [...new Set([member.name, ...(SAID[member.id] ?? [])].map(squash))],
|
|
884
|
+
}));
|
|
423
885
|
const seen = new Set(entries.map((entry) => squash(entry.display)));
|
|
424
886
|
for (const name of names) {
|
|
425
887
|
const display = String(name ?? "").trim();
|
|
426
888
|
const key = squash(display);
|
|
427
|
-
if (
|
|
889
|
+
if (
|
|
890
|
+
key.length < 4 ||
|
|
891
|
+
display.length > 60 ||
|
|
892
|
+
/\s/.test(display) ||
|
|
893
|
+
seen.has(key)
|
|
894
|
+
)
|
|
895
|
+
continue;
|
|
428
896
|
seen.add(key);
|
|
429
|
-
entries.push({
|
|
897
|
+
entries.push({
|
|
898
|
+
display,
|
|
899
|
+
kind: /\.[a-z][a-z0-9]{0,4}$/i.test(display) ? "file" : "word",
|
|
900
|
+
keys: [key],
|
|
901
|
+
});
|
|
430
902
|
}
|
|
431
903
|
return entries;
|
|
432
904
|
}
|
|
@@ -438,7 +910,11 @@ export function vocabularyEntries(names = []) {
|
|
|
438
910
|
* recogniser's own one-piece words, which are never taken for a name of the person's.
|
|
439
911
|
*/
|
|
440
912
|
export function snapNames(text, entries, common = new Set()) {
|
|
441
|
-
const tokens = [...String(text).matchAll(/[A-Za-z0-9]+/g)].map((m) => ({
|
|
913
|
+
const tokens = [...String(text).matchAll(/[A-Za-z0-9]+/g)].map((m) => ({
|
|
914
|
+
word: m[0],
|
|
915
|
+
at: m.index,
|
|
916
|
+
end: m.index + m[0].length,
|
|
917
|
+
}));
|
|
442
918
|
const found = [];
|
|
443
919
|
for (let i = 0; i < tokens.length; i += 1) {
|
|
444
920
|
for (let n = 1; n <= 6 && i + n <= tokens.length; n += 1) {
|
|
@@ -446,27 +922,64 @@ export function snapNames(text, entries, common = new Set()) {
|
|
|
446
922
|
const between = String(text).slice(window[0].at, window.at(-1).end);
|
|
447
923
|
// Words run together only across spaces, commas, dots and hyphens.
|
|
448
924
|
if (/[^A-Za-z0-9\s,.\-]/.test(between)) break;
|
|
449
|
-
const dotted =
|
|
450
|
-
|
|
925
|
+
const dotted =
|
|
926
|
+
n > 1 &&
|
|
927
|
+
(/\./.test(between) ||
|
|
928
|
+
window.slice(1, -1).some((t) => t.word.toLowerCase() === "dot"));
|
|
929
|
+
const key = squash(
|
|
930
|
+
window
|
|
931
|
+
.filter((t, k) => !(k > 0 && t.word.toLowerCase() === "dot"))
|
|
932
|
+
.map((t) => t.word)
|
|
933
|
+
.join("")
|
|
934
|
+
);
|
|
451
935
|
// A file's name before its dot, as heard: one too short to judge by sound ("a dot mjs") must be exact.
|
|
452
936
|
const stem = dotted ? squash(between.split(/\.|\bdot\b/i)[0]) : key;
|
|
453
937
|
for (const entry of entries) {
|
|
454
|
-
if (
|
|
938
|
+
if (
|
|
939
|
+
entry.kind === "file"
|
|
940
|
+
? !dotted
|
|
941
|
+
: entry.kind === "word"
|
|
942
|
+
? n > 1 || common.has(key)
|
|
943
|
+
: n > 3
|
|
944
|
+
)
|
|
945
|
+
continue;
|
|
455
946
|
let score = 0;
|
|
456
947
|
for (const candidate of entry.keys) {
|
|
457
|
-
if (candidate === key) {
|
|
458
|
-
|
|
459
|
-
|
|
948
|
+
if (candidate === key) {
|
|
949
|
+
score = 1;
|
|
950
|
+
break;
|
|
951
|
+
}
|
|
952
|
+
if (
|
|
953
|
+
entry.kind === "file" &&
|
|
954
|
+
(stem.length < 3 ||
|
|
955
|
+
squash(entry.display.replace(/\.[^.]+$/, "")).length < 3)
|
|
956
|
+
)
|
|
957
|
+
continue;
|
|
958
|
+
if (
|
|
959
|
+
candidate.length < 4 ||
|
|
960
|
+
Math.abs(candidate.length - key.length) > candidate.length * 0.4 ||
|
|
961
|
+
sound(candidate)[0] !== sound(key)[0]
|
|
962
|
+
)
|
|
963
|
+
continue;
|
|
460
964
|
score = Math.max(score, similar(candidate, key));
|
|
461
965
|
}
|
|
462
|
-
if (score >= SNAP)
|
|
966
|
+
if (score >= SNAP)
|
|
967
|
+
found.push({
|
|
968
|
+
from: window[0].at,
|
|
969
|
+
to: window.at(-1).end,
|
|
970
|
+
display: entry.display,
|
|
971
|
+
score,
|
|
972
|
+
span: n,
|
|
973
|
+
});
|
|
463
974
|
}
|
|
464
975
|
}
|
|
465
976
|
}
|
|
466
977
|
// The closest names first, longer spans before shorter ones; none overlapping.
|
|
467
978
|
found.sort((a, b) => b.score - a.score || b.span - a.span);
|
|
468
979
|
const taken = [];
|
|
469
|
-
for (const match of found)
|
|
980
|
+
for (const match of found)
|
|
981
|
+
if (!taken.some((t) => match.from < t.to && t.from < match.to))
|
|
982
|
+
taken.push(match);
|
|
470
983
|
taken.sort((a, b) => b.from - a.from);
|
|
471
984
|
let out = String(text);
|
|
472
985
|
const changes = [];
|