tryharness 0.7.45 → 0.7.54
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +239 -14
- package/assets/dsh-plugin.mjs +72 -0
- package/assets/opencode-plugin.js +4 -3
- package/cli.mjs +497 -196
- package/lib/actorRoster.mjs +28 -5
- package/lib/adapters.mjs +165 -222
- package/lib/agentRuntimeConfig.mjs +36 -0
- package/lib/agentRuntimes.mjs +570 -0
- package/lib/agentSignIn.mjs +226 -0
- package/lib/atlas.mjs +7 -2
- package/lib/audit.mjs +643 -0
- package/lib/backupSetup.mjs +141 -0
- package/lib/boardAccess.mjs +8 -1
- package/lib/boardBackup.mjs +283 -0
- package/lib/boardCommands.mjs +5 -10
- package/lib/boardFileCommands.mjs +2 -1
- package/lib/boardMirror.mjs +523 -0
- package/lib/boardProtocol.json +263 -6
- package/lib/boardTools.mjs +2 -6
- package/lib/boardWaits.mjs +102 -0
- package/lib/book.mjs +659 -0
- package/lib/bridge.mjs +26 -4
- package/lib/browserWake.mjs +60 -0
- package/lib/cards.mjs +159 -16
- package/lib/cloudBoot.mjs +124 -0
- package/lib/commands.mjs +25 -5
- package/lib/connectionTools.d.ts +13 -0
- package/lib/connectionTools.js +45 -0
- package/lib/connectorAuth.mjs +309 -0
- package/lib/connectorCatalog.json +41 -0
- package/lib/connectorCommands.mjs +121 -0
- package/lib/connectorGoogle.mjs +513 -0
- package/lib/connectors.mjs +805 -0
- package/lib/context.mjs +18 -4
- package/lib/contractGate.mjs +377 -0
- package/lib/coordinator.mjs +152 -39
- package/lib/coreModels.mjs +78 -0
- package/lib/daemon.mjs +973 -54
- package/lib/desk.mjs +239 -0
- package/lib/desktop.mjs +113 -8
- package/lib/digest.mjs +32 -0
- package/lib/directLink.mjs +283 -0
- package/lib/environments.mjs +825 -0
- package/lib/folders.mjs +4 -4
- package/lib/gitHost.mjs +174 -0
- package/lib/goalCommands.mjs +58 -14
- package/lib/hookDialects.mjs +298 -0
- package/lib/inference.mjs +263 -152
- package/lib/landed.mjs +887 -0
- package/lib/lessons.mjs +6 -0
- package/lib/lineup.mjs +324 -0
- package/lib/live.mjs +284 -13
- package/lib/localmodels.mjs +10 -0
- package/lib/machineIdentity.mjs +3 -0
- package/lib/machineRoutes.mjs +40 -2
- package/lib/machineRuns.mjs +30 -3
- package/lib/machineSeal.mjs +69 -10
- package/lib/mcp-stdio.mjs +189 -120
- package/lib/mcpRelay.mjs +119 -0
- package/lib/members.json +26 -0
- package/lib/members.mjs +33 -0
- package/lib/modelMemory.mjs +100 -0
- package/lib/models.mjs +48 -15
- package/lib/museSession.mjs +122 -0
- package/lib/nativeHost.mjs +203 -0
- package/lib/neovim.mjs +126 -0
- package/lib/network.mjs +7 -2
- package/lib/nudge.mjs +57 -18
- package/lib/onboard.mjs +430 -1
- package/lib/oneclawLocal.mjs +150 -0
- package/lib/ownMachines.mjs +828 -39
- package/lib/pairing.mjs +182 -0
- package/lib/paths.mjs +8 -0
- package/lib/personModel.mjs +89 -0
- package/lib/phones.mjs +119 -3
- package/lib/placeCommands.mjs +27 -0
- package/lib/places.mjs +337 -9
- package/lib/plans.mjs +151 -27
- package/lib/profile.mjs +152 -0
- package/lib/projects.mjs +624 -137
- package/lib/readiness.mjs +400 -0
- package/lib/record.mjs +115 -0
- package/lib/repoLinks.mjs +354 -0
- package/lib/roomChannels.mjs +641 -0
- package/lib/runBrief.mjs +32 -2
- package/lib/runRequests.mjs +157 -0
- package/lib/runWorker.mjs +37 -9
- package/lib/runner.mjs +74 -11
- package/lib/secretPatterns.json +55 -0
- package/lib/secrets.mjs +10 -27
- package/lib/seen.mjs +102 -0
- package/lib/service.mjs +11 -6
- package/lib/sessionControl.mjs +386 -96
- package/lib/sessionIndex.mjs +5 -3
- package/lib/specialties.json +74 -0
- package/lib/specialties.mjs +477 -0
- package/lib/specialtyKeys.d.ts +3 -0
- package/lib/specialtyKeys.js +58 -0
- package/lib/specialtyNotice.mjs +147 -0
- package/lib/specialtySnapshot.mjs +138 -0
- package/lib/store.mjs +829 -29
- package/lib/subagents.mjs +8 -4
- package/lib/swarm.mjs +483 -37
- package/lib/swarmMemory.mjs +947 -0
- package/lib/t3.mjs +56 -17
- package/lib/taskCommands.mjs +91 -13
- package/lib/taskRuns.mjs +68 -11
- package/lib/team.json +253 -144
- package/lib/team.mjs +538 -147
- package/lib/threadLine.mjs +116 -0
- package/lib/threadRuns.mjs +78 -12
- package/lib/threadView.mjs +66 -16
- package/lib/tokenizer.mjs +405 -0
- package/lib/ui.mjs +16 -2
- package/lib/usageTotals.mjs +148 -0
- package/lib/vault.mjs +1144 -0
- package/lib/vaultCloud.mjs +117 -0
- package/lib/vaultCommands.mjs +201 -0
- package/lib/vaultTools.d.ts +12 -0
- package/lib/vaultTools.js +42 -0
- package/lib/windowAsks.mjs +42 -0
- package/lib/workFocus.mjs +59 -0
- package/lib/workLinks.mjs +90 -0
- package/lib/worktrees.mjs +51 -13
- package/package.json +10 -5
package/lib/inference.mjs
CHANGED
|
@@ -13,6 +13,12 @@ import {
|
|
|
13
13
|
llamaServer,
|
|
14
14
|
} from "./localmodels.mjs";
|
|
15
15
|
import { ensureDir } from "./files.mjs";
|
|
16
|
+
import { loadCompactTokenizer } from "./tokenizer.mjs";
|
|
17
|
+
import {
|
|
18
|
+
createModelMemory,
|
|
19
|
+
memoryCeilingMb,
|
|
20
|
+
memoryCostMb,
|
|
21
|
+
} from "./modelMemory.mjs";
|
|
16
22
|
|
|
17
23
|
// The relay's local inference host. GGUF models (the search model) run in one llama.cpp server in
|
|
18
24
|
// router mode, on the GPU when the machine has one that is faster, the CPU otherwise; the small ONNX classifiers run in
|
|
@@ -52,6 +58,7 @@ export function createPower({
|
|
|
52
58
|
onBattery: false,
|
|
53
59
|
percent: null,
|
|
54
60
|
lowPower: false,
|
|
61
|
+
heldAwake: false,
|
|
55
62
|
hot: false,
|
|
56
63
|
at: 0,
|
|
57
64
|
};
|
|
@@ -60,6 +67,7 @@ export function createPower({
|
|
|
60
67
|
onBattery: false,
|
|
61
68
|
percent: null,
|
|
62
69
|
lowPower: false,
|
|
70
|
+
heldAwake: false,
|
|
63
71
|
hot: false,
|
|
64
72
|
at: Date.now(),
|
|
65
73
|
};
|
|
@@ -69,9 +77,12 @@ export function createPower({
|
|
|
69
77
|
.stdout || "";
|
|
70
78
|
next.onBattery = /Battery Power/.test(batt);
|
|
71
79
|
next.percent = Number(batt.match(/(\d+)%/)?.[1] ?? NaN);
|
|
72
|
-
|
|
73
|
-
run("pmset", ["-g"], { encoding: "utf8", timeout: 3_000 }).stdout || ""
|
|
74
|
-
);
|
|
80
|
+
const settings =
|
|
81
|
+
run("pmset", ["-g"], { encoding: "utf8", timeout: 3_000 }).stdout || "";
|
|
82
|
+
// macOS 26 prints `powermode 1` (0 automatic, 1 low, 2 high); older macOS `lowpowermode 1`.
|
|
83
|
+
next.lowPower = /^\s*(?:low)?powermode\s+1\b/m.test(settings);
|
|
84
|
+
// Lid-Closed Mode's helper keeps the Mac awake for the person's agents (packages/desktop lid.rs reads the same).
|
|
85
|
+
next.heldAwake = /^\s*SleepDisabled\s+1\b/m.test(settings);
|
|
75
86
|
const therm =
|
|
76
87
|
run("pmset", ["-g", "therm"], { encoding: "utf8", timeout: 3_000 })
|
|
77
88
|
.stdout || "";
|
|
@@ -111,6 +122,20 @@ export function createPower({
|
|
|
111
122
|
(now.onBattery && now.percent !== null && now.percent < 20)
|
|
112
123
|
);
|
|
113
124
|
},
|
|
125
|
+
/**
|
|
126
|
+
* Whether the dream waits. It runs whenever the computer is on (owner, October 22: "if the computer is just on
|
|
127
|
+
* obviously"), but not hot, below 20% on battery, or in Low Power Mode, unless Lid-Closed Mode holds the Mac awake:
|
|
128
|
+
* it turns Low Power Mode on with the lid closed, and the dream should run then ("if insomnia is on dream mode
|
|
129
|
+
* should run on low power").
|
|
130
|
+
*/
|
|
131
|
+
dreamWaits() {
|
|
132
|
+
const now = this.state();
|
|
133
|
+
return (
|
|
134
|
+
now.hot ||
|
|
135
|
+
(now.onBattery && now.percent !== null && now.percent < 20) ||
|
|
136
|
+
(now.lowPower && !now.heldAwake)
|
|
137
|
+
);
|
|
138
|
+
},
|
|
114
139
|
/** First index passes wait for the charger as well. */
|
|
115
140
|
onCharger() {
|
|
116
141
|
return !this.state().onBattery;
|
|
@@ -123,6 +148,7 @@ export function createInference({
|
|
|
123
148
|
log = () => {},
|
|
124
149
|
power = createPower(),
|
|
125
150
|
list = MODELS,
|
|
151
|
+
ceilingMb = memoryCeilingMb(),
|
|
126
152
|
}) {
|
|
127
153
|
ensureDir(root);
|
|
128
154
|
const deviceFile = path.join(root, "device.json");
|
|
@@ -131,6 +157,7 @@ export function createInference({
|
|
|
131
157
|
const queues = { llama: [], onnx: [] };
|
|
132
158
|
const busy = { llama: false, onnx: false };
|
|
133
159
|
const sessions = new Map();
|
|
160
|
+
const memory = createModelMemory({ ceilingMb, log });
|
|
134
161
|
|
|
135
162
|
const device = () => {
|
|
136
163
|
try {
|
|
@@ -277,10 +304,15 @@ export function createInference({
|
|
|
277
304
|
starting ||= (async () => {
|
|
278
305
|
if (!modelsPresent(root, ["search"], list))
|
|
279
306
|
throw new Error("the search model is not downloaded yet");
|
|
280
|
-
|
|
281
|
-
server
|
|
307
|
+
const started = await launch(await chooseDevice());
|
|
308
|
+
// Only this server's exit clears it: a restarted server must not be forgotten when the old one goes.
|
|
309
|
+
started.child.on("exit", () => {
|
|
310
|
+
if (server !== started) return;
|
|
282
311
|
server = null;
|
|
312
|
+
llamaServing.clear();
|
|
313
|
+
memory.gone("llama");
|
|
283
314
|
});
|
|
315
|
+
server = started;
|
|
284
316
|
return server;
|
|
285
317
|
})().finally(() => {
|
|
286
318
|
starting = null;
|
|
@@ -288,6 +320,34 @@ export function createInference({
|
|
|
288
320
|
return starting;
|
|
289
321
|
};
|
|
290
322
|
|
|
323
|
+
// The llama models live in one server process, so the memory ceiling counts the process: its charge is the cost of
|
|
324
|
+
// the models it has served, and unloading it stops the process.
|
|
325
|
+
const llamaServing = new Set();
|
|
326
|
+
const stopServer = async () => {
|
|
327
|
+
const child = server?.child;
|
|
328
|
+
if (!child || child.exitCode !== null) return;
|
|
329
|
+
const exited = new Promise((resolve) => child.once("exit", resolve));
|
|
330
|
+
child.kill();
|
|
331
|
+
await exited;
|
|
332
|
+
};
|
|
333
|
+
/** The server, with room made for the model about to run on it. Call while the llama lane holds "llama" in use. */
|
|
334
|
+
const llamaFor = async (id) => {
|
|
335
|
+
const serving = new Set([...llamaServing, id]);
|
|
336
|
+
const mb = [...serving].reduce(
|
|
337
|
+
(sum, model) => sum + memoryCostMb(modelById(model, list)),
|
|
338
|
+
0
|
|
339
|
+
);
|
|
340
|
+
await memory.admit("llama", mb, stopServer);
|
|
341
|
+
try {
|
|
342
|
+
const ready = await ensureServer();
|
|
343
|
+
llamaServing.add(id);
|
|
344
|
+
return ready;
|
|
345
|
+
} catch (error) {
|
|
346
|
+
if (!server) memory.gone("llama");
|
|
347
|
+
throw error;
|
|
348
|
+
}
|
|
349
|
+
};
|
|
350
|
+
|
|
291
351
|
async function post(base, pathname, body) {
|
|
292
352
|
const response = await fetch(`${base}${pathname}`, {
|
|
293
353
|
method: "POST",
|
|
@@ -356,29 +416,62 @@ export function createInference({
|
|
|
356
416
|
pump(backend);
|
|
357
417
|
});
|
|
358
418
|
|
|
359
|
-
// Models on one vocabulary share its tokenizer while any of them is open
|
|
419
|
+
// Models on one vocabulary share its tokenizer while any of them is open; it goes with the last of them, and its
|
|
420
|
+
// memory is part of each model's cost.
|
|
360
421
|
const tokenizers = new Map();
|
|
361
|
-
const
|
|
422
|
+
const tokenizerUsers = new Map();
|
|
423
|
+
const tokenizerOf = (Tokenizer, source, user) => {
|
|
362
424
|
const dir = modelDir(root, source);
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
425
|
+
if (!tokenizerUsers.has(dir)) tokenizerUsers.set(dir, new Set());
|
|
426
|
+
tokenizerUsers.get(dir).add(user);
|
|
427
|
+
if (!tokenizers.has(dir)) {
|
|
428
|
+
const tokenizerFile = source.files.find((file) =>
|
|
366
429
|
file.path.endsWith("tokenizer.json")
|
|
367
|
-
)
|
|
430
|
+
);
|
|
368
431
|
const configPath = source.files.find((file) =>
|
|
369
432
|
file.path.endsWith("tokenizer_config.json")
|
|
370
433
|
)?.path;
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
434
|
+
const tokenizerPath = path.join(dir, tokenizerFile.path);
|
|
435
|
+
// Saved under the file's pinned hash, so a new tokenizer.json never meets old tables.
|
|
436
|
+
const tablesPath = path.join(
|
|
437
|
+
root,
|
|
438
|
+
"tokenizers",
|
|
439
|
+
`${tokenizerFile.sha256}.bin`
|
|
440
|
+
);
|
|
441
|
+
const library = () =>
|
|
442
|
+
new Tokenizer(
|
|
443
|
+
JSON.parse(fs.readFileSync(tokenizerPath, "utf8")),
|
|
444
|
+
configPath
|
|
445
|
+
? JSON.parse(fs.readFileSync(path.join(dir, configPath), "utf8"))
|
|
446
|
+
: {}
|
|
447
|
+
);
|
|
448
|
+
tokenizers.set(
|
|
449
|
+
dir,
|
|
450
|
+
loadCompactTokenizer(tokenizerPath, tablesPath) ?? library()
|
|
376
451
|
);
|
|
377
|
-
tokenizers.set(dir, new WeakRef(tokenizer));
|
|
378
452
|
}
|
|
379
|
-
return
|
|
453
|
+
return tokenizers.get(dir);
|
|
454
|
+
};
|
|
455
|
+
const releaseTokenizer = (dir, user) => {
|
|
456
|
+
const users = tokenizerUsers.get(dir);
|
|
457
|
+
users?.delete(user);
|
|
458
|
+
if (users?.size) return;
|
|
459
|
+
tokenizerUsers.delete(dir);
|
|
460
|
+
tokenizers.delete(dir);
|
|
380
461
|
};
|
|
381
462
|
|
|
463
|
+
// An ONNX model is counted against the memory ceiling from just before it opens until it closes, idle or unloaded
|
|
464
|
+
// to make room. The ONNX lane runs one job at a time, so the models not in that job are always free to unload.
|
|
465
|
+
const closeSession = async (id) => {
|
|
466
|
+
const held = sessions.get(id);
|
|
467
|
+
sessions.delete(id);
|
|
468
|
+
memory.gone(id);
|
|
469
|
+
const ready = await held?.catch(() => null);
|
|
470
|
+
if (!ready) return;
|
|
471
|
+
clearTimeout(ready.timer);
|
|
472
|
+
releaseTokenizer(ready.tokenizerDir, id);
|
|
473
|
+
await ready.onnx.release?.();
|
|
474
|
+
};
|
|
382
475
|
const session = async (id) => {
|
|
383
476
|
const entry = modelById(id, list);
|
|
384
477
|
if (!entry || entry.runtime !== "onnx")
|
|
@@ -391,6 +484,7 @@ export function createInference({
|
|
|
391
484
|
: entry;
|
|
392
485
|
if (!modelsPresent(root, [...new Set([id, source.id])], list))
|
|
393
486
|
throw new Error(`${entry.name} is not downloaded yet`);
|
|
487
|
+
await memory.admit(id, memoryCostMb(entry), () => closeSession(id));
|
|
394
488
|
const [{ Tokenizer }, ort] = await Promise.all([
|
|
395
489
|
import("@huggingface/tokenizers"),
|
|
396
490
|
import("onnxruntime-node"),
|
|
@@ -403,21 +497,22 @@ export function createInference({
|
|
|
403
497
|
});
|
|
404
498
|
return {
|
|
405
499
|
ort,
|
|
406
|
-
tokenizer: tokenizerOf(Tokenizer, source),
|
|
500
|
+
tokenizer: tokenizerOf(Tokenizer, source, id),
|
|
501
|
+
tokenizerDir: modelDir(root, source),
|
|
407
502
|
prepare: entry.tokenizer?.prepare ?? ((text) => text),
|
|
408
503
|
onnx,
|
|
409
504
|
timer: null,
|
|
410
505
|
};
|
|
411
506
|
})();
|
|
412
507
|
sessions.set(id, held);
|
|
413
|
-
held.catch(() =>
|
|
508
|
+
held.catch(() => {
|
|
509
|
+
sessions.delete(id);
|
|
510
|
+
memory.gone(id);
|
|
511
|
+
});
|
|
414
512
|
}
|
|
415
513
|
const ready = await held;
|
|
416
514
|
clearTimeout(ready.timer);
|
|
417
|
-
ready.timer = setTimeout(() =>
|
|
418
|
-
sessions.delete(id);
|
|
419
|
-
ready.onnx.release?.();
|
|
420
|
-
}, ONNX_IDLE_MS);
|
|
515
|
+
ready.timer = setTimeout(() => void closeSession(id), ONNX_IDLE_MS);
|
|
421
516
|
ready.timer.unref?.();
|
|
422
517
|
return ready;
|
|
423
518
|
};
|
|
@@ -425,6 +520,15 @@ export function createInference({
|
|
|
425
520
|
// Models made of several ONNX graphs (OCR's detector and recogniser, speech's encoder and decoder) open each graph
|
|
426
521
|
// on first use and drop all of them together once idle.
|
|
427
522
|
const graphs = new Map();
|
|
523
|
+
const closeGraphs = async (id) => {
|
|
524
|
+
const held = graphs.get(id);
|
|
525
|
+
graphs.delete(id);
|
|
526
|
+
memory.gone(id);
|
|
527
|
+
if (!held) return;
|
|
528
|
+
clearTimeout(held.timer);
|
|
529
|
+
for (const graph of held.open.values())
|
|
530
|
+
await graph.then((onnx) => onnx.release?.()).catch(() => {});
|
|
531
|
+
};
|
|
428
532
|
const graphsOf = async (id) => {
|
|
429
533
|
const entry = modelById(id, list);
|
|
430
534
|
if (!entry || entry.runtime !== "onnx")
|
|
@@ -432,22 +536,18 @@ export function createInference({
|
|
|
432
536
|
if (!modelsPresent(root, [id], list))
|
|
433
537
|
throw new Error(`${entry.name} is not downloaded yet`);
|
|
434
538
|
let held = graphs.get(id);
|
|
435
|
-
if (!held)
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
539
|
+
if (!held) {
|
|
540
|
+
await memory.admit(id, memoryCostMb(entry), () => closeGraphs(id));
|
|
541
|
+
held = {
|
|
542
|
+
ort: await import("onnxruntime-node"),
|
|
543
|
+
open: new Map(),
|
|
544
|
+
timer: null,
|
|
545
|
+
threads: entry.threads ?? 1,
|
|
546
|
+
};
|
|
547
|
+
graphs.set(id, held);
|
|
548
|
+
}
|
|
445
549
|
clearTimeout(held.timer);
|
|
446
|
-
held.timer = setTimeout(() =>
|
|
447
|
-
graphs.delete(id);
|
|
448
|
-
for (const graph of held.open.values())
|
|
449
|
-
graph.then((onnx) => onnx.release?.()).catch(() => {});
|
|
450
|
-
}, ONNX_IDLE_MS);
|
|
550
|
+
held.timer = setTimeout(() => void closeGraphs(id), ONNX_IDLE_MS);
|
|
451
551
|
held.timer.unref?.();
|
|
452
552
|
const graph = (file) => {
|
|
453
553
|
if (!held.open.has(file)) {
|
|
@@ -468,6 +568,16 @@ export function createInference({
|
|
|
468
568
|
return { ort: held.ort, graph, dir: modelDir(root, entry) };
|
|
469
569
|
};
|
|
470
570
|
|
|
571
|
+
/** Runs work with a model marked in use, so the memory ceiling never unloads it mid-job. */
|
|
572
|
+
const inUse = async (id, work) => {
|
|
573
|
+
const done = memory.use(id);
|
|
574
|
+
try {
|
|
575
|
+
return await work();
|
|
576
|
+
} finally {
|
|
577
|
+
done();
|
|
578
|
+
}
|
|
579
|
+
};
|
|
580
|
+
|
|
471
581
|
/** Runs an ONNX model on each encoded input; returns the first output's leading row (its logits). */
|
|
472
582
|
const logits = async (id, pairs, maxLength = 512) => {
|
|
473
583
|
const { ort, tokenizer, prepare, onnx } = await session(id);
|
|
@@ -515,118 +625,128 @@ export function createInference({
|
|
|
515
625
|
power,
|
|
516
626
|
/** Embeds texts with the search model. kind: "query" | "code-query" | "document" (title is a path or "none"). */
|
|
517
627
|
embed: (items, { lane = "handoff" } = {}) =>
|
|
518
|
-
schedule("llama", lane,
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
628
|
+
schedule("llama", lane, () =>
|
|
629
|
+
inUse("llama", async () => {
|
|
630
|
+
const { base } = await llamaFor("search");
|
|
631
|
+
const input = items.map((item) =>
|
|
632
|
+
typeof item === "string"
|
|
633
|
+
? item
|
|
634
|
+
: item.kind === "document"
|
|
635
|
+
? `title: ${item.title || "none"} | text: ${item.text}`
|
|
636
|
+
: `task: ${item.kind === "code-query" ? "code retrieval" : "search result"} | query: ${item.text}`
|
|
637
|
+
);
|
|
638
|
+
// A text too long for the model's window is retried alone, halved until it fits.
|
|
639
|
+
const one = async (text) => {
|
|
640
|
+
for (
|
|
641
|
+
let cut = text;
|
|
642
|
+
;
|
|
643
|
+
cut = cut.slice(0, Math.floor(cut.length / 2))
|
|
644
|
+
) {
|
|
645
|
+
try {
|
|
646
|
+
return (
|
|
647
|
+
await post(base, "/v1/embeddings", {
|
|
648
|
+
model: "search",
|
|
649
|
+
input: [cut],
|
|
650
|
+
})
|
|
651
|
+
).data[0];
|
|
652
|
+
} catch (error) {
|
|
653
|
+
if (cut.length < 200) throw error;
|
|
654
|
+
}
|
|
655
|
+
}
|
|
656
|
+
};
|
|
657
|
+
const out = [];
|
|
658
|
+
for (let i = 0; i < input.length; i += 16) {
|
|
659
|
+
const slice = input.slice(i, i + 16);
|
|
660
|
+
const data = await post(base, "/v1/embeddings", {
|
|
661
|
+
model: "search",
|
|
662
|
+
input: slice,
|
|
663
|
+
})
|
|
664
|
+
.then((reply) => reply.data.sort((a, b) => a.index - b.index))
|
|
665
|
+
.catch(async () => {
|
|
666
|
+
const rows = [];
|
|
667
|
+
for (const text of slice) rows.push(await one(text));
|
|
668
|
+
return rows;
|
|
669
|
+
});
|
|
670
|
+
for (const row of data) {
|
|
671
|
+
const vector = Float32Array.from(row.embedding);
|
|
672
|
+
let norm = 0;
|
|
673
|
+
for (const value of vector) norm += value * value;
|
|
674
|
+
norm = Math.sqrt(norm) || 1;
|
|
675
|
+
for (let j = 0; j < vector.length; j += 1) vector[j] /= norm;
|
|
676
|
+
out.push(vector);
|
|
543
677
|
}
|
|
544
678
|
}
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
const slice = input.slice(i, i + 16);
|
|
549
|
-
const data = await post(base, "/v1/embeddings", {
|
|
550
|
-
model: "search",
|
|
551
|
-
input: slice,
|
|
552
|
-
})
|
|
553
|
-
.then((reply) => reply.data.sort((a, b) => a.index - b.index))
|
|
554
|
-
.catch(async () => {
|
|
555
|
-
const rows = [];
|
|
556
|
-
for (const text of slice) rows.push(await one(text));
|
|
557
|
-
return rows;
|
|
558
|
-
});
|
|
559
|
-
for (const row of data) {
|
|
560
|
-
const vector = Float32Array.from(row.embedding);
|
|
561
|
-
let norm = 0;
|
|
562
|
-
for (const value of vector) norm += value * value;
|
|
563
|
-
norm = Math.sqrt(norm) || 1;
|
|
564
|
-
for (let j = 0; j < vector.length; j += 1) vector[j] /= norm;
|
|
565
|
-
out.push(vector);
|
|
566
|
-
}
|
|
567
|
-
}
|
|
568
|
-
return out;
|
|
569
|
-
}),
|
|
679
|
+
return out;
|
|
680
|
+
})
|
|
681
|
+
),
|
|
570
682
|
/**
|
|
571
683
|
* Embeds recordings and images with the search model's projector, in the same space as typed questions. items:
|
|
572
684
|
* { title, data (a Buffer of WAV, MP3, FLAC, PNG or JPEG) }. A server started before the projector came down is
|
|
573
685
|
* restarted once with it.
|
|
574
686
|
*/
|
|
575
687
|
embedMedia: (items, { lane = "background" } = {}) =>
|
|
576
|
-
schedule("llama", lane,
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
ready.
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
688
|
+
schedule("llama", lane, () =>
|
|
689
|
+
inUse("llama", async () => {
|
|
690
|
+
let ready = await llamaFor("media");
|
|
691
|
+
if (!ready.media) {
|
|
692
|
+
await stopServer();
|
|
693
|
+
ready = await llamaFor("media");
|
|
694
|
+
}
|
|
695
|
+
if (!ready.media)
|
|
696
|
+
throw new Error(
|
|
697
|
+
"the image and audio part of the search model is not downloaded yet"
|
|
698
|
+
);
|
|
699
|
+
// The marker that stands for a recording or an image in the text is the server's own, read once it has loaded.
|
|
700
|
+
if (!ready.marker) {
|
|
701
|
+
await post(ready.base, "/v1/embeddings", {
|
|
702
|
+
model: "media",
|
|
703
|
+
input: ["warm up"],
|
|
704
|
+
});
|
|
705
|
+
ready.marker = (
|
|
706
|
+
await (await fetch(`${ready.base}/props?model=media`)).json()
|
|
707
|
+
).media_marker;
|
|
708
|
+
}
|
|
709
|
+
const out = [];
|
|
710
|
+
for (const item of items) {
|
|
711
|
+
const reply = await post(ready.base, "/v1/embeddings", {
|
|
712
|
+
model: "media",
|
|
713
|
+
input: [
|
|
714
|
+
{
|
|
715
|
+
prompt_string: `title: ${item.title || "none"} | text: ${ready.marker}`,
|
|
716
|
+
multimodal_data: [Buffer.from(item.data).toString("base64")],
|
|
717
|
+
},
|
|
718
|
+
],
|
|
719
|
+
});
|
|
720
|
+
const vector = Float32Array.from(reply.data[0].embedding);
|
|
721
|
+
let norm = 0;
|
|
722
|
+
for (const value of vector) norm += value * value;
|
|
723
|
+
norm = Math.sqrt(norm) || 1;
|
|
724
|
+
out.push(vector.map((value) => value / norm));
|
|
725
|
+
}
|
|
726
|
+
return out;
|
|
727
|
+
})
|
|
728
|
+
),
|
|
616
729
|
/** Raw logits rows from an ONNX sequence classifier, for models calibrated on logit margins. */
|
|
617
730
|
logits: (id, texts, { lane = "handoff", maxLength } = {}) =>
|
|
618
731
|
schedule("onnx", lane, () =>
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
732
|
+
inUse(id, () =>
|
|
733
|
+
logits(
|
|
734
|
+
id,
|
|
735
|
+
texts.map((text) => [text]),
|
|
736
|
+
maxLength
|
|
737
|
+
)
|
|
623
738
|
)
|
|
624
739
|
),
|
|
625
740
|
/** Runs work against a multi-graph ONNX model in the scheduler: run({ ort, graph(file), dir }) opens graphs by file path. */
|
|
626
741
|
onnx: (id, run, { lane = "handoff" } = {}) =>
|
|
627
|
-
schedule("onnx", lane,
|
|
742
|
+
schedule("onnx", lane, () =>
|
|
743
|
+
inUse(id, async () => run(await graphsOf(id)))
|
|
744
|
+
),
|
|
628
745
|
running: () => Boolean(server && server.child.exitCode === null),
|
|
629
|
-
/**
|
|
746
|
+
/**
|
|
747
|
+
* What is in memory now: the router's models and their state, the ONNX models open (for `tryharness local`), and
|
|
748
|
+
* the memory ceiling with what is counted against it.
|
|
749
|
+
*/
|
|
630
750
|
loaded: async () => {
|
|
631
751
|
let llama = [];
|
|
632
752
|
if (server && server.child.exitCode === null) {
|
|
@@ -639,25 +759,16 @@ export function createInference({
|
|
|
639
759
|
}));
|
|
640
760
|
} catch {}
|
|
641
761
|
}
|
|
642
|
-
return {
|
|
762
|
+
return {
|
|
763
|
+
llama,
|
|
764
|
+
onnx: [...sessions.keys(), ...graphs.keys()],
|
|
765
|
+
memory: memory.status(),
|
|
766
|
+
};
|
|
643
767
|
},
|
|
644
768
|
stop() {
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
for (const
|
|
648
|
-
held
|
|
649
|
-
.then((ready) => {
|
|
650
|
-
clearTimeout(ready.timer);
|
|
651
|
-
ready.onnx.release?.();
|
|
652
|
-
})
|
|
653
|
-
.catch(() => {});
|
|
654
|
-
sessions.clear();
|
|
655
|
-
for (const held of graphs.values()) {
|
|
656
|
-
clearTimeout(held.timer);
|
|
657
|
-
for (const graph of held.open.values())
|
|
658
|
-
graph.then((onnx) => onnx.release?.()).catch(() => {});
|
|
659
|
-
}
|
|
660
|
-
graphs.clear();
|
|
769
|
+
void stopServer();
|
|
770
|
+
for (const id of [...sessions.keys()]) void closeSession(id);
|
|
771
|
+
for (const id of [...graphs.keys()]) void closeGraphs(id);
|
|
661
772
|
},
|
|
662
773
|
};
|
|
663
774
|
}
|