tryharness 0.7.59 → 0.7.60
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/boardProtocol.json +5 -1
- package/lib/cards.mjs +2 -1
- package/lib/commands.mjs +33 -1
- package/lib/daemon.mjs +32 -0
- package/lib/inference.mjs +115 -205
- package/lib/library.mjs +3 -0
- package/lib/listVersions.mjs +61 -0
- package/lib/localmodels.mjs +2 -1
- package/lib/ocr.mjs +54 -22
- package/lib/onboard.mjs +9 -6
- package/lib/onnxWorker.mjs +191 -0
- package/lib/ownMachines.mjs +164 -9
- package/lib/phones.mjs +3 -0
- package/lib/readers.mjs +15 -8
- package/lib/sessionControl.mjs +2 -2
- package/lib/sessionIndex.mjs +2 -2
- package/lib/speech.mjs +190 -175
- package/lib/swarm.mjs +34 -10
- package/lib/threadActions.mjs +6 -2
- package/lib/threadRuns.mjs +53 -62
- package/npm-shrinkwrap.json +14 -14
- package/package.json +4 -4
- package/release.json +24 -22
package/lib/boardProtocol.json
CHANGED
|
@@ -689,6 +689,8 @@
|
|
|
689
689
|
"machines.home",
|
|
690
690
|
"machines.models",
|
|
691
691
|
"machines.model",
|
|
692
|
+
"machines.live",
|
|
693
|
+
"machines.live.read",
|
|
692
694
|
"swarm.lineup",
|
|
693
695
|
"swarm.add",
|
|
694
696
|
"swarm.remove",
|
|
@@ -774,7 +776,9 @@
|
|
|
774
776
|
],
|
|
775
777
|
"browserMachines": [
|
|
776
778
|
"machines.add",
|
|
777
|
-
"machines.list"
|
|
779
|
+
"machines.list",
|
|
780
|
+
"machines.live",
|
|
781
|
+
"machines.live.read"
|
|
778
782
|
],
|
|
779
783
|
"browserSignIn": [
|
|
780
784
|
"agents.signin.status",
|
package/lib/cards.mjs
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { computerHandle } from "./ownMachines.mjs";
|
|
1
2
|
import { createHash } from "node:crypto";
|
|
2
3
|
import { member, nameOf } from "./members.mjs";
|
|
3
4
|
import { agentModelInUse, describeAgentModel } from "./models.mjs";
|
|
@@ -259,7 +260,7 @@ export function cardLine(card, now = Date.now(), { evidence = true } = {}) {
|
|
|
259
260
|
if (specialties) parts.push(specialties);
|
|
260
261
|
if (card.machines?.length)
|
|
261
262
|
parts.push(
|
|
262
|
-
`On the person's computers: ${card.machines.map((computer) => `${card.name} (${computer.label}${computer.online ? "" : ", offline"})`).join(", ")}; name one with # in a thread (@${card.id === "claude-code" ? "claude" : card.id} #${card.machines[0].label}), or target_machine on a task.`
|
|
263
|
+
`On the person's computers: ${card.machines.map((computer) => `${card.name} (${computer.label}${computer.online ? "" : ", offline"})`).join(", ")}; name one with # in a thread (@${card.id === "claude-code" ? "claude" : card.id} #${computerHandle(card.machines[0].label)}), or target_machine on a task.`
|
|
263
264
|
);
|
|
264
265
|
parts.push(sentence(card.answers));
|
|
265
266
|
const { done, failed, ended, medianMinutes } = card.record;
|
package/lib/commands.mjs
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import fs from "node:fs";
|
|
2
2
|
import path from "node:path";
|
|
3
|
-
import { spawnSync } from "node:child_process";
|
|
3
|
+
import { execFile, spawnSync } from "node:child_process";
|
|
4
4
|
import { MEMBERS, member } from "./members.mjs";
|
|
5
5
|
import { runtimeOf } from "./agentRuntimeConfig.mjs";
|
|
6
6
|
|
|
@@ -74,6 +74,38 @@ export const AGENT_COMMANDS = {
|
|
|
74
74
|
|
|
75
75
|
// An agent whose protocol server is a download of its own (Antigravity's ACP server) is still found by the command the
|
|
76
76
|
// person runs (its member's `cli`), so setup connects their own sessions.
|
|
77
|
+
/**
|
|
78
|
+
* `run` without holding up the daemon while the command works: for a command that can take a while, where the daemon
|
|
79
|
+
* answers requests meanwhile (OpenCode's session list took 0.6 s, every minute).
|
|
80
|
+
*/
|
|
81
|
+
export function runAsync(command, args, options = {}) {
|
|
82
|
+
return new Promise((resolve) => {
|
|
83
|
+
execFile(
|
|
84
|
+
command,
|
|
85
|
+
args,
|
|
86
|
+
{
|
|
87
|
+
encoding: "utf8",
|
|
88
|
+
timeout: options.timeout ?? 20_000,
|
|
89
|
+
maxBuffer: options.maxBuffer ?? 10_000_000,
|
|
90
|
+
cwd: options.cwd,
|
|
91
|
+
env: options.env,
|
|
92
|
+
},
|
|
93
|
+
(error, stdout, stderr) =>
|
|
94
|
+
resolve({
|
|
95
|
+
ok: !error,
|
|
96
|
+
status: error
|
|
97
|
+
? typeof error.code === "number"
|
|
98
|
+
? error.code
|
|
99
|
+
: null
|
|
100
|
+
: 0,
|
|
101
|
+
stdout: String(stdout || ""),
|
|
102
|
+
stderr: String(stderr || ""),
|
|
103
|
+
error: error?.message,
|
|
104
|
+
})
|
|
105
|
+
);
|
|
106
|
+
});
|
|
107
|
+
}
|
|
108
|
+
|
|
77
109
|
export function detectAgents() {
|
|
78
110
|
return Object.entries(AGENT_COMMANDS).map(([agent, defaultCommand]) => {
|
|
79
111
|
const command = runtimeOf(agent)?.command || defaultCommand;
|
package/lib/daemon.mjs
CHANGED
|
@@ -2194,6 +2194,26 @@ export async function startDaemon() {
|
|
|
2194
2194
|
log: (line) => log(`ssh: ${line}`),
|
|
2195
2195
|
})
|
|
2196
2196
|
: null;
|
|
2197
|
+
// This computer's own running sessions, and one session's latest messages from its own file, as the person: what home
|
|
2198
|
+
// gathers from every computer for the Agents tab, the phone and PiP (ownMachines.mjs sessions, readSession).
|
|
2199
|
+
const ownLive = async (live) => {
|
|
2200
|
+
if (live?.read)
|
|
2201
|
+
return liveCommand(
|
|
2202
|
+
{
|
|
2203
|
+
command: "board.live.read",
|
|
2204
|
+
to: live.read.to,
|
|
2205
|
+
sessionId: live.read.sessionId,
|
|
2206
|
+
folder: live.read.folder,
|
|
2207
|
+
limit: live.read.limit,
|
|
2208
|
+
},
|
|
2209
|
+
"user"
|
|
2210
|
+
);
|
|
2211
|
+
if (live?.list) {
|
|
2212
|
+
const listed = liveCommand({ command: "board.live" }, "user");
|
|
2213
|
+
return { ok: true, sessions: listed.sessions ?? [] };
|
|
2214
|
+
}
|
|
2215
|
+
return { ok: false, error: "only sessions are listed or read this way" };
|
|
2216
|
+
};
|
|
2197
2217
|
// The window's sign-in screen also shows what is still installing.
|
|
2198
2218
|
// This computer's agents' models, read or one set as the person: on its own store, never forwarded home, since its
|
|
2199
2219
|
// runs read the choice here (models.mjs). For the buddy's Swarm tab through home (ownMachines.mjs) and for the
|
|
@@ -2350,6 +2370,7 @@ export async function startDaemon() {
|
|
|
2350
2370
|
onOwnBoard: () => boardMirror.markBring(),
|
|
2351
2371
|
askOwnWindows: (call) => askOwnWindows(call),
|
|
2352
2372
|
ownModels,
|
|
2373
|
+
ownLive,
|
|
2353
2374
|
answerLink: proxyLinks
|
|
2354
2375
|
? (from, call) => proxyLinks.answer(from, call)
|
|
2355
2376
|
: null,
|
|
@@ -2606,6 +2627,17 @@ export async function startDaemon() {
|
|
|
2606
2627
|
"machines.models": onMachines((request) =>
|
|
2607
2628
|
ownMachines.models({ refresh: request.refresh === true })
|
|
2608
2629
|
),
|
|
2630
|
+
// Every computer's running sessions, and one read on the computer it runs on (ownMachines.mjs).
|
|
2631
|
+
"machines.live": onMachines((request) =>
|
|
2632
|
+
request.agent === "user"
|
|
2633
|
+
? ownMachines.sessions()
|
|
2634
|
+
: { ok: false, error: "only the person reads their sessions" }
|
|
2635
|
+
),
|
|
2636
|
+
"machines.live.read": onMachines((request) =>
|
|
2637
|
+
request.agent === "user"
|
|
2638
|
+
? ownMachines.readSession(request)
|
|
2639
|
+
: { ok: false, error: "only the person reads their sessions" }
|
|
2640
|
+
),
|
|
2609
2641
|
"machines.model": onMachines((request) =>
|
|
2610
2642
|
request.agent === "user"
|
|
2611
2643
|
? ownMachines.setModel(request)
|
package/lib/inference.mjs
CHANGED
|
@@ -4,17 +4,16 @@ import os from "node:os";
|
|
|
4
4
|
import path from "node:path";
|
|
5
5
|
import { spawn, spawnSync } from "node:child_process";
|
|
6
6
|
import { randomBytes } from "node:crypto";
|
|
7
|
+
import { Worker } from "node:worker_threads";
|
|
7
8
|
import {
|
|
8
9
|
MODELS,
|
|
9
10
|
modelById,
|
|
10
|
-
modelDir,
|
|
11
11
|
modelFile,
|
|
12
12
|
modelsPresent,
|
|
13
13
|
ensureLlama,
|
|
14
14
|
llamaServer,
|
|
15
15
|
} from "./localmodels.mjs";
|
|
16
16
|
import { ensureDir } from "./files.mjs";
|
|
17
|
-
import { loadCompactTokenizer } from "./tokenizer.mjs";
|
|
18
17
|
import {
|
|
19
18
|
createModelMemory,
|
|
20
19
|
memoryCeilingMb,
|
|
@@ -24,8 +23,9 @@ import {
|
|
|
24
23
|
// The relay's local inference host. GGUF models (the search model) run in one llama.cpp server in
|
|
25
24
|
// router mode, on the GPU when the machine has one that is faster, the CPU otherwise; the small ONNX classifiers run in
|
|
26
25
|
// native ONNX Runtime inside the daemon, one thread each (measured: about 13x less CPU energy per item than the
|
|
27
|
-
// WebAssembly build)
|
|
28
|
-
// handoff (a brief or post is in flight) and background
|
|
26
|
+
// WebAssembly build), on a thread of their own beside the daemon's (onnxWorker.mjs). Every request goes through one
|
|
27
|
+
// scheduler with three lanes: interactive (the person is waiting), handoff (a brief or post is in flight) and background
|
|
28
|
+
// (indexing, the nightly learner). Background work runs at the
|
|
29
29
|
// OS's background priority and waits while the machine is on low battery, in Low Power Mode or hot.
|
|
30
30
|
|
|
31
31
|
const LANES = { interactive: 0, handoff: 1, background: 2 };
|
|
@@ -162,7 +162,6 @@ export function createInference({
|
|
|
162
162
|
let starting = null;
|
|
163
163
|
const queues = { llama: [], onnx: [] };
|
|
164
164
|
const busy = { llama: false, onnx: false };
|
|
165
|
-
const sessions = new Map();
|
|
166
165
|
const memory = createModelMemory({ ceilingMb, log });
|
|
167
166
|
|
|
168
167
|
const device = () => {
|
|
@@ -426,156 +425,81 @@ export function createInference({
|
|
|
426
425
|
pump(backend);
|
|
427
426
|
});
|
|
428
427
|
|
|
429
|
-
//
|
|
430
|
-
//
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
file.path.endsWith("tokenizer.json")
|
|
440
|
-
);
|
|
441
|
-
const configPath = source.files.find((file) =>
|
|
442
|
-
file.path.endsWith("tokenizer_config.json")
|
|
443
|
-
)?.path;
|
|
444
|
-
const tokenizerPath = path.join(dir, tokenizerFile.path);
|
|
445
|
-
// Saved under the file's pinned hash, so a new tokenizer.json never meets old tables.
|
|
446
|
-
const tablesPath = path.join(
|
|
447
|
-
root,
|
|
448
|
-
"tokenizers",
|
|
449
|
-
`${tokenizerFile.sha256}.bin`
|
|
450
|
-
);
|
|
451
|
-
const library = () =>
|
|
452
|
-
new Tokenizer(
|
|
453
|
-
JSON.parse(fs.readFileSync(tokenizerPath, "utf8")),
|
|
454
|
-
configPath
|
|
455
|
-
? JSON.parse(fs.readFileSync(path.join(dir, configPath), "utf8"))
|
|
456
|
-
: {}
|
|
457
|
-
);
|
|
458
|
-
tokenizers.set(
|
|
459
|
-
dir,
|
|
460
|
-
loadCompactTokenizer(tokenizerPath, tablesPath) ?? library()
|
|
461
|
-
);
|
|
462
|
-
}
|
|
463
|
-
return tokenizers.get(dir);
|
|
464
|
-
};
|
|
465
|
-
const releaseTokenizer = (dir, user) => {
|
|
466
|
-
const users = tokenizerUsers.get(dir);
|
|
467
|
-
users?.delete(user);
|
|
468
|
-
if (users?.size) return;
|
|
469
|
-
tokenizerUsers.delete(dir);
|
|
470
|
-
tokenizers.delete(dir);
|
|
471
|
-
};
|
|
472
|
-
|
|
473
|
-
// An ONNX model is counted against the memory ceiling from just before it opens until it closes, idle or unloaded
|
|
474
|
-
// to make room. The ONNX lane runs one job at a time, so the models not in that job are always free to unload.
|
|
475
|
-
const closeSession = async (id) => {
|
|
476
|
-
const held = sessions.get(id);
|
|
477
|
-
sessions.delete(id);
|
|
428
|
+
// ONNX models run on a thread of their own (onnxWorker.mjs): ONNX Runtime runs a model synchronously on the thread
|
|
429
|
+
// that calls it, and this one answers every request. What decides stays here: which job runs next (one at a time, by
|
|
430
|
+
// lane), the memory ceiling, and when an idle model goes.
|
|
431
|
+
let worker = null;
|
|
432
|
+
let jobs = 0;
|
|
433
|
+
const pending = new Map();
|
|
434
|
+
const opened = new Map();
|
|
435
|
+
const forget = (id) => {
|
|
436
|
+
clearTimeout(opened.get(id));
|
|
437
|
+
opened.delete(id);
|
|
478
438
|
memory.gone(id);
|
|
479
|
-
const ready = await held?.catch(() => null);
|
|
480
|
-
if (!ready) return;
|
|
481
|
-
clearTimeout(ready.timer);
|
|
482
|
-
releaseTokenizer(ready.tokenizerDir, id);
|
|
483
|
-
await ready.onnx.release?.();
|
|
484
439
|
};
|
|
485
|
-
const
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
prepare: entry.tokenizer?.prepare ?? ((text) => text),
|
|
513
|
-
onnx,
|
|
514
|
-
timer: null,
|
|
515
|
-
};
|
|
516
|
-
})();
|
|
517
|
-
sessions.set(id, held);
|
|
518
|
-
held.catch(() => {
|
|
519
|
-
sessions.delete(id);
|
|
520
|
-
memory.gone(id);
|
|
521
|
-
});
|
|
522
|
-
}
|
|
523
|
-
const ready = await held;
|
|
524
|
-
clearTimeout(ready.timer);
|
|
525
|
-
ready.timer = setTimeout(() => void closeSession(id), ONNX_IDLE_MS);
|
|
526
|
-
ready.timer.unref?.();
|
|
527
|
-
return ready;
|
|
440
|
+
const thread = () => {
|
|
441
|
+
if (worker) return worker;
|
|
442
|
+
const started = new Worker(new URL("./onnxWorker.mjs", import.meta.url), {
|
|
443
|
+
workerData: { root, list },
|
|
444
|
+
});
|
|
445
|
+
started.unref();
|
|
446
|
+
started.on("message", ({ job, ok, value, error }) => {
|
|
447
|
+
const waiting = pending.get(job);
|
|
448
|
+
pending.delete(job);
|
|
449
|
+
if (!pending.size) started.unref();
|
|
450
|
+
if (ok) waiting?.resolve(value);
|
|
451
|
+
else waiting?.reject(new Error(error));
|
|
452
|
+
});
|
|
453
|
+
// A thread that died takes its models with it: each opens again on its next job, in a new thread.
|
|
454
|
+
const lost = (reason) => {
|
|
455
|
+
if (worker !== started) return;
|
|
456
|
+
worker = null;
|
|
457
|
+
for (const waiting of pending.values())
|
|
458
|
+
waiting.reject(new Error(`the model thread stopped (${reason})`));
|
|
459
|
+
pending.clear();
|
|
460
|
+
for (const id of [...opened.keys()]) forget(id);
|
|
461
|
+
log(`local models: the model thread stopped (${reason})`);
|
|
462
|
+
};
|
|
463
|
+
started.on("error", (error) => lost(error.message));
|
|
464
|
+
started.on("exit", (code) => lost(`exit ${code}`));
|
|
465
|
+
worker = started;
|
|
466
|
+
return worker;
|
|
528
467
|
};
|
|
468
|
+
const call = (op, request) =>
|
|
469
|
+
new Promise((resolve, reject) => {
|
|
470
|
+
const job = ++jobs;
|
|
471
|
+
const to = thread();
|
|
472
|
+
pending.set(job, { resolve, reject });
|
|
473
|
+
to.ref();
|
|
474
|
+
to.postMessage({ job, op, ...request });
|
|
475
|
+
});
|
|
529
476
|
|
|
530
|
-
//
|
|
531
|
-
//
|
|
532
|
-
const
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
memory.gone(id);
|
|
537
|
-
if (!held) return;
|
|
538
|
-
clearTimeout(held.timer);
|
|
539
|
-
for (const graph of held.open.values())
|
|
540
|
-
await graph.then((onnx) => onnx.release?.()).catch(() => {});
|
|
477
|
+
// An ONNX model is counted against the memory ceiling from just before its first job until it closes, idle or
|
|
478
|
+
// unloaded to make room. The ONNX lane runs one job at a time, so the models not in that job are always free to unload.
|
|
479
|
+
const unload = async (id) => {
|
|
480
|
+
if (!opened.has(id)) return;
|
|
481
|
+
forget(id);
|
|
482
|
+
if (worker) await call("close", { id }).catch(() => {});
|
|
541
483
|
};
|
|
542
|
-
const
|
|
484
|
+
const idle = (id) => {
|
|
485
|
+
clearTimeout(opened.get(id));
|
|
486
|
+
const timer = setTimeout(() => void unload(id), ONNX_IDLE_MS);
|
|
487
|
+
timer.unref?.();
|
|
488
|
+
opened.set(id, timer);
|
|
489
|
+
};
|
|
490
|
+
const admit = async (id) => {
|
|
543
491
|
const entry = modelById(id, list);
|
|
544
492
|
if (!entry || entry.runtime !== "onnx")
|
|
545
493
|
throw new Error(`no ONNX model ${id}`);
|
|
546
|
-
if (!
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
open: new Map(),
|
|
554
|
-
timer: null,
|
|
555
|
-
threads: entry.threads ?? 1,
|
|
556
|
-
};
|
|
557
|
-
graphs.set(id, held);
|
|
494
|
+
if (!opened.has(id)) {
|
|
495
|
+
const source = entry.tokenizer
|
|
496
|
+
? modelById(entry.tokenizer.from, list)
|
|
497
|
+
: entry;
|
|
498
|
+
if (!modelsPresent(root, [...new Set([id, source.id])], list))
|
|
499
|
+
throw new Error(`${entry.name} is not downloaded yet`);
|
|
500
|
+
await memory.admit(id, memoryCostMb(entry), () => unload(id));
|
|
558
501
|
}
|
|
559
|
-
|
|
560
|
-
held.timer = setTimeout(() => void closeGraphs(id), ONNX_IDLE_MS);
|
|
561
|
-
held.timer.unref?.();
|
|
562
|
-
const graph = (file) => {
|
|
563
|
-
if (!held.open.has(file)) {
|
|
564
|
-
const opened = held.ort.InferenceSession.create(
|
|
565
|
-
path.join(modelDir(root, entry), ...file.split("/")),
|
|
566
|
-
{
|
|
567
|
-
executionProviders: ["cpu"],
|
|
568
|
-
intraOpNumThreads: held.threads,
|
|
569
|
-
interOpNumThreads: 1,
|
|
570
|
-
logSeverityLevel: 3,
|
|
571
|
-
}
|
|
572
|
-
);
|
|
573
|
-
held.open.set(file, opened);
|
|
574
|
-
opened.catch(() => held.open.delete(file));
|
|
575
|
-
}
|
|
576
|
-
return held.open.get(file);
|
|
577
|
-
};
|
|
578
|
-
return { ort: held.ort, graph, dir: modelDir(root, entry) };
|
|
502
|
+
idle(id);
|
|
579
503
|
};
|
|
580
504
|
|
|
581
505
|
/** Runs work with a model marked in use, so the memory ceiling never unloads it mid-job. */
|
|
@@ -588,47 +512,18 @@ export function createInference({
|
|
|
588
512
|
}
|
|
589
513
|
};
|
|
590
514
|
|
|
591
|
-
/**
|
|
592
|
-
const
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
)
|
|
602
|
-
|
|
603
|
-
const feeds = {};
|
|
604
|
-
for (const name of onnx.inputNames) {
|
|
605
|
-
if (name === "input_ids")
|
|
606
|
-
feeds[name] = new ort.Tensor(
|
|
607
|
-
"int64",
|
|
608
|
-
BigInt64Array.from(ids, BigInt),
|
|
609
|
-
[1, ids.length]
|
|
610
|
-
);
|
|
611
|
-
else if (name === "attention_mask")
|
|
612
|
-
feeds[name] = new ort.Tensor(
|
|
613
|
-
"int64",
|
|
614
|
-
new BigInt64Array(ids.length).fill(1n),
|
|
615
|
-
[1, ids.length]
|
|
616
|
-
);
|
|
617
|
-
else if (name === "token_type_ids")
|
|
618
|
-
feeds[name] = new ort.Tensor(
|
|
619
|
-
"int64",
|
|
620
|
-
BigInt64Array.from(
|
|
621
|
-
(encoded.type_ids ?? ids.map(() => 0)).slice(0, ids.length),
|
|
622
|
-
BigInt
|
|
623
|
-
),
|
|
624
|
-
[1, ids.length]
|
|
625
|
-
);
|
|
626
|
-
}
|
|
627
|
-
const out = await onnx.run(feeds);
|
|
628
|
-
rows.push(Array.from(out[onnx.outputNames[0]].data, Number));
|
|
629
|
-
}
|
|
630
|
-
return rows;
|
|
631
|
-
};
|
|
515
|
+
/** One job on an ONNX model, run on its thread in the scheduler's lane. */
|
|
516
|
+
const onnxJob = (id, lane, op, request) =>
|
|
517
|
+
schedule("onnx", lane, () =>
|
|
518
|
+
inUse(id, async () => {
|
|
519
|
+
await admit(id);
|
|
520
|
+
try {
|
|
521
|
+
return await call(op, { id, ...request });
|
|
522
|
+
} finally {
|
|
523
|
+
if (opened.has(id)) idle(id);
|
|
524
|
+
}
|
|
525
|
+
})
|
|
526
|
+
);
|
|
632
527
|
|
|
633
528
|
return {
|
|
634
529
|
device,
|
|
@@ -740,22 +635,32 @@ export function createInference({
|
|
|
740
635
|
return out;
|
|
741
636
|
})
|
|
742
637
|
),
|
|
743
|
-
/**
|
|
638
|
+
/**
|
|
639
|
+
* Raw logits rows from an ONNX sequence classifier, for models calibrated on logit margins.
|
|
640
|
+
* @param {string} id
|
|
641
|
+
* @param {string[]} texts
|
|
642
|
+
* @param {{ lane?: string, maxLength?: number }} [options]
|
|
643
|
+
* @returns {Promise<number[][]>}
|
|
644
|
+
*/
|
|
744
645
|
logits: (id, texts, { lane = "handoff", maxLength } = {}) =>
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
646
|
+
onnxJob(id, lane, "logits", {
|
|
647
|
+
pairs: texts.map((text) => [text]),
|
|
648
|
+
maxLength,
|
|
649
|
+
}),
|
|
650
|
+
/**
|
|
651
|
+
* Runs a job against a multi-graph ONNX model on the model thread: `task` names a function a module exports,
|
|
652
|
+
* called there as fn({ ort, graph(file), dir }, ...args), graphs opened by their path in the model's folder. What
|
|
653
|
+
* goes in and comes back is copied between threads, so neither can be a function.
|
|
654
|
+
* @param {string} id
|
|
655
|
+
* @param {{ module: string, name: string }} task
|
|
656
|
+
* @param {unknown[]} [args]
|
|
657
|
+
* @param {{ lane?: string }} [options]
|
|
658
|
+
*/
|
|
659
|
+
onnx: (id, task, args = [], { lane = "handoff" } = {}) =>
|
|
660
|
+
onnxJob(id, lane, "task", { module: task.module, name: task.name, args }),
|
|
661
|
+
/** Opens a multi-graph model's graphs ahead of its first job (the dictation eval times loading apart from use). */
|
|
662
|
+
open: (id, files, { lane = "handoff" } = {}) =>
|
|
663
|
+
onnxJob(id, lane, "open", { files }),
|
|
759
664
|
running: () => Boolean(server && server.child.exitCode === null),
|
|
760
665
|
/**
|
|
761
666
|
* What is in memory now: the router's models and their state, the ONNX models open (for `tryharness local`), and
|
|
@@ -779,14 +684,19 @@ export function createInference({
|
|
|
779
684
|
}
|
|
780
685
|
return {
|
|
781
686
|
llama,
|
|
782
|
-
onnx: [...
|
|
687
|
+
onnx: [...opened.keys()],
|
|
783
688
|
memory: memory.status(),
|
|
784
689
|
};
|
|
785
690
|
},
|
|
786
691
|
stop() {
|
|
787
692
|
void stopServer();
|
|
788
|
-
|
|
789
|
-
|
|
693
|
+
const stopping = worker;
|
|
694
|
+
worker = null;
|
|
695
|
+
for (const waiting of pending.values())
|
|
696
|
+
waiting.reject(new Error("the inference host stopped"));
|
|
697
|
+
pending.clear();
|
|
698
|
+
for (const id of [...opened.keys()]) forget(id);
|
|
699
|
+
void stopping?.terminate();
|
|
790
700
|
},
|
|
791
701
|
};
|
|
792
702
|
}
|
package/lib/library.mjs
CHANGED
|
@@ -90,6 +90,9 @@ export function skipReason(file, stat) {
|
|
|
90
90
|
const name = path.basename(file);
|
|
91
91
|
if (SECRET_NAME.test(name)) return "looks like a key or password file";
|
|
92
92
|
if (name.endsWith(".icloud")) return "only in iCloud";
|
|
93
|
+
// macOS's AppleDouble companions (._photo.png beside photo.png on a drive or in a zip from a Mac) hold its metadata,
|
|
94
|
+
// not the file: read as pictures, every one failed.
|
|
95
|
+
if (name.startsWith("._")) return "macOS metadata, not the file";
|
|
93
96
|
// A file the cloud took back keeps its size but no blocks on the disk; reading it would download it.
|
|
94
97
|
if (stat.size > 0 && stat.blocks === 0) return "only in the cloud";
|
|
95
98
|
const kind = fileKind(name);
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
2
|
+
|
|
3
|
+
// A list a reader keeps a copy of (the buddy's threads, board.swarm.list), sent as what changed since the reader's copy
|
|
4
|
+
// instead of whole: on pop-os, the buddy read all 267 of the person's threads, 435 KB from the computer at home, on
|
|
5
|
+
// every move of the board. Each item is numbered by the version it last changed at; an item gone is remembered with the
|
|
6
|
+
// version it went at. Versions start again with each run of the daemon (its epoch), and a reader from another run, or
|
|
7
|
+
// from before the removals still remembered, gets the whole list.
|
|
8
|
+
|
|
9
|
+
// Items gone that a reader may still hold; one further behind than these gets the whole list.
|
|
10
|
+
const GONE_KEPT = 1_000;
|
|
11
|
+
|
|
12
|
+
export function createListVersions() {
|
|
13
|
+
const epoch = randomUUID();
|
|
14
|
+
let version = 0;
|
|
15
|
+
const current = new Map();
|
|
16
|
+
const gone = new Map();
|
|
17
|
+
// A reader behind this version may miss a removal: gone ones before it are forgotten.
|
|
18
|
+
let forgotten = 0;
|
|
19
|
+
const digestOf = (item) =>
|
|
20
|
+
createHash("sha256").update(JSON.stringify(item)).digest("base64url");
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* The list for a reader that holds it up to `after` (from `epoch`): { items, gone, whole, version: { epoch, at, count } }.
|
|
24
|
+
* `items` are those changed since, or all of them (`whole`) when the reader holds none from this run; `gone` the ids
|
|
25
|
+
* removed since; `count` how many there are, so a reader whose copy differs asks for all of it.
|
|
26
|
+
*/
|
|
27
|
+
return function since(items, { epoch: theirs, after } = {}) {
|
|
28
|
+
const present = new Set();
|
|
29
|
+
for (const item of items) {
|
|
30
|
+
present.add(item.id);
|
|
31
|
+
const digest = digestOf(item);
|
|
32
|
+
if (current.get(item.id)?.digest === digest) continue;
|
|
33
|
+
current.set(item.id, { digest, at: ++version });
|
|
34
|
+
gone.delete(item.id);
|
|
35
|
+
}
|
|
36
|
+
for (const id of [...current.keys()]) {
|
|
37
|
+
if (present.has(id)) continue;
|
|
38
|
+
current.delete(id);
|
|
39
|
+
gone.set(id, ++version);
|
|
40
|
+
}
|
|
41
|
+
for (const [id, at] of [...gone].slice(
|
|
42
|
+
0,
|
|
43
|
+
Math.max(0, gone.size - GONE_KEPT)
|
|
44
|
+
)) {
|
|
45
|
+
gone.delete(id);
|
|
46
|
+
forgotten = at;
|
|
47
|
+
}
|
|
48
|
+
const whole =
|
|
49
|
+
theirs !== epoch || !Number.isInteger(after) || after < forgotten;
|
|
50
|
+
return {
|
|
51
|
+
items: whole
|
|
52
|
+
? items
|
|
53
|
+
: items.filter((item) => current.get(item.id).at > after),
|
|
54
|
+
gone: whole
|
|
55
|
+
? []
|
|
56
|
+
: [...gone].filter(([, at]) => at > after).map(([id]) => id),
|
|
57
|
+
whole,
|
|
58
|
+
version: { epoch, at: version, count: items.length },
|
|
59
|
+
};
|
|
60
|
+
};
|
|
61
|
+
}
|
package/lib/localmodels.mjs
CHANGED
|
@@ -162,7 +162,8 @@ export const MODELS = [
|
|
|
162
162
|
size: 2031,
|
|
163
163
|
},
|
|
164
164
|
],
|
|
165
|
-
|
|
165
|
+
// By name: the model list goes to the inference worker (onnxWorker.mjs), which cannot be handed a function.
|
|
166
|
+
tokenizer: { from: "guard", prepare: "unhide" },
|
|
166
167
|
},
|
|
167
168
|
{
|
|
168
169
|
id: "credentials",
|