tryharness 0.7.58 → 0.7.60
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/boardProtocol.json +5 -1
- package/lib/cards.mjs +2 -1
- package/lib/commands.mjs +33 -1
- package/lib/daemon.mjs +45 -0
- package/lib/inference.mjs +115 -205
- package/lib/library.mjs +3 -0
- package/lib/listVersions.mjs +61 -0
- package/lib/live.mjs +2 -2
- package/lib/localmodels.mjs +2 -1
- package/lib/ocr.mjs +54 -22
- package/lib/onboard.mjs +9 -6
- package/lib/onnxWorker.mjs +191 -0
- package/lib/ownMachines.mjs +209 -8
- package/lib/phones.mjs +3 -0
- package/lib/readers.mjs +15 -8
- package/lib/sessionControl.mjs +6 -4
- package/lib/sessionIndex.mjs +2 -2
- package/lib/speech.mjs +190 -175
- package/lib/swarm.mjs +34 -10
- package/lib/threadActions.mjs +11 -0
- package/lib/threadRuns.mjs +64 -9
- package/npm-shrinkwrap.json +14 -14
- package/package.json +4 -4
- package/release.json +25 -23
package/lib/ocr.mjs
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import fs from "node:fs";
|
|
2
|
+
import { randomUUID } from "node:crypto";
|
|
2
3
|
import path from "node:path";
|
|
3
4
|
import pngjs from "pngjs";
|
|
4
5
|
import jpeg from "jpeg-js";
|
|
@@ -335,43 +336,74 @@ const readingOrder = (lines) =>
|
|
|
335
336
|
: a.box[1] - b.box[1]
|
|
336
337
|
);
|
|
337
338
|
|
|
339
|
+
// Pictures being read, by their read's token, on the model thread (onnxWorker.mjs) that runs the jobs below: decoded
|
|
340
|
+
// once by the detector's job and read by the recogniser's, then dropped. A read that failed partway leaves its picture
|
|
341
|
+
// here until newer reads push it out.
|
|
342
|
+
const OPEN_PICTURES = 4;
|
|
343
|
+
const pictures = new Map();
|
|
344
|
+
let characters = null;
|
|
345
|
+
|
|
346
|
+
/** The detector's job, on the model thread: decodes the picture, keeps it for the recogniser and finds its text boxes. */
|
|
347
|
+
export async function detectText(model, token, input) {
|
|
348
|
+
const image = input?.data && input.width ? input : decodeImage(input);
|
|
349
|
+
if (!image) throw new Error("only PNG and JPEG images can be read");
|
|
350
|
+
const boxes = await detect(image, model);
|
|
351
|
+
if (boxes.length) {
|
|
352
|
+
pictures.set(token, image);
|
|
353
|
+
for (const old of [...pictures.keys()].slice(0, -OPEN_PICTURES))
|
|
354
|
+
pictures.delete(old);
|
|
355
|
+
}
|
|
356
|
+
return { width: image.width, height: image.height, boxes };
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
/** A recogniser job, on the model thread: some of the picture's boxes read; the last one lets the picture go. */
|
|
360
|
+
export async function readBoxes(model, token, boxes, last) {
|
|
361
|
+
const image = pictures.get(token);
|
|
362
|
+
if (!image) throw new Error("the picture being read was let go");
|
|
363
|
+
characters ||= characterList(
|
|
364
|
+
fs.readFileSync(path.join(model.dir, "rec", "inference.yml"), "utf8")
|
|
365
|
+
);
|
|
366
|
+
const read = [];
|
|
367
|
+
for (const box of boxes)
|
|
368
|
+
read.push(await recognise(image, box, characters, model));
|
|
369
|
+
if (last) pictures.delete(token);
|
|
370
|
+
return read;
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
const DETECT = { module: import.meta.url, name: "detectText" };
|
|
374
|
+
const READ_BOXES = { module: import.meta.url, name: "readBoxes" };
|
|
375
|
+
|
|
338
376
|
export function createOcr({ inference }) {
|
|
339
|
-
let characters = null;
|
|
340
377
|
/**
|
|
341
378
|
* The text in an image: { width, height, lines: [{ text, box: [x0, y0, x1, y1], score, chars: [[x0, x1]] }], ms }, boxes
|
|
342
379
|
* in the image's pixels. Takes PNG or JPEG bytes or a decoded { width, height, data: RGBA }.
|
|
343
380
|
*/
|
|
344
381
|
const read = async (input, { lane = "handoff" } = {}) => {
|
|
345
382
|
const started = performance.now();
|
|
346
|
-
const
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
383
|
+
const token = randomUUID();
|
|
384
|
+
const { width, height, boxes } = await inference.onnx(
|
|
385
|
+
"ocr",
|
|
386
|
+
DETECT,
|
|
387
|
+
[token, input],
|
|
388
|
+
{ lane }
|
|
389
|
+
);
|
|
351
390
|
const lines = [];
|
|
352
|
-
for (let i = 0; i < boxes.length; i += REC_PER_JOB)
|
|
391
|
+
for (let i = 0; i < boxes.length; i += REC_PER_JOB)
|
|
353
392
|
lines.push(
|
|
354
393
|
...(await inference.onnx(
|
|
355
394
|
"ocr",
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
);
|
|
363
|
-
const read = [];
|
|
364
|
-
for (const box of boxes.slice(i, i + REC_PER_JOB))
|
|
365
|
-
read.push(await recognise(image, box, characters, model));
|
|
366
|
-
return read;
|
|
367
|
-
},
|
|
395
|
+
READ_BOXES,
|
|
396
|
+
[
|
|
397
|
+
token,
|
|
398
|
+
boxes.slice(i, i + REC_PER_JOB),
|
|
399
|
+
i + REC_PER_JOB >= boxes.length,
|
|
400
|
+
],
|
|
368
401
|
{ lane }
|
|
369
402
|
))
|
|
370
403
|
);
|
|
371
|
-
}
|
|
372
404
|
return {
|
|
373
|
-
width
|
|
374
|
-
height
|
|
405
|
+
width,
|
|
406
|
+
height,
|
|
375
407
|
lines: readingOrder(lines.filter((line) => line.text)),
|
|
376
408
|
ms: Math.round(performance.now() - started),
|
|
377
409
|
};
|
package/lib/onboard.mjs
CHANGED
|
@@ -70,17 +70,17 @@ const copy = (text) => {
|
|
|
70
70
|
return result.status === 0;
|
|
71
71
|
};
|
|
72
72
|
|
|
73
|
-
// How long setup waits for
|
|
74
|
-
//
|
|
75
|
-
|
|
76
|
-
const
|
|
73
|
+
// How long setup waits for any agent's answer to its check before moving on: a minute at most (owner, October 27: "10
|
|
74
|
+
// minutes is entirely too long"). Grok Bot's email wakes have taken up to 59 s, so a slow one can miss it; the person
|
|
75
|
+
// skips the wait with Enter and tests again with `agents test`.
|
|
76
|
+
const ANSWER_WAIT_MS = 60_000;
|
|
77
77
|
|
|
78
78
|
/** Post a setup check for the agent and wait for its report: the same path real work takes. Enter skips the wait. */
|
|
79
79
|
export async function testAgent({
|
|
80
80
|
agent,
|
|
81
81
|
daemonRequest,
|
|
82
82
|
ui = createUi(),
|
|
83
|
-
waitMs = ANSWER_WAIT_MS
|
|
83
|
+
waitMs = ANSWER_WAIT_MS,
|
|
84
84
|
sleep = (ms) => new Promise((r) => setTimeout(r, ms)),
|
|
85
85
|
}) {
|
|
86
86
|
const name = LABEL[agent] || agent;
|
|
@@ -675,7 +675,10 @@ export async function setupElsewhere({
|
|
|
675
675
|
: [];
|
|
676
676
|
const joined = [];
|
|
677
677
|
for (const id of joining) {
|
|
678
|
-
const added = await daemonRequest({
|
|
678
|
+
const added = await daemonRequest({
|
|
679
|
+
command: "swarm.add",
|
|
680
|
+
target: id,
|
|
681
|
+
}).catch(() => null);
|
|
679
682
|
if (added?.ok) joined.push({ id, name: added.name });
|
|
680
683
|
}
|
|
681
684
|
if (joined.length)
|
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
import fs from "node:fs";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { parentPort, workerData } from "node:worker_threads";
|
|
4
|
+
import { modelById, modelDir, modelFile, unhide } from "./localmodels.mjs";
|
|
5
|
+
import { loadCompactTokenizer } from "./tokenizer.mjs";
|
|
6
|
+
|
|
7
|
+
// The ONNX half of the inference host (inference.mjs), on a thread of its own. ONNX Runtime runs a model synchronously
|
|
8
|
+
// on the thread that calls it (onnxruntime-node's run is the native call inside a setImmediate), so on the daemon's
|
|
9
|
+
// thread every OCR band, guard window and speech step held up every request: up to 0.8 s at a time while a fresh install
|
|
10
|
+
// read the pictures in the person's folders. Here the sessions, their tokenizers and each job's own work around them
|
|
11
|
+
// (decoding a picture, preparing a tensor, reading the outputs) run beside the models, and the daemon only waits for
|
|
12
|
+
// the answer. The daemon's scheduler still decides which job runs next and when a model is unloaded; this thread runs
|
|
13
|
+
// one job at a time, as it is sent them.
|
|
14
|
+
|
|
15
|
+
const { root, list } = workerData;
|
|
16
|
+
// A tokenizer step named in the model list, which cannot carry a function to this thread.
|
|
17
|
+
const PREPARE = { unhide };
|
|
18
|
+
const SESSION_OPTIONS = {
|
|
19
|
+
executionProviders: ["cpu"],
|
|
20
|
+
interOpNumThreads: 1,
|
|
21
|
+
logSeverityLevel: 3,
|
|
22
|
+
};
|
|
23
|
+
|
|
24
|
+
let runtime = null;
|
|
25
|
+
const ort = () => (runtime ??= import("onnxruntime-node"));
|
|
26
|
+
|
|
27
|
+
// Models on one vocabulary share its tokenizer while any of them is open; it goes with the last of them.
|
|
28
|
+
const tokenizers = new Map();
|
|
29
|
+
const tokenizerUsers = new Map();
|
|
30
|
+
async function tokenizerOf(source, user) {
|
|
31
|
+
const dir = modelDir(root, source);
|
|
32
|
+
if (!tokenizerUsers.has(dir)) tokenizerUsers.set(dir, new Set());
|
|
33
|
+
tokenizerUsers.get(dir).add(user);
|
|
34
|
+
if (!tokenizers.has(dir)) {
|
|
35
|
+
const tokenizerFile = source.files.find((file) =>
|
|
36
|
+
file.path.endsWith("tokenizer.json")
|
|
37
|
+
);
|
|
38
|
+
const configPath = source.files.find((file) =>
|
|
39
|
+
file.path.endsWith("tokenizer_config.json")
|
|
40
|
+
)?.path;
|
|
41
|
+
const tokenizerPath = path.join(dir, tokenizerFile.path);
|
|
42
|
+
// Saved under the file's pinned hash, so a new tokenizer.json never meets old tables.
|
|
43
|
+
const tablesPath = path.join(
|
|
44
|
+
root,
|
|
45
|
+
"tokenizers",
|
|
46
|
+
`${tokenizerFile.sha256}.bin`
|
|
47
|
+
);
|
|
48
|
+
const library = async () => {
|
|
49
|
+
const { Tokenizer } = await import("@huggingface/tokenizers");
|
|
50
|
+
return new Tokenizer(
|
|
51
|
+
JSON.parse(fs.readFileSync(tokenizerPath, "utf8")),
|
|
52
|
+
configPath
|
|
53
|
+
? JSON.parse(fs.readFileSync(path.join(dir, configPath), "utf8"))
|
|
54
|
+
: {}
|
|
55
|
+
);
|
|
56
|
+
};
|
|
57
|
+
tokenizers.set(
|
|
58
|
+
dir,
|
|
59
|
+
loadCompactTokenizer(tokenizerPath, tablesPath) ?? (await library())
|
|
60
|
+
);
|
|
61
|
+
}
|
|
62
|
+
return tokenizers.get(dir);
|
|
63
|
+
}
|
|
64
|
+
function releaseTokenizer(dir, user) {
|
|
65
|
+
const users = tokenizerUsers.get(dir);
|
|
66
|
+
users?.delete(user);
|
|
67
|
+
if (users?.size) return;
|
|
68
|
+
tokenizerUsers.delete(dir);
|
|
69
|
+
tokenizers.delete(dir);
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
// Each open model: its graphs by file (a classifier has one, OCR and speech several), opened on first use, and the
|
|
73
|
+
// tokenizer a classifier reads with.
|
|
74
|
+
const models = new Map();
|
|
75
|
+
function modelOf(id) {
|
|
76
|
+
const entry = modelById(id, list);
|
|
77
|
+
if (!entry || entry.runtime !== "onnx")
|
|
78
|
+
throw new Error(`no ONNX model ${id}`);
|
|
79
|
+
if (!models.has(id)) models.set(id, { entry, graphs: new Map() });
|
|
80
|
+
return models.get(id);
|
|
81
|
+
}
|
|
82
|
+
async function graph(model, file) {
|
|
83
|
+
if (!model.graphs.has(file)) {
|
|
84
|
+
const opened = ort().then((runtime) =>
|
|
85
|
+
runtime.InferenceSession.create(file, {
|
|
86
|
+
...SESSION_OPTIONS,
|
|
87
|
+
intraOpNumThreads: model.entry.threads ?? 1,
|
|
88
|
+
})
|
|
89
|
+
);
|
|
90
|
+
model.graphs.set(file, opened);
|
|
91
|
+
opened.catch(() => model.graphs.delete(file));
|
|
92
|
+
}
|
|
93
|
+
return model.graphs.get(file);
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/** What a job runs against: ONNX Runtime, the model's graphs opened by their path in its folder, and the folder. */
|
|
97
|
+
async function graphsOf(id) {
|
|
98
|
+
const model = modelOf(id);
|
|
99
|
+
const dir = modelDir(root, model.entry);
|
|
100
|
+
return {
|
|
101
|
+
ort: await ort(),
|
|
102
|
+
graph: (file) => graph(model, path.join(dir, ...file.split("/"))),
|
|
103
|
+
dir,
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/** Runs a sequence classifier on each encoded input; returns the first output's leading row (its logits). */
|
|
108
|
+
async function logits(id, pairs, maxLength = 512) {
|
|
109
|
+
const model = modelOf(id);
|
|
110
|
+
const { entry } = model;
|
|
111
|
+
const source = entry.tokenizer
|
|
112
|
+
? modelById(entry.tokenizer.from, list)
|
|
113
|
+
: entry;
|
|
114
|
+
if (!model.tokenizer) {
|
|
115
|
+
model.tokenizer = await tokenizerOf(source, id);
|
|
116
|
+
model.tokenizerDir = modelDir(root, source);
|
|
117
|
+
}
|
|
118
|
+
const prepare = PREPARE[entry.tokenizer?.prepare] ?? ((text) => text);
|
|
119
|
+
const runtime = await ort();
|
|
120
|
+
const onnx = await graph(model, modelFile(root, entry));
|
|
121
|
+
const rows = [];
|
|
122
|
+
for (const [text, pair] of pairs) {
|
|
123
|
+
const encoded = model.tokenizer.encode(
|
|
124
|
+
prepare(text),
|
|
125
|
+
pair === undefined
|
|
126
|
+
? { add_special_tokens: true }
|
|
127
|
+
: { text_pair: prepare(pair), add_special_tokens: true }
|
|
128
|
+
);
|
|
129
|
+
const ids = encoded.ids.slice(0, maxLength);
|
|
130
|
+
const feeds = {};
|
|
131
|
+
for (const name of onnx.inputNames) {
|
|
132
|
+
if (name === "input_ids")
|
|
133
|
+
feeds[name] = new runtime.Tensor(
|
|
134
|
+
"int64",
|
|
135
|
+
BigInt64Array.from(ids, BigInt),
|
|
136
|
+
[1, ids.length]
|
|
137
|
+
);
|
|
138
|
+
else if (name === "attention_mask")
|
|
139
|
+
feeds[name] = new runtime.Tensor(
|
|
140
|
+
"int64",
|
|
141
|
+
new BigInt64Array(ids.length).fill(1n),
|
|
142
|
+
[1, ids.length]
|
|
143
|
+
);
|
|
144
|
+
else if (name === "token_type_ids")
|
|
145
|
+
feeds[name] = new runtime.Tensor(
|
|
146
|
+
"int64",
|
|
147
|
+
BigInt64Array.from(
|
|
148
|
+
(encoded.type_ids ?? ids.map(() => 0)).slice(0, ids.length),
|
|
149
|
+
BigInt
|
|
150
|
+
),
|
|
151
|
+
[1, ids.length]
|
|
152
|
+
);
|
|
153
|
+
}
|
|
154
|
+
const out = await onnx.run(feeds);
|
|
155
|
+
rows.push(Array.from(out[onnx.outputNames[0]].data, Number));
|
|
156
|
+
}
|
|
157
|
+
return rows;
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
async function close(id) {
|
|
161
|
+
const model = models.get(id);
|
|
162
|
+
models.delete(id);
|
|
163
|
+
if (!model) return;
|
|
164
|
+
if (model.tokenizerDir) releaseTokenizer(model.tokenizerDir, id);
|
|
165
|
+
for (const opened of model.graphs.values())
|
|
166
|
+
await opened.then((session) => session.release?.()).catch(() => {});
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
const OPS = {
|
|
170
|
+
logits: ({ id, pairs, maxLength }) => logits(id, pairs, maxLength),
|
|
171
|
+
// A job's own function, by the module that exports it: run({ ort, graph(file), dir }, ...args).
|
|
172
|
+
task: async ({ id, module, name, args }) =>
|
|
173
|
+
(await import(module))[name](await graphsOf(id), ...args),
|
|
174
|
+
open: async ({ id, files }) => {
|
|
175
|
+
const { graph: open } = await graphsOf(id);
|
|
176
|
+
await Promise.all(files.map((file) => open(file)));
|
|
177
|
+
},
|
|
178
|
+
close: ({ id }) => close(id),
|
|
179
|
+
};
|
|
180
|
+
|
|
181
|
+
parentPort.on("message", async ({ job, op, ...request }) => {
|
|
182
|
+
try {
|
|
183
|
+
parentPort.postMessage({ job, ok: true, value: await OPS[op](request) });
|
|
184
|
+
} catch (error) {
|
|
185
|
+
parentPort.postMessage({
|
|
186
|
+
job,
|
|
187
|
+
ok: false,
|
|
188
|
+
error: error instanceof Error ? error.message : String(error),
|
|
189
|
+
});
|
|
190
|
+
}
|
|
191
|
+
});
|
package/lib/ownMachines.mjs
CHANGED
|
@@ -4,6 +4,7 @@ import { randomBytes, randomUUID } from "node:crypto";
|
|
|
4
4
|
import { documentText, ensureDir, isDocument, mimeFor } from "./files.mjs";
|
|
5
5
|
import { FILE_PART_BYTES } from "./boardFiles.mjs";
|
|
6
6
|
import { fingerprint } from "./sealed.mjs";
|
|
7
|
+
import { nameOf } from "./members.mjs";
|
|
7
8
|
import {
|
|
8
9
|
isPairCode,
|
|
9
10
|
joinProof,
|
|
@@ -44,6 +45,19 @@ const DIRECT_COMMAND = "machine.direct";
|
|
|
44
45
|
// The person's model and effort for an agent live on the computer that runs it (models.mjs reads its own store), so
|
|
45
46
|
// the buddy's Swarm tab reads and sets them on each computer: a worker asks home, home asks the worker that has it.
|
|
46
47
|
const MODELS_COMMAND = "machine.models";
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* How a computer is written after # in a message ("Work Mac" is #Work-Mac): its name with spaces as dashes and anything
|
|
51
|
+
* but letters, digits, dots, dashes and underscores left out. Every menu writes it this way and computerNamed reads it.
|
|
52
|
+
*/
|
|
53
|
+
export const computerHandle = (label) =>
|
|
54
|
+
String(label ?? "")
|
|
55
|
+
.trim()
|
|
56
|
+
.replace(/\s+/g, "-")
|
|
57
|
+
.replace(/[^\w.-]/g, "");
|
|
58
|
+
// Every computer's running agent sessions, and one session's latest messages read on the computer it runs on, for the
|
|
59
|
+
// buddy's Agents tab, the phone and PiP: a worker asks home, home asks the worker that has it.
|
|
60
|
+
const LIVE_COMMAND = "machine.live";
|
|
47
61
|
// A worker passing on the person's choice of another computer as Home (makeHome).
|
|
48
62
|
const HOME_COMMAND = "machine.home";
|
|
49
63
|
// Since when this worker has found home unreachable: { home, since, checked } (watchHome).
|
|
@@ -146,6 +160,12 @@ export function createOwnMachines({
|
|
|
146
160
|
ok: false,
|
|
147
161
|
error: "this computer cannot list its agents' models",
|
|
148
162
|
}),
|
|
163
|
+
// This computer's own running sessions ({ list: true }) or one session's latest messages ({ read }), as the person
|
|
164
|
+
// (sessionControl.mjs board.live and board.live.read).
|
|
165
|
+
ownLive = async () => ({
|
|
166
|
+
ok: false,
|
|
167
|
+
error: "this computer cannot list its sessions",
|
|
168
|
+
}),
|
|
149
169
|
// A proxy link's model call from one of the person's cloud computers, on the computer holding the link
|
|
150
170
|
// (proxyLinks.mjs answer); null where none is held.
|
|
151
171
|
answerLink = null,
|
|
@@ -1454,6 +1474,15 @@ export function createOwnMachines({
|
|
|
1454
1474
|
: { ok: false, error: "this computer holds no proxy link" };
|
|
1455
1475
|
if (command === MODELS_COMMAND || command === MODEL_COMMAND)
|
|
1456
1476
|
return answerModelsCall(call, from, !pinned.hosted);
|
|
1477
|
+
// What runs in the person's sessions is theirs: a cloud computer, which runs agents only, never reads them.
|
|
1478
|
+
if (command === LIVE_COMMAND)
|
|
1479
|
+
return pinned.hosted
|
|
1480
|
+
? {
|
|
1481
|
+
ok: false,
|
|
1482
|
+
error:
|
|
1483
|
+
"That computer runs agents only; it does not read your sessions.",
|
|
1484
|
+
}
|
|
1485
|
+
: answerLiveCall(call, from);
|
|
1457
1486
|
if (command === SSH_COMMAND)
|
|
1458
1487
|
return answerSshCall(call, from, !pinned.hosted);
|
|
1459
1488
|
// Only the person, from a computer they drive, moves Home (a hosted computer is Harness's to run).
|
|
@@ -1784,14 +1813,16 @@ export function createOwnMachines({
|
|
|
1784
1813
|
? await answerLink(pinned.machine_id, call.link)
|
|
1785
1814
|
: call.models
|
|
1786
1815
|
? await ownModels(call.models)
|
|
1787
|
-
: call.
|
|
1788
|
-
?
|
|
1789
|
-
:
|
|
1790
|
-
?
|
|
1791
|
-
:
|
|
1792
|
-
|
|
1793
|
-
|
|
1794
|
-
|
|
1816
|
+
: call.live
|
|
1817
|
+
? await ownLive(call.live)
|
|
1818
|
+
: call.ssh
|
|
1819
|
+
? ownSshAllow(call.ssh)
|
|
1820
|
+
: WINDOW_TOOLS.has(call.tool)
|
|
1821
|
+
? await askOwnWindows(call)
|
|
1822
|
+
: {
|
|
1823
|
+
ok: false,
|
|
1824
|
+
error: `home cannot ask this computer's windows for ${call.tool}`,
|
|
1825
|
+
};
|
|
1795
1826
|
return {
|
|
1796
1827
|
envelope: await sealAskAnswer(
|
|
1797
1828
|
key,
|
|
@@ -1839,6 +1870,7 @@ export function createOwnMachines({
|
|
|
1839
1870
|
id: machine.id,
|
|
1840
1871
|
label: machine.label,
|
|
1841
1872
|
role: machine.role ?? "home",
|
|
1873
|
+
agents,
|
|
1842
1874
|
alone: actsAlone(),
|
|
1843
1875
|
standingInFor: standingIn()
|
|
1844
1876
|
? (labelOf(standingIn()) ?? standingIn())
|
|
@@ -2008,6 +2040,129 @@ export function createOwnMachines({
|
|
|
2008
2040
|
};
|
|
2009
2041
|
};
|
|
2010
2042
|
|
|
2043
|
+
// ---- every computer's running sessions, for the Agents tab, the phone and PiP ----
|
|
2044
|
+
|
|
2045
|
+
/** One computer's sessions as it answered, or why there are none: offline, or it did not answer. */
|
|
2046
|
+
const liveEntry = (computer, answer) => ({
|
|
2047
|
+
id: computer.id,
|
|
2048
|
+
label: computer.label,
|
|
2049
|
+
role: computer.role ?? "home",
|
|
2050
|
+
online: computer.online,
|
|
2051
|
+
sessions: answer?.ok ? (answer.sessions ?? []) : [],
|
|
2052
|
+
...(answer?.ok
|
|
2053
|
+
? {}
|
|
2054
|
+
: {
|
|
2055
|
+
error: computer.online
|
|
2056
|
+
? (answer?.error ?? `${computer.label} did not answer`)
|
|
2057
|
+
: `${computer.label} is offline`,
|
|
2058
|
+
}),
|
|
2059
|
+
});
|
|
2060
|
+
|
|
2061
|
+
/** On home: home and each confirmed worker but `except`, each listing its own sessions. */
|
|
2062
|
+
const everyComputersLive = async (except = null) => {
|
|
2063
|
+
const workers = siblings.filter(
|
|
2064
|
+
(sibling) => sibling.id !== except && confirmed(sibling)
|
|
2065
|
+
);
|
|
2066
|
+
const [own, ...answers] = await Promise.all([
|
|
2067
|
+
except === machine.id ? null : ownLive({ list: true }),
|
|
2068
|
+
...workers.map((worker) =>
|
|
2069
|
+
worker.connected ? askWorker(worker.id, { live: { list: true } }) : null
|
|
2070
|
+
),
|
|
2071
|
+
]);
|
|
2072
|
+
return [
|
|
2073
|
+
liveEntry(
|
|
2074
|
+
{
|
|
2075
|
+
id: machine.id,
|
|
2076
|
+
label: machine.label,
|
|
2077
|
+
role: machine.role ?? "home",
|
|
2078
|
+
online: true,
|
|
2079
|
+
},
|
|
2080
|
+
own
|
|
2081
|
+
),
|
|
2082
|
+
...workers.map((worker, index) =>
|
|
2083
|
+
liveEntry({ ...worker, online: worker.connected }, answers[index])
|
|
2084
|
+
),
|
|
2085
|
+
];
|
|
2086
|
+
};
|
|
2087
|
+
|
|
2088
|
+
/** The person's computers, this one first, each with the agent sessions running on it. */
|
|
2089
|
+
const sessions = async () => {
|
|
2090
|
+
if (isHome())
|
|
2091
|
+
return {
|
|
2092
|
+
ok: true,
|
|
2093
|
+
here: machine.id,
|
|
2094
|
+
computers: await everyComputersLive(),
|
|
2095
|
+
};
|
|
2096
|
+
const own = liveEntry(
|
|
2097
|
+
{ id: machine.id, label: machine.label, role: "worker", online: true },
|
|
2098
|
+
await ownLive({ list: true })
|
|
2099
|
+
);
|
|
2100
|
+
const pinned = pinnedHome();
|
|
2101
|
+
const fromHome = pinned
|
|
2102
|
+
? await callOnce(pinned, {
|
|
2103
|
+
command: LIVE_COMMAND,
|
|
2104
|
+
agent: "user",
|
|
2105
|
+
list: true,
|
|
2106
|
+
})
|
|
2107
|
+
: null;
|
|
2108
|
+
const others = fromHome?.ok
|
|
2109
|
+
? fromHome.computers.filter((computer) => computer.id !== machine.id)
|
|
2110
|
+
: siblings.map((sibling) =>
|
|
2111
|
+
liveEntry({ ...sibling, online: false }, null)
|
|
2112
|
+
);
|
|
2113
|
+
return {
|
|
2114
|
+
ok: true,
|
|
2115
|
+
here: machine.id,
|
|
2116
|
+
computers: [own, ...others],
|
|
2117
|
+
...(fromHome?.ok ? {} : { note: unreachableNote(fromHome) }),
|
|
2118
|
+
};
|
|
2119
|
+
};
|
|
2120
|
+
|
|
2121
|
+
/**
|
|
2122
|
+
* One session's latest messages and steps, read on the computer it runs on (`machine`, this one by default): through
|
|
2123
|
+
* home, which asks the worker that has it.
|
|
2124
|
+
*/
|
|
2125
|
+
const readSession = async (request) => {
|
|
2126
|
+
const target = String(request.machine || machine.id);
|
|
2127
|
+
const read = {
|
|
2128
|
+
to: request.to,
|
|
2129
|
+
sessionId: request.sessionId,
|
|
2130
|
+
folder: request.folder,
|
|
2131
|
+
limit: request.limit,
|
|
2132
|
+
};
|
|
2133
|
+
if (target === machine.id) return ownLive({ read });
|
|
2134
|
+
if (isHome()) {
|
|
2135
|
+
if (
|
|
2136
|
+
!siblings.some((sibling) => sibling.id === target && confirmed(sibling))
|
|
2137
|
+
)
|
|
2138
|
+
return { ok: false, error: "that computer is not in your swarm" };
|
|
2139
|
+
return (
|
|
2140
|
+
(await askWorker(target, { live: { read } })) ?? {
|
|
2141
|
+
ok: false,
|
|
2142
|
+
error: `${labelOf(target) ?? "That computer"} did not answer`,
|
|
2143
|
+
}
|
|
2144
|
+
);
|
|
2145
|
+
}
|
|
2146
|
+
const pinned = pinnedHome();
|
|
2147
|
+
if (!pinned) return { ok: false, error: unreachableNote(null) };
|
|
2148
|
+
return (
|
|
2149
|
+
(await callOnce(pinned, {
|
|
2150
|
+
command: LIVE_COMMAND,
|
|
2151
|
+
agent: "user",
|
|
2152
|
+
read,
|
|
2153
|
+
machine: target,
|
|
2154
|
+
})) ?? offline()
|
|
2155
|
+
);
|
|
2156
|
+
};
|
|
2157
|
+
|
|
2158
|
+
/** On home: a worker's call for every computer's sessions, or for one session read where it runs. */
|
|
2159
|
+
const answerLiveCall = async (call, from) => {
|
|
2160
|
+
if (call.agent !== "user")
|
|
2161
|
+
return { ok: false, error: "only the person reads their sessions" };
|
|
2162
|
+
if (call.read) return readSession({ ...call.read, machine: call.machine });
|
|
2163
|
+
return { ok: true, computers: await everyComputersLive(from) };
|
|
2164
|
+
};
|
|
2165
|
+
|
|
2011
2166
|
/** The person's model and effort for one agent on one computer, set on that computer. */
|
|
2012
2167
|
const setModel = async (request) => {
|
|
2013
2168
|
const pick = {
|
|
@@ -2214,6 +2369,29 @@ export function createOwnMachines({
|
|
|
2214
2369
|
return null;
|
|
2215
2370
|
};
|
|
2216
2371
|
|
|
2372
|
+
/**
|
|
2373
|
+
* On home: the other computer named for an agent's thread turn (`@codex #pop-os`) and its checkout of the thread's
|
|
2374
|
+
* repository, or why it cannot take it.
|
|
2375
|
+
*/
|
|
2376
|
+
const placeOn = (agent, workspace, id) => {
|
|
2377
|
+
const sibling = siblings.find((each) => each.id === id);
|
|
2378
|
+
const label = labelOf(id) ?? "that computer";
|
|
2379
|
+
if (!sibling || !store.ownMachine(id))
|
|
2380
|
+
return { error: `${label} is not connected` };
|
|
2381
|
+
if (!sibling.agents.includes(agent))
|
|
2382
|
+
return { error: `${nameOf(agent)} is not on ${label}` };
|
|
2383
|
+
const repository = workspace ? repositoryOf(workspace) : null;
|
|
2384
|
+
const folder = repository
|
|
2385
|
+
? infos.get(id)?.folders.find((each) => each.repository === repository)
|
|
2386
|
+
?.path
|
|
2387
|
+
: null;
|
|
2388
|
+
if (!folder)
|
|
2389
|
+
return {
|
|
2390
|
+
error: `${label} has no copy of this thread's repository: clone it there first`,
|
|
2391
|
+
};
|
|
2392
|
+
return { machine: id, folder };
|
|
2393
|
+
};
|
|
2394
|
+
|
|
2217
2395
|
/** The computers the board may name: this one and every other the person has, with their agents and repositories. */
|
|
2218
2396
|
const all = () => [
|
|
2219
2397
|
{
|
|
@@ -2229,6 +2407,25 @@ export function createOwnMachines({
|
|
|
2229
2407
|
})),
|
|
2230
2408
|
];
|
|
2231
2409
|
|
|
2410
|
+
/**
|
|
2411
|
+
* The computer a message names with # (`#pop-os`), one of the person's own by its exact name in any case; null when
|
|
2412
|
+
* it names none. Anything else after a # (an issue number, a heading) is not a computer.
|
|
2413
|
+
*/
|
|
2414
|
+
const computerNamed = (text) => {
|
|
2415
|
+
const computers = all();
|
|
2416
|
+
if (computers.length < 2) return null;
|
|
2417
|
+
for (const match of String(text || "").matchAll(
|
|
2418
|
+
/(?:^|[\s(])#([A-Za-z0-9][\w.-]*)/g
|
|
2419
|
+
)) {
|
|
2420
|
+
const name = match[1].replace(/[.,;:!?]+$/, "").toLowerCase();
|
|
2421
|
+
const found = computers.find(
|
|
2422
|
+
(computer) => computerHandle(computer.label).toLowerCase() === name
|
|
2423
|
+
);
|
|
2424
|
+
if (found) return { id: found.id, label: found.label };
|
|
2425
|
+
}
|
|
2426
|
+
return null;
|
|
2427
|
+
};
|
|
2428
|
+
|
|
2232
2429
|
/** The name beside an agent's words from that computer ("Codex (linux)"): only once the person has more than one. */
|
|
2233
2430
|
const shownLabel = (id) => (all().length > 1 ? labelOf(id) : null);
|
|
2234
2431
|
|
|
@@ -2297,12 +2494,16 @@ export function createOwnMachines({
|
|
|
2297
2494
|
directStatus: () => directLink?.status() ?? [],
|
|
2298
2495
|
list,
|
|
2299
2496
|
models,
|
|
2497
|
+
sessions,
|
|
2498
|
+
readSession,
|
|
2300
2499
|
setModel,
|
|
2301
2500
|
allowSsh,
|
|
2302
2501
|
status,
|
|
2303
2502
|
all,
|
|
2304
2503
|
learn,
|
|
2305
2504
|
placeElsewhere,
|
|
2505
|
+
placeOn,
|
|
2506
|
+
computerNamed,
|
|
2306
2507
|
computersWith,
|
|
2307
2508
|
computersFor,
|
|
2308
2509
|
labelOf,
|
package/lib/phones.mjs
CHANGED
|
@@ -719,6 +719,9 @@ export async function createPhoneDesk({
|
|
|
719
719
|
if (!own || callInput.as === "harness")
|
|
720
720
|
return { ok: false, error: `a browser cannot call ${command} here` };
|
|
721
721
|
if (command === "machines.list") return own.list();
|
|
722
|
+
// Every computer's running sessions, and one read where it runs: the person's, so the same on every device.
|
|
723
|
+
if (command === "machines.live") return own.sessions();
|
|
724
|
+
if (command === "machines.live.read") return own.readSession(callInput);
|
|
722
725
|
// Only a join code, which confirms a computer already connected to the account and waiting to join; setup's
|
|
723
726
|
// 16-character code would also release a new computer's connection, which stays the buddy's and the CLI's.
|
|
724
727
|
const code = normalizeJoinCode(callInput.code);
|
package/lib/readers.mjs
CHANGED
|
@@ -328,14 +328,21 @@ async function recording(file) {
|
|
|
328
328
|
}
|
|
329
329
|
}
|
|
330
330
|
|
|
331
|
-
const soundParts = async (file) =>
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
331
|
+
const soundParts = async (file) => {
|
|
332
|
+
const parts = [];
|
|
333
|
+
for (const window of windows(await recording(file))) {
|
|
334
|
+
parts.push({
|
|
335
|
+
modality: "audio",
|
|
336
|
+
data: encodeWav(window.samples, RATE),
|
|
337
|
+
loc: window.startMs,
|
|
338
|
+
locEnd: window.endMs,
|
|
339
|
+
embed: true,
|
|
340
|
+
});
|
|
341
|
+
// A window at a time: a long recording read in one go held up the daemon's requests for as long.
|
|
342
|
+
await new Promise((resolve) => setImmediate(resolve));
|
|
343
|
+
}
|
|
344
|
+
return parts;
|
|
345
|
+
};
|
|
339
346
|
|
|
340
347
|
/**
|
|
341
348
|
* A file's parts: [{ modality: "text" | "image" | "audio", text?, data?, loc, locEnd?, embed }]. loc is a page (PDF),
|
package/lib/sessionControl.mjs
CHANGED
|
@@ -378,8 +378,10 @@ export function createSessionControl({
|
|
|
378
378
|
waiting: said(session.agent, session.sessionId).length,
|
|
379
379
|
}));
|
|
380
380
|
// A running session no hook placed (it started before Harness was set up, or its agent has none): read from its process.
|
|
381
|
-
|
|
382
|
-
|
|
381
|
+
// One in no terminal, app or editor is a program's, not the person's: claude-mem's worker runs Claude Code that way,
|
|
382
|
+
// and left 53 running on one computer. One another agent or a Harness run started stays, in its own group.
|
|
383
|
+
return sessions.filter((session) => {
|
|
384
|
+
if (placesOf(session.agent)[session.sessionId]) return true;
|
|
383
385
|
const env = {
|
|
384
386
|
...processEnv(session.pid),
|
|
385
387
|
...(session.entrypoint
|
|
@@ -399,8 +401,8 @@ export function createSessionControl({
|
|
|
399
401
|
? { ...place, startedBy: session.startedBy }
|
|
400
402
|
: place
|
|
401
403
|
);
|
|
402
|
-
|
|
403
|
-
|
|
404
|
+
return Boolean(place) || Boolean(session.startedBy?.by);
|
|
405
|
+
});
|
|
404
406
|
};
|
|
405
407
|
|
|
406
408
|
/**
|