tryharness 0.7.64 → 0.7.66
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/atlas.mjs +87 -39
- package/lib/boardMirror.mjs +114 -6
- package/lib/callSessions.mjs +51 -0
- package/lib/context.mjs +89 -63
- package/lib/coordinator.mjs +10 -0
- package/lib/daemon.mjs +46 -3
- package/lib/directLink.mjs +47 -6
- package/lib/jsonPatch.mjs +86 -0
- package/lib/liveViews.mjs +308 -0
- package/lib/localAgentAsk.mjs +13 -1
- package/lib/machineRoutes.mjs +3 -0
- package/lib/machineRuns.mjs +28 -2
- package/lib/machineSeal.mjs +35 -6
- package/lib/models.mjs +6 -1
- package/lib/ownMachines.mjs +206 -11
- package/lib/phones.mjs +6 -0
- package/lib/scratch.mjs +39 -0
- package/lib/store.mjs +17 -7
- package/lib/t3.mjs +15 -3
- package/lib/threadActions.mjs +39 -2
- package/lib/threadRuns.mjs +17 -15
- package/npm-shrinkwrap.json +14 -14
- package/package.json +4 -4
- package/release.json +25 -21
package/lib/context.mjs
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import fs from "node:fs";
|
|
2
2
|
import path from "node:path";
|
|
3
|
-
import { spawnSync } from "node:child_process";
|
|
4
3
|
import {
|
|
4
|
+
git as gitRun,
|
|
5
5
|
openAtlas,
|
|
6
6
|
treeFiles,
|
|
7
7
|
addFiles,
|
|
@@ -45,39 +45,40 @@ const HELD_OUT_SHARE = 0.2;
|
|
|
45
45
|
const FILES_IN_PACK = 3;
|
|
46
46
|
const MIN_TRAINING_RUNS = 30;
|
|
47
47
|
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
maxBuffer: 1 << 28,
|
|
53
|
-
});
|
|
54
|
-
return out.status === 0 ? out.stdout : "";
|
|
48
|
+
// git's output, or "" when it failed; it runs beside the daemon (atlas.mjs git).
|
|
49
|
+
const git = async (cwd, args) => {
|
|
50
|
+
const out = await gitRun(cwd, args);
|
|
51
|
+
return out.status === 0 ? out.stdout.toString("utf8") : "";
|
|
55
52
|
};
|
|
56
|
-
const commitBefore = (root, at) =>
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
53
|
+
const commitBefore = async (root, at) =>
|
|
54
|
+
(
|
|
55
|
+
await git(root, [
|
|
56
|
+
"rev-list",
|
|
57
|
+
"-1",
|
|
58
|
+
`--before=${new Date(at).toISOString()}`,
|
|
59
|
+
"HEAD",
|
|
60
|
+
])
|
|
61
|
+
).trim() || null;
|
|
63
62
|
// Files changed in the two weeks before a commit, by how often; kept per commit, since a commit's history is fixed.
|
|
64
63
|
const recentByCommit = new Map();
|
|
65
|
-
export function recentChanges(root, commit = "HEAD") {
|
|
66
|
-
const sha = git(root, ["rev-parse", commit]).trim();
|
|
64
|
+
export async function recentChanges(root, commit = "HEAD") {
|
|
65
|
+
const sha = (await git(root, ["rev-parse", commit])).trim();
|
|
67
66
|
if (recentByCommit.has(sha)) return recentByCommit.get(sha);
|
|
68
67
|
const counts = new Map();
|
|
69
68
|
const since = new Date(
|
|
70
|
-
(Number(git(root, ["show", "-s", "--format=%ct", sha]).trim()) -
|
|
69
|
+
(Number((await git(root, ["show", "-s", "--format=%ct", sha])).trim()) -
|
|
71
70
|
14 * 86_400) *
|
|
72
71
|
1000
|
|
73
72
|
).toISOString();
|
|
74
|
-
for (const file of
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
73
|
+
for (const file of (
|
|
74
|
+
await git(root, [
|
|
75
|
+
"log",
|
|
76
|
+
`--since=${since}`,
|
|
77
|
+
"--name-only",
|
|
78
|
+
"--format=",
|
|
79
|
+
sha,
|
|
80
|
+
])
|
|
81
|
+
).split("\n"))
|
|
81
82
|
if (file) counts.set(file, (counts.get(file) ?? 0) + 1);
|
|
82
83
|
if (recentByCommit.size > 500) recentByCommit.clear();
|
|
83
84
|
recentByCommit.set(sha, counts);
|
|
@@ -200,14 +201,16 @@ export function createContextEngine({
|
|
|
200
201
|
};
|
|
201
202
|
|
|
202
203
|
/** The repository a folder belongs to and where its files are (a worktree is read where it is, named as its repository). */
|
|
203
|
-
const placeOf = (folder) => {
|
|
204
|
+
const placeOf = async (folder) => {
|
|
204
205
|
const repo = repositoryOf(folder);
|
|
205
206
|
if (!repo) return null;
|
|
206
|
-
const common =
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
207
|
+
const common = (
|
|
208
|
+
await git(repo.root, [
|
|
209
|
+
"rev-parse",
|
|
210
|
+
"--path-format=absolute",
|
|
211
|
+
"--git-common-dir",
|
|
212
|
+
])
|
|
213
|
+
).trim();
|
|
211
214
|
const root =
|
|
212
215
|
common && path.basename(common) === ".git"
|
|
213
216
|
? path.dirname(common)
|
|
@@ -218,32 +221,37 @@ export function createContextEngine({
|
|
|
218
221
|
// times for each of 683 sessions every 15 minutes, holding everything else (T3, the buddy, hooks) for up to 11 s.
|
|
219
222
|
// A brief's pack still asks fresh.
|
|
220
223
|
const keptPlaces = new Map();
|
|
221
|
-
const keptPlaceOf = (folder) => {
|
|
224
|
+
const keptPlaceOf = async (folder) => {
|
|
222
225
|
const kept = keptPlaces.get(folder);
|
|
223
226
|
if (kept && Date.now() - kept.at < PLACE_KEPT_MS) return kept.place;
|
|
224
|
-
const place = placeOf(folder);
|
|
227
|
+
const place = await placeOf(folder);
|
|
225
228
|
keptPlaces.set(folder, { place, at: Date.now() });
|
|
226
229
|
return place;
|
|
227
230
|
};
|
|
228
231
|
|
|
229
|
-
const roots = () =>
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
),
|
|
235
|
-
];
|
|
232
|
+
const roots = async () => {
|
|
233
|
+
const places = [];
|
|
234
|
+
for (const place of known()) places.push(await keptPlaceOf(place.path));
|
|
235
|
+
return [...new Set(places.map((place) => place?.root).filter(Boolean))];
|
|
236
|
+
};
|
|
236
237
|
|
|
238
|
+
// Each checkout's files as last indexed, for scope(), which a task frame asks for without waiting.
|
|
239
|
+
const scopes = new Map();
|
|
237
240
|
/** Brings one checkout's files into the atlas; returns its files. Pieces are embedded later on the background lane. */
|
|
238
|
-
const indexTree = (top, commit = null) => {
|
|
239
|
-
const rows = treeFiles(top, commit);
|
|
240
|
-
addFiles(atlas, top, rows);
|
|
241
|
+
const indexTree = async (top, commit = null) => {
|
|
242
|
+
const rows = await treeFiles(top, commit);
|
|
243
|
+
await addFiles(atlas, top, rows);
|
|
244
|
+
if (!commit) scopes.set(top, rows);
|
|
241
245
|
return rows;
|
|
242
246
|
};
|
|
243
247
|
|
|
248
|
+
// Sessions read once and found to hold no run to learn from (no request, or no commit before it), by the source's
|
|
249
|
+
// mtime: reading one again only finds the same, and some transcripts are over 100 MB.
|
|
250
|
+
const passedRuns = new Map();
|
|
251
|
+
|
|
244
252
|
/** Records every coding session that ran in a known repository: what it was asked and what it needed. */
|
|
245
253
|
const recordRuns = async () => {
|
|
246
|
-
const rootList = roots();
|
|
254
|
+
const rootList = await roots();
|
|
247
255
|
const since = new Date(
|
|
248
256
|
Date.now() - HISTORY_DAYS * 86_400_000
|
|
249
257
|
).toISOString();
|
|
@@ -257,7 +265,9 @@ export function createContextEngine({
|
|
|
257
265
|
} catch {}
|
|
258
266
|
// Seen as it is now: nothing to read, and no need to ask git where it ran.
|
|
259
267
|
if (atlas.runSeen(session.agent, session.session_id) === mtime) continue;
|
|
260
|
-
const
|
|
268
|
+
const key = `${session.agent}:${session.session_id}`;
|
|
269
|
+
if (passedRuns.get(key) === mtime) continue;
|
|
270
|
+
const place = await keptPlaceOf(session.cwd);
|
|
261
271
|
if (!place || !rootList.includes(place.root)) continue;
|
|
262
272
|
const trail = mtime
|
|
263
273
|
? await readTrail(session.agent, session.source)
|
|
@@ -269,19 +279,26 @@ export function createContextEngine({
|
|
|
269
279
|
startedAt: Date.parse(session.started_at) || null,
|
|
270
280
|
}
|
|
271
281
|
);
|
|
272
|
-
if (!trail?.brief || trail.startedAt === null)
|
|
273
|
-
|
|
274
|
-
|
|
282
|
+
if (!trail?.brief || trail.startedAt === null) {
|
|
283
|
+
passedRuns.set(key, mtime);
|
|
284
|
+
continue;
|
|
285
|
+
}
|
|
286
|
+
const commit = await commitBefore(place.root, trail.startedAt);
|
|
287
|
+
if (!commit) {
|
|
288
|
+
passedRuns.set(key, mtime);
|
|
289
|
+
continue;
|
|
290
|
+
}
|
|
275
291
|
if (!trees.has(commit))
|
|
276
292
|
trees.set(
|
|
277
293
|
commit,
|
|
278
|
-
new Set(treeFiles(place.root, commit).map((row) => row.path))
|
|
294
|
+
new Set((await treeFiles(place.root, commit)).map((row) => row.path))
|
|
279
295
|
);
|
|
296
|
+
const trailPlace = trail.cwd ? await keptPlaceOf(trail.cwd) : null;
|
|
280
297
|
const need = needs(
|
|
281
298
|
{
|
|
282
299
|
...trail,
|
|
283
300
|
cwd:
|
|
284
|
-
|
|
301
|
+
trailPlace?.top === place.top
|
|
285
302
|
? path.join(place.root, path.relative(place.top, trail.cwd))
|
|
286
303
|
: trail.cwd,
|
|
287
304
|
},
|
|
@@ -332,8 +349,8 @@ export function createContextEngine({
|
|
|
332
349
|
if (first && !power.onCharger())
|
|
333
350
|
return { skipped: "waiting for the charger" };
|
|
334
351
|
let files = 0;
|
|
335
|
-
for (const root of roots()) {
|
|
336
|
-
const rows = indexTree(root);
|
|
352
|
+
for (const root of await roots()) {
|
|
353
|
+
const rows = await indexTree(root);
|
|
337
354
|
atlas.setFiles(root, rows);
|
|
338
355
|
files += rows.length;
|
|
339
356
|
}
|
|
@@ -370,9 +387,10 @@ export function createContextEngine({
|
|
|
370
387
|
*/
|
|
371
388
|
const pack = async ({ brief, folder, thread = null, lane = "handoff" }) => {
|
|
372
389
|
const started = performance.now();
|
|
373
|
-
const place = folder && placeOf(folder);
|
|
390
|
+
const place = folder && (await placeOf(folder));
|
|
391
|
+
if (place) keptPlaces.set(folder, { place, at: Date.now() });
|
|
374
392
|
if (!place || !String(brief).trim()) return null;
|
|
375
|
-
const scope = indexTree(place.top);
|
|
393
|
+
const scope = await indexTree(place.top);
|
|
376
394
|
// A few changed pieces are embedded now; a repository the relay has not indexed yet is searched by words and names
|
|
377
395
|
// this time while the background pass embeds it.
|
|
378
396
|
const missing = atlas.missing(ATLAS_VERSION);
|
|
@@ -386,7 +404,7 @@ export function createContextEngine({
|
|
|
386
404
|
query: String(brief).slice(0, 4_000),
|
|
387
405
|
scope,
|
|
388
406
|
history,
|
|
389
|
-
recent: recentChanges(place.top),
|
|
407
|
+
recent: await recentChanges(place.top),
|
|
390
408
|
model,
|
|
391
409
|
lane,
|
|
392
410
|
vectors: vectorsFor(scope),
|
|
@@ -502,17 +520,17 @@ export function createContextEngine({
|
|
|
502
520
|
await refresh();
|
|
503
521
|
const report = { at: new Date().toISOString(), repositories: [] };
|
|
504
522
|
const rows = [];
|
|
505
|
-
for (const root of roots()) {
|
|
523
|
+
for (const root of await roots()) {
|
|
506
524
|
const runs = atlas
|
|
507
525
|
.runs(root, Date.now() - HISTORY_DAYS * 86_400_000)
|
|
508
526
|
.filter((run) => run.edited.length);
|
|
509
527
|
if (!runs.length) continue;
|
|
510
528
|
const snapshots = new Map();
|
|
511
529
|
for (const run of runs) {
|
|
512
|
-
const commit = commitBefore(root, run.at);
|
|
530
|
+
const commit = await commitBefore(root, run.at);
|
|
513
531
|
if (!commit) continue;
|
|
514
532
|
if (!snapshots.has(commit))
|
|
515
|
-
snapshots.set(commit, indexTree(root, commit));
|
|
533
|
+
snapshots.set(commit, await indexTree(root, commit));
|
|
516
534
|
rows.push({ root, run, commit });
|
|
517
535
|
}
|
|
518
536
|
report.repositories.push({
|
|
@@ -534,10 +552,10 @@ export function createContextEngine({
|
|
|
534
552
|
atlas,
|
|
535
553
|
inference,
|
|
536
554
|
query: run.brief,
|
|
537
|
-
scope: treeFiles(root, commit),
|
|
555
|
+
scope: await treeFiles(root, commit),
|
|
538
556
|
history,
|
|
539
557
|
at: run.at,
|
|
540
|
-
recent: recentChanges(root, commit),
|
|
558
|
+
recent: await recentChanges(root, commit),
|
|
541
559
|
model: null,
|
|
542
560
|
lane: "background",
|
|
543
561
|
});
|
|
@@ -607,10 +625,18 @@ export function createContextEngine({
|
|
|
607
625
|
|
|
608
626
|
return {
|
|
609
627
|
atlas,
|
|
610
|
-
/**
|
|
628
|
+
/**
|
|
629
|
+
* A folder's files as last indexed ([{ path, blob }]); [] outside a repository, or before its first index, which
|
|
630
|
+
* this starts. A task frame asks without waiting, and a run's pack has indexed its folder just before.
|
|
631
|
+
*/
|
|
611
632
|
scope: (folder) => {
|
|
612
|
-
const place = folder &&
|
|
613
|
-
|
|
633
|
+
const place = folder && keptPlaces.get(folder)?.place;
|
|
634
|
+
const rows = place && scopes.get(place.top);
|
|
635
|
+
if (!rows && folder)
|
|
636
|
+
void keptPlaceOf(folder)
|
|
637
|
+
.then((found) => found && indexTree(found.top))
|
|
638
|
+
.catch((error) => log(`context engine: ${error.message}`));
|
|
639
|
+
return rows ?? [];
|
|
614
640
|
},
|
|
615
641
|
pack,
|
|
616
642
|
search,
|
package/lib/coordinator.mjs
CHANGED
|
@@ -764,6 +764,16 @@ export function createCoordinator({
|
|
|
764
764
|
kind: "open_item",
|
|
765
765
|
text: String(decision.openItem).slice(0, 2_000),
|
|
766
766
|
});
|
|
767
|
+
// Harness's part waiting for a window is never a reason to pause: it starts by itself once it can, and a paused
|
|
768
|
+
// thread is skipped when it is offered again (owner, October 28: a thread paused for it stayed paused for hours
|
|
769
|
+
// after the person let Harness Relay use their endpoint). The round waits instead; the thread already says why.
|
|
770
|
+
if (
|
|
771
|
+
decision.action === "pause" &&
|
|
772
|
+
store.threadRuns(thread.id).some(waitsForWindow)
|
|
773
|
+
) {
|
|
774
|
+
notifyBoard(store.getTask(thread.id));
|
|
775
|
+
return;
|
|
776
|
+
}
|
|
767
777
|
if (decision.action === "pause") {
|
|
768
778
|
pauseGoal(
|
|
769
779
|
thread.id,
|
package/lib/daemon.mjs
CHANGED
|
@@ -17,6 +17,11 @@ import { underperforming } from "./record.mjs";
|
|
|
17
17
|
import { createSubagents } from "./subagents.mjs";
|
|
18
18
|
import { createEnvironments } from "./environments.mjs";
|
|
19
19
|
import { createGuestRequests } from "./guestRequests.mjs";
|
|
20
|
+
import {
|
|
21
|
+
createLiveViews,
|
|
22
|
+
createViewCopies,
|
|
23
|
+
viewRequest,
|
|
24
|
+
} from "./liveViews.mjs";
|
|
20
25
|
import { createBoardWaits } from "./boardWaits.mjs";
|
|
21
26
|
import { createCoordinator } from "./coordinator.mjs";
|
|
22
27
|
import { createServerWords } from "./serverWords.mjs";
|
|
@@ -233,6 +238,7 @@ export async function startDaemon() {
|
|
|
233
238
|
// Which of the person's computers this is; ownMachines (created once this computer's key is read) keeps it.
|
|
234
239
|
const machine = loadMachine(paths.machine);
|
|
235
240
|
let ownMachines = null;
|
|
241
|
+
let viewCopies = null;
|
|
236
242
|
// Where an agent calling from a computer is, and the name beside its words ("Codex (linux)") once the person has more
|
|
237
243
|
// than one computer (ownMachines.mjs); before ownMachines starts, this computer alone.
|
|
238
244
|
const whereIs = (id) =>
|
|
@@ -262,6 +268,8 @@ export async function startDaemon() {
|
|
|
262
268
|
after,
|
|
263
269
|
feed,
|
|
264
270
|
standIn,
|
|
271
|
+
// This computer applies a row's changed columns (boardMirror.mjs changedSince).
|
|
272
|
+
diffs: true,
|
|
265
273
|
})) ?? null,
|
|
266
274
|
pushPage: async (page, { bring = false } = {}) =>
|
|
267
275
|
(await ownMachines?.mirrorCall({
|
|
@@ -1106,6 +1114,7 @@ export async function startDaemon() {
|
|
|
1106
1114
|
homeOf: (...args) => homeOf(...args),
|
|
1107
1115
|
placeElsewhere: (agent, workspace) =>
|
|
1108
1116
|
ownMachines?.placeElsewhere(agent, workspace) ?? null,
|
|
1117
|
+
threadFiles: (thread) => boardFiles.threadFiles(thread).files,
|
|
1109
1118
|
// Read when a turn starts: the person's computers are known once ownMachines is made, after this.
|
|
1110
1119
|
computers: {
|
|
1111
1120
|
self: () => ownMachines?.self() ?? null,
|
|
@@ -2076,13 +2085,22 @@ export async function startDaemon() {
|
|
|
2076
2085
|
return result;
|
|
2077
2086
|
}
|
|
2078
2087
|
return {
|
|
2079
|
-
pending:
|
|
2088
|
+
pending: (async () => {
|
|
2089
|
+
// A read this computer keeps a copy of is answered here (liveViews.mjs), and asks home for nothing more.
|
|
2090
|
+
const kept = await viewCopies?.read(request);
|
|
2091
|
+
if (kept) return kept;
|
|
2092
|
+
const answer = await ownMachines.callHome(request);
|
|
2080
2093
|
if (answer?.code === "home_offline" && readsFromCopy(request.command))
|
|
2081
2094
|
return fromCopy(request, answer);
|
|
2082
|
-
|
|
2095
|
+
// A write that reached home brings this computer's copy of the board up to date; a read changes nothing.
|
|
2096
|
+
if (
|
|
2097
|
+
answer?.ok !== false &&
|
|
2098
|
+
!readsFromCopy(request.command) &&
|
|
2099
|
+
!viewRequest(request)
|
|
2100
|
+
)
|
|
2083
2101
|
void boardMirror.sync();
|
|
2084
2102
|
return answer;
|
|
2085
|
-
}),
|
|
2103
|
+
})(),
|
|
2086
2104
|
};
|
|
2087
2105
|
};
|
|
2088
2106
|
const fromCopy = async (request, offline) => {
|
|
@@ -2302,6 +2320,7 @@ export async function startDaemon() {
|
|
|
2302
2320
|
};
|
|
2303
2321
|
const phones = await createPhoneDesk({
|
|
2304
2322
|
store,
|
|
2323
|
+
log,
|
|
2305
2324
|
identityFile: paths.phoneIdentity,
|
|
2306
2325
|
setupOfferFile: paths.browserOffer,
|
|
2307
2326
|
// A window linked to a worker works the swarm's board, which is home's, not one of its own.
|
|
@@ -2379,7 +2398,21 @@ export async function startDaemon() {
|
|
|
2379
2398
|
onRequest: (peerId, body) =>
|
|
2380
2399
|
ownMachines?.fromLink(peerId, body) ?? { ok: false },
|
|
2381
2400
|
});
|
|
2401
|
+
// The board as another computer shows it, answered there (liveViews.mjs): on home, the views it keeps for them; on
|
|
2402
|
+
// a worker, its copies, which a waiting surface here hears move.
|
|
2403
|
+
const liveViews = createLiveViews({
|
|
2404
|
+
answer: (workerId, request) => ownMachines.answerAs(workerId, request),
|
|
2405
|
+
push: (workerId, body) => ownMachines.pushViews(workerId, body),
|
|
2406
|
+
onMove: (listener) => boardWaits.onChange(listener),
|
|
2407
|
+
log,
|
|
2408
|
+
});
|
|
2409
|
+
viewCopies = createViewCopies({
|
|
2410
|
+
callHome: (request) => ownMachines.callHome(request),
|
|
2411
|
+
changed: () => boardWaits.bump(),
|
|
2412
|
+
});
|
|
2382
2413
|
ownMachines = createOwnMachines({
|
|
2414
|
+
liveViews: () => liveViews,
|
|
2415
|
+
viewCopies: () => viewCopies,
|
|
2383
2416
|
directLink,
|
|
2384
2417
|
store,
|
|
2385
2418
|
log,
|
|
@@ -2394,6 +2427,7 @@ export async function startDaemon() {
|
|
|
2394
2427
|
folders: () => machineFolders(),
|
|
2395
2428
|
repositoryOf: (folder) => repositoryName(folder),
|
|
2396
2429
|
homeBack: () => {
|
|
2430
|
+
viewCopies.reset();
|
|
2397
2431
|
void machineRuns.check();
|
|
2398
2432
|
void boardMirror.sync();
|
|
2399
2433
|
void swarmDevices.report();
|
|
@@ -2445,6 +2479,15 @@ export async function startDaemon() {
|
|
|
2445
2479
|
agents: installedAgents,
|
|
2446
2480
|
// Only in a folder this computer's own agents' sessions or T3 Code worked in, never just one home names.
|
|
2447
2481
|
knownFolders: () => folders.known({ own: true }).map((place) => place.path),
|
|
2482
|
+
// A folder-less thread's files, as this computer's copy of the board lists them (boardMirror.mjs).
|
|
2483
|
+
threadFileIds: (threadId) => {
|
|
2484
|
+
const thread = store.getTask(threadId);
|
|
2485
|
+
return thread
|
|
2486
|
+
? [thread, ...store.children(thread.id)].flatMap((task) =>
|
|
2487
|
+
store.taskFiles(task.id).map((file) => file.id)
|
|
2488
|
+
)
|
|
2489
|
+
: [];
|
|
2490
|
+
},
|
|
2448
2491
|
log,
|
|
2449
2492
|
modelFor: modelForRun,
|
|
2450
2493
|
});
|
package/lib/directLink.mjs
CHANGED
|
@@ -27,6 +27,11 @@ const bootstrapOf = (value) =>
|
|
|
27
27
|
: undefined;
|
|
28
28
|
// A frame larger than this is not one the other computer sent (a sealed message is at most 1.4 MB).
|
|
29
29
|
const MAX_FRAME_BYTES = 4 * 1024 * 1024;
|
|
30
|
+
// How long a new link has to answer one round trip before it is dropped and dialed again (owner's value, October 28).
|
|
31
|
+
// Measured: 2 to 11 ms over 7 links in the direct-link smoke (one Mac, a local HyperDHT network), about 45 ms over the
|
|
32
|
+
// internet in the October 21 trial (11 hole-punched links). A half-open first link stayed silent until HyperDHT dropped
|
|
33
|
+
// it about 24 s later.
|
|
34
|
+
const HELLO_MS = 2_000;
|
|
30
35
|
|
|
31
36
|
export function createDirectLink({
|
|
32
37
|
keyFile,
|
|
@@ -69,6 +74,8 @@ export function createDirectLink({
|
|
|
69
74
|
}
|
|
70
75
|
};
|
|
71
76
|
|
|
77
|
+
const peerIdOf = (peer) =>
|
|
78
|
+
[...peers.entries()].find(([, each]) => each === peer)?.[0] ?? null;
|
|
72
79
|
const peerOfKey = (hex) =>
|
|
73
80
|
[...peers.entries()].find(([, peer]) => peer.key === hex)?.[0] ?? null;
|
|
74
81
|
|
|
@@ -119,11 +126,12 @@ export function createDirectLink({
|
|
|
119
126
|
|
|
120
127
|
// One link: newline-framed JSON, requests and answers matched by number. The peer is looked up when something
|
|
121
128
|
// happens, never kept: a key swap updates it in place (a copied peer once kept a dead link "linked").
|
|
122
|
-
const attach = (id, socket) => {
|
|
129
|
+
const attach = (id, socket, { verified = true } = {}) => {
|
|
123
130
|
const peer = peers.get(id);
|
|
124
131
|
if (!peer) return socket.destroy();
|
|
125
132
|
const before = peer.socket;
|
|
126
133
|
peer.socket = socket;
|
|
134
|
+
peer.verified = verified;
|
|
127
135
|
peer.heardAt = Date.now();
|
|
128
136
|
if (before && before !== socket) {
|
|
129
137
|
release(peer, before);
|
|
@@ -240,10 +248,40 @@ export function createDirectLink({
|
|
|
240
248
|
socket.once("error", failed);
|
|
241
249
|
socket.once("open", () => {
|
|
242
250
|
socket.off("error", failed);
|
|
243
|
-
attach(id, socket);
|
|
251
|
+
attach(id, socket, { verified: false });
|
|
244
252
|
resolve();
|
|
245
253
|
});
|
|
246
254
|
});
|
|
255
|
+
// Open on this side is not yet a link: the other computer sometimes never saw the connection, and every call
|
|
256
|
+
// over it waited out the whole timeout before going through Harness (smoke direct-link, one run in two). It counts
|
|
257
|
+
// once the other computer answers over it; until then calls go through Harness.
|
|
258
|
+
const current = peers.get(id);
|
|
259
|
+
if (current?.socket && !current.verified) {
|
|
260
|
+
const socket = current.socket;
|
|
261
|
+
const openedAt = Date.now();
|
|
262
|
+
const heard = await send(current, socket, { kind: "hello" }, HELLO_MS);
|
|
263
|
+
const still = peers.get(id);
|
|
264
|
+
// Gone, or a newer link took its place; a link the timeout already dropped is still dialed again below.
|
|
265
|
+
if (!still || (still.socket && still.socket !== socket)) return;
|
|
266
|
+
if (heard === null) {
|
|
267
|
+
log(
|
|
268
|
+
`the direct link with ${id} did not answer; going through Harness and dialing again`
|
|
269
|
+
);
|
|
270
|
+
release(still, socket);
|
|
271
|
+
still.socket = null;
|
|
272
|
+
socket.destroy();
|
|
273
|
+
// A half-open link (this side open, the other never saw it) is dialed again at once rather than when the next
|
|
274
|
+
// call needs it.
|
|
275
|
+
still.redials = (still.redials ?? 0) + 1;
|
|
276
|
+
if (still.redials <= 3) setImmediate(() => void dial(id));
|
|
277
|
+
} else {
|
|
278
|
+
still.verified = true;
|
|
279
|
+
still.redials = 0;
|
|
280
|
+
log(
|
|
281
|
+
`the direct link with ${id} answered in ${Date.now() - openedAt} ms`
|
|
282
|
+
);
|
|
283
|
+
}
|
|
284
|
+
}
|
|
247
285
|
})().finally(() => {
|
|
248
286
|
peer.dialing = null;
|
|
249
287
|
});
|
|
@@ -253,11 +291,14 @@ export function createDirectLink({
|
|
|
253
291
|
/** A request to that computer over the link; null when there is none or it did not answer in time. */
|
|
254
292
|
const request = (id, body) => {
|
|
255
293
|
const peer = peers.get(id);
|
|
256
|
-
if (!peer?.socket) {
|
|
294
|
+
if (!peer?.socket || !peer.verified) {
|
|
257
295
|
if (peer?.dialer) void dial(id);
|
|
258
296
|
return Promise.resolve(null);
|
|
259
297
|
}
|
|
260
|
-
|
|
298
|
+
return send(peer, peer.socket, body);
|
|
299
|
+
};
|
|
300
|
+
const send = (peer, socket, body, waitMs = timeoutMs) => {
|
|
301
|
+
const id = peerIdOf(peer);
|
|
261
302
|
const i = ++sequence;
|
|
262
303
|
const sentAt = Date.now();
|
|
263
304
|
return new Promise((resolve) => {
|
|
@@ -270,7 +311,7 @@ export function createDirectLink({
|
|
|
270
311
|
socket.destroy();
|
|
271
312
|
}
|
|
272
313
|
resolve(null);
|
|
273
|
-
},
|
|
314
|
+
}, waitMs);
|
|
274
315
|
peer.pending.set(i, { resolve, timer, socket });
|
|
275
316
|
socket.write(JSON.stringify({ i, q: body }) + "\n");
|
|
276
317
|
});
|
|
@@ -280,7 +321,7 @@ export function createDirectLink({
|
|
|
280
321
|
const status = () =>
|
|
281
322
|
[...peers.entries()].map(([id, peer]) => ({
|
|
282
323
|
id,
|
|
283
|
-
linked: Boolean(peer.socket),
|
|
324
|
+
linked: Boolean(peer.socket && peer.verified),
|
|
284
325
|
at: peer.socket?.rawStream?.remoteHost ?? null,
|
|
285
326
|
}));
|
|
286
327
|
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
// What changed between two answers of the same read, small enough to send as it is, and that change applied to the
|
|
2
|
+
// earlier answer to give the later one exactly (liveViews.mjs: home pushes each view's change to the computers that
|
|
3
|
+
// read it; the thread list alone was 362 KB a read, mostly the same each time).
|
|
4
|
+
//
|
|
5
|
+
// A change is one of:
|
|
6
|
+
// - { v: value }: the whole value, for a value of another kind, a list without ids, or anything short;
|
|
7
|
+
// - { o: { key: change }, x: [key] }: an object, with the changes to its keys and the keys it lost;
|
|
8
|
+
// - { l: [id], c: { id: change } }: a list whose items each have their own string id, in their new order, with the
|
|
9
|
+
// changes to the items that changed or came new (a new one as { v: item }).
|
|
10
|
+
// Two equal values have no change (undefined).
|
|
11
|
+
|
|
12
|
+
const isObject = (value) =>
|
|
13
|
+
value !== null && typeof value === "object" && !Array.isArray(value);
|
|
14
|
+
|
|
15
|
+
// A list whose every item is an object with its own string id, none twice.
|
|
16
|
+
const idsOf = (list) => {
|
|
17
|
+
if (!Array.isArray(list) || !list.length) return null;
|
|
18
|
+
const ids = [];
|
|
19
|
+
const seen = new Set();
|
|
20
|
+
for (const item of list) {
|
|
21
|
+
if (!isObject(item) || typeof item.id !== "string" || seen.has(item.id))
|
|
22
|
+
return null;
|
|
23
|
+
seen.add(item.id);
|
|
24
|
+
ids.push(item.id);
|
|
25
|
+
}
|
|
26
|
+
return ids;
|
|
27
|
+
};
|
|
28
|
+
|
|
29
|
+
const same = (before, after) =>
|
|
30
|
+
before === after || JSON.stringify(before) === JSON.stringify(after);
|
|
31
|
+
|
|
32
|
+
/** The change from `before` to `after`, or undefined when there is none. */
|
|
33
|
+
export function diff(before, after) {
|
|
34
|
+
if (same(before, after)) return undefined;
|
|
35
|
+
if (isObject(before) && isObject(after)) {
|
|
36
|
+
const o = {};
|
|
37
|
+
for (const [key, value] of Object.entries(after)) {
|
|
38
|
+
const change = diff(before[key], value);
|
|
39
|
+
if (change !== undefined) o[key] = change;
|
|
40
|
+
}
|
|
41
|
+
const x = Object.keys(before).filter((key) => !(key in after));
|
|
42
|
+
return { o, ...(x.length ? { x } : {}) };
|
|
43
|
+
}
|
|
44
|
+
const beforeIds = idsOf(before);
|
|
45
|
+
const afterIds = idsOf(after);
|
|
46
|
+
if (beforeIds && afterIds) {
|
|
47
|
+
const was = new Map(before.map((item) => [item.id, item]));
|
|
48
|
+
const c = {};
|
|
49
|
+
for (const item of after) {
|
|
50
|
+
const change = was.has(item.id)
|
|
51
|
+
? diff(was.get(item.id), item)
|
|
52
|
+
: { v: item };
|
|
53
|
+
if (change !== undefined) c[item.id] = change;
|
|
54
|
+
}
|
|
55
|
+
return { l: afterIds, c };
|
|
56
|
+
}
|
|
57
|
+
return { v: after };
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/** `before` with `change` applied: the later answer. Throws on a change that does not fit `before`. */
|
|
61
|
+
export function apply(before, change) {
|
|
62
|
+
if (change === undefined) return before;
|
|
63
|
+
if ("v" in change) return change.v;
|
|
64
|
+
if (change.o) {
|
|
65
|
+
if (!isObject(before))
|
|
66
|
+
throw new Error("an object's change on a non-object");
|
|
67
|
+
const after = { ...before };
|
|
68
|
+
for (const key of change.x ?? []) delete after[key];
|
|
69
|
+
for (const [key, sub] of Object.entries(change.o))
|
|
70
|
+
after[key] = apply(before[key], sub);
|
|
71
|
+
return after;
|
|
72
|
+
}
|
|
73
|
+
if (change.l) {
|
|
74
|
+
const was = new Map(
|
|
75
|
+
(Array.isArray(before) ? before : []).map((item) => [item?.id, item])
|
|
76
|
+
);
|
|
77
|
+
return change.l.map((id) => {
|
|
78
|
+
const sub = change.c?.[id];
|
|
79
|
+
if (sub !== undefined) return apply(was.get(id), sub);
|
|
80
|
+
if (!was.has(id))
|
|
81
|
+
throw new Error(`a list's change names ${id}, not in it`);
|
|
82
|
+
return was.get(id);
|
|
83
|
+
});
|
|
84
|
+
}
|
|
85
|
+
throw new Error("not a change");
|
|
86
|
+
}
|