tryharness 0.7.64 → 0.7.67
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cli.mjs +12 -36
- package/lib/atlas.mjs +87 -39
- package/lib/boardMirror.mjs +114 -6
- package/lib/callSessions.mjs +51 -0
- package/lib/connection.mjs +39 -0
- package/lib/context.mjs +89 -63
- package/lib/coordinator.mjs +10 -0
- package/lib/daemon.mjs +46 -3
- package/lib/directLink.mjs +74 -19
- package/lib/jsonPatch.mjs +86 -0
- package/lib/liveViews.mjs +308 -0
- package/lib/localAgentAsk.mjs +13 -1
- package/lib/machineRoutes.mjs +3 -0
- package/lib/machineRuns.mjs +28 -2
- package/lib/machineSeal.mjs +35 -6
- package/lib/models.mjs +6 -1
- package/lib/ownMachines.mjs +247 -11
- package/lib/phones.mjs +6 -0
- package/lib/scratch.mjs +39 -0
- package/lib/store.mjs +19 -9
- package/lib/swarm.mjs +3 -1
- package/lib/t3.mjs +15 -3
- package/lib/threadActions.mjs +39 -2
- package/lib/threadRuns.mjs +17 -15
- package/lib/threadView.mjs +3 -1
- package/npm-shrinkwrap.json +14 -14
- package/package.json +4 -4
- package/release.json +29 -24
package/lib/context.mjs
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import fs from "node:fs";
|
|
2
2
|
import path from "node:path";
|
|
3
|
-
import { spawnSync } from "node:child_process";
|
|
4
3
|
import {
|
|
4
|
+
git as gitRun,
|
|
5
5
|
openAtlas,
|
|
6
6
|
treeFiles,
|
|
7
7
|
addFiles,
|
|
@@ -45,39 +45,40 @@ const HELD_OUT_SHARE = 0.2;
|
|
|
45
45
|
const FILES_IN_PACK = 3;
|
|
46
46
|
const MIN_TRAINING_RUNS = 30;
|
|
47
47
|
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
maxBuffer: 1 << 28,
|
|
53
|
-
});
|
|
54
|
-
return out.status === 0 ? out.stdout : "";
|
|
48
|
+
// git's output, or "" when it failed; it runs beside the daemon (atlas.mjs git).
|
|
49
|
+
const git = async (cwd, args) => {
|
|
50
|
+
const out = await gitRun(cwd, args);
|
|
51
|
+
return out.status === 0 ? out.stdout.toString("utf8") : "";
|
|
55
52
|
};
|
|
56
|
-
const commitBefore = (root, at) =>
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
53
|
+
const commitBefore = async (root, at) =>
|
|
54
|
+
(
|
|
55
|
+
await git(root, [
|
|
56
|
+
"rev-list",
|
|
57
|
+
"-1",
|
|
58
|
+
`--before=${new Date(at).toISOString()}`,
|
|
59
|
+
"HEAD",
|
|
60
|
+
])
|
|
61
|
+
).trim() || null;
|
|
63
62
|
// Files changed in the two weeks before a commit, by how often; kept per commit, since a commit's history is fixed.
|
|
64
63
|
const recentByCommit = new Map();
|
|
65
|
-
export function recentChanges(root, commit = "HEAD") {
|
|
66
|
-
const sha = git(root, ["rev-parse", commit]).trim();
|
|
64
|
+
export async function recentChanges(root, commit = "HEAD") {
|
|
65
|
+
const sha = (await git(root, ["rev-parse", commit])).trim();
|
|
67
66
|
if (recentByCommit.has(sha)) return recentByCommit.get(sha);
|
|
68
67
|
const counts = new Map();
|
|
69
68
|
const since = new Date(
|
|
70
|
-
(Number(git(root, ["show", "-s", "--format=%ct", sha]).trim()) -
|
|
69
|
+
(Number((await git(root, ["show", "-s", "--format=%ct", sha])).trim()) -
|
|
71
70
|
14 * 86_400) *
|
|
72
71
|
1000
|
|
73
72
|
).toISOString();
|
|
74
|
-
for (const file of
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
73
|
+
for (const file of (
|
|
74
|
+
await git(root, [
|
|
75
|
+
"log",
|
|
76
|
+
`--since=${since}`,
|
|
77
|
+
"--name-only",
|
|
78
|
+
"--format=",
|
|
79
|
+
sha,
|
|
80
|
+
])
|
|
81
|
+
).split("\n"))
|
|
81
82
|
if (file) counts.set(file, (counts.get(file) ?? 0) + 1);
|
|
82
83
|
if (recentByCommit.size > 500) recentByCommit.clear();
|
|
83
84
|
recentByCommit.set(sha, counts);
|
|
@@ -200,14 +201,16 @@ export function createContextEngine({
|
|
|
200
201
|
};
|
|
201
202
|
|
|
202
203
|
/** The repository a folder belongs to and where its files are (a worktree is read where it is, named as its repository). */
|
|
203
|
-
const placeOf = (folder) => {
|
|
204
|
+
const placeOf = async (folder) => {
|
|
204
205
|
const repo = repositoryOf(folder);
|
|
205
206
|
if (!repo) return null;
|
|
206
|
-
const common =
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
207
|
+
const common = (
|
|
208
|
+
await git(repo.root, [
|
|
209
|
+
"rev-parse",
|
|
210
|
+
"--path-format=absolute",
|
|
211
|
+
"--git-common-dir",
|
|
212
|
+
])
|
|
213
|
+
).trim();
|
|
211
214
|
const root =
|
|
212
215
|
common && path.basename(common) === ".git"
|
|
213
216
|
? path.dirname(common)
|
|
@@ -218,32 +221,37 @@ export function createContextEngine({
|
|
|
218
221
|
// times for each of 683 sessions every 15 minutes, holding everything else (T3, the buddy, hooks) for up to 11 s.
|
|
219
222
|
// A brief's pack still asks fresh.
|
|
220
223
|
const keptPlaces = new Map();
|
|
221
|
-
const keptPlaceOf = (folder) => {
|
|
224
|
+
const keptPlaceOf = async (folder) => {
|
|
222
225
|
const kept = keptPlaces.get(folder);
|
|
223
226
|
if (kept && Date.now() - kept.at < PLACE_KEPT_MS) return kept.place;
|
|
224
|
-
const place = placeOf(folder);
|
|
227
|
+
const place = await placeOf(folder);
|
|
225
228
|
keptPlaces.set(folder, { place, at: Date.now() });
|
|
226
229
|
return place;
|
|
227
230
|
};
|
|
228
231
|
|
|
229
|
-
const roots = () =>
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
),
|
|
235
|
-
];
|
|
232
|
+
const roots = async () => {
|
|
233
|
+
const places = [];
|
|
234
|
+
for (const place of known()) places.push(await keptPlaceOf(place.path));
|
|
235
|
+
return [...new Set(places.map((place) => place?.root).filter(Boolean))];
|
|
236
|
+
};
|
|
236
237
|
|
|
238
|
+
// Each checkout's files as last indexed, for scope(), which a task frame asks for without waiting.
|
|
239
|
+
const scopes = new Map();
|
|
237
240
|
/** Brings one checkout's files into the atlas; returns its files. Pieces are embedded later on the background lane. */
|
|
238
|
-
const indexTree = (top, commit = null) => {
|
|
239
|
-
const rows = treeFiles(top, commit);
|
|
240
|
-
addFiles(atlas, top, rows);
|
|
241
|
+
const indexTree = async (top, commit = null) => {
|
|
242
|
+
const rows = await treeFiles(top, commit);
|
|
243
|
+
await addFiles(atlas, top, rows);
|
|
244
|
+
if (!commit) scopes.set(top, rows);
|
|
241
245
|
return rows;
|
|
242
246
|
};
|
|
243
247
|
|
|
248
|
+
// Sessions read once and found to hold no run to learn from (no request, or no commit before it), by the source's
|
|
249
|
+
// mtime: reading one again only finds the same, and some transcripts are over 100 MB.
|
|
250
|
+
const passedRuns = new Map();
|
|
251
|
+
|
|
244
252
|
/** Records every coding session that ran in a known repository: what it was asked and what it needed. */
|
|
245
253
|
const recordRuns = async () => {
|
|
246
|
-
const rootList = roots();
|
|
254
|
+
const rootList = await roots();
|
|
247
255
|
const since = new Date(
|
|
248
256
|
Date.now() - HISTORY_DAYS * 86_400_000
|
|
249
257
|
).toISOString();
|
|
@@ -257,7 +265,9 @@ export function createContextEngine({
|
|
|
257
265
|
} catch {}
|
|
258
266
|
// Seen as it is now: nothing to read, and no need to ask git where it ran.
|
|
259
267
|
if (atlas.runSeen(session.agent, session.session_id) === mtime) continue;
|
|
260
|
-
const
|
|
268
|
+
const key = `${session.agent}:${session.session_id}`;
|
|
269
|
+
if (passedRuns.get(key) === mtime) continue;
|
|
270
|
+
const place = await keptPlaceOf(session.cwd);
|
|
261
271
|
if (!place || !rootList.includes(place.root)) continue;
|
|
262
272
|
const trail = mtime
|
|
263
273
|
? await readTrail(session.agent, session.source)
|
|
@@ -269,19 +279,26 @@ export function createContextEngine({
|
|
|
269
279
|
startedAt: Date.parse(session.started_at) || null,
|
|
270
280
|
}
|
|
271
281
|
);
|
|
272
|
-
if (!trail?.brief || trail.startedAt === null)
|
|
273
|
-
|
|
274
|
-
|
|
282
|
+
if (!trail?.brief || trail.startedAt === null) {
|
|
283
|
+
passedRuns.set(key, mtime);
|
|
284
|
+
continue;
|
|
285
|
+
}
|
|
286
|
+
const commit = await commitBefore(place.root, trail.startedAt);
|
|
287
|
+
if (!commit) {
|
|
288
|
+
passedRuns.set(key, mtime);
|
|
289
|
+
continue;
|
|
290
|
+
}
|
|
275
291
|
if (!trees.has(commit))
|
|
276
292
|
trees.set(
|
|
277
293
|
commit,
|
|
278
|
-
new Set(treeFiles(place.root, commit).map((row) => row.path))
|
|
294
|
+
new Set((await treeFiles(place.root, commit)).map((row) => row.path))
|
|
279
295
|
);
|
|
296
|
+
const trailPlace = trail.cwd ? await keptPlaceOf(trail.cwd) : null;
|
|
280
297
|
const need = needs(
|
|
281
298
|
{
|
|
282
299
|
...trail,
|
|
283
300
|
cwd:
|
|
284
|
-
|
|
301
|
+
trailPlace?.top === place.top
|
|
285
302
|
? path.join(place.root, path.relative(place.top, trail.cwd))
|
|
286
303
|
: trail.cwd,
|
|
287
304
|
},
|
|
@@ -332,8 +349,8 @@ export function createContextEngine({
|
|
|
332
349
|
if (first && !power.onCharger())
|
|
333
350
|
return { skipped: "waiting for the charger" };
|
|
334
351
|
let files = 0;
|
|
335
|
-
for (const root of roots()) {
|
|
336
|
-
const rows = indexTree(root);
|
|
352
|
+
for (const root of await roots()) {
|
|
353
|
+
const rows = await indexTree(root);
|
|
337
354
|
atlas.setFiles(root, rows);
|
|
338
355
|
files += rows.length;
|
|
339
356
|
}
|
|
@@ -370,9 +387,10 @@ export function createContextEngine({
|
|
|
370
387
|
*/
|
|
371
388
|
const pack = async ({ brief, folder, thread = null, lane = "handoff" }) => {
|
|
372
389
|
const started = performance.now();
|
|
373
|
-
const place = folder && placeOf(folder);
|
|
390
|
+
const place = folder && (await placeOf(folder));
|
|
391
|
+
if (place) keptPlaces.set(folder, { place, at: Date.now() });
|
|
374
392
|
if (!place || !String(brief).trim()) return null;
|
|
375
|
-
const scope = indexTree(place.top);
|
|
393
|
+
const scope = await indexTree(place.top);
|
|
376
394
|
// A few changed pieces are embedded now; a repository the relay has not indexed yet is searched by words and names
|
|
377
395
|
// this time while the background pass embeds it.
|
|
378
396
|
const missing = atlas.missing(ATLAS_VERSION);
|
|
@@ -386,7 +404,7 @@ export function createContextEngine({
|
|
|
386
404
|
query: String(brief).slice(0, 4_000),
|
|
387
405
|
scope,
|
|
388
406
|
history,
|
|
389
|
-
recent: recentChanges(place.top),
|
|
407
|
+
recent: await recentChanges(place.top),
|
|
390
408
|
model,
|
|
391
409
|
lane,
|
|
392
410
|
vectors: vectorsFor(scope),
|
|
@@ -502,17 +520,17 @@ export function createContextEngine({
|
|
|
502
520
|
await refresh();
|
|
503
521
|
const report = { at: new Date().toISOString(), repositories: [] };
|
|
504
522
|
const rows = [];
|
|
505
|
-
for (const root of roots()) {
|
|
523
|
+
for (const root of await roots()) {
|
|
506
524
|
const runs = atlas
|
|
507
525
|
.runs(root, Date.now() - HISTORY_DAYS * 86_400_000)
|
|
508
526
|
.filter((run) => run.edited.length);
|
|
509
527
|
if (!runs.length) continue;
|
|
510
528
|
const snapshots = new Map();
|
|
511
529
|
for (const run of runs) {
|
|
512
|
-
const commit = commitBefore(root, run.at);
|
|
530
|
+
const commit = await commitBefore(root, run.at);
|
|
513
531
|
if (!commit) continue;
|
|
514
532
|
if (!snapshots.has(commit))
|
|
515
|
-
snapshots.set(commit, indexTree(root, commit));
|
|
533
|
+
snapshots.set(commit, await indexTree(root, commit));
|
|
516
534
|
rows.push({ root, run, commit });
|
|
517
535
|
}
|
|
518
536
|
report.repositories.push({
|
|
@@ -534,10 +552,10 @@ export function createContextEngine({
|
|
|
534
552
|
atlas,
|
|
535
553
|
inference,
|
|
536
554
|
query: run.brief,
|
|
537
|
-
scope: treeFiles(root, commit),
|
|
555
|
+
scope: await treeFiles(root, commit),
|
|
538
556
|
history,
|
|
539
557
|
at: run.at,
|
|
540
|
-
recent: recentChanges(root, commit),
|
|
558
|
+
recent: await recentChanges(root, commit),
|
|
541
559
|
model: null,
|
|
542
560
|
lane: "background",
|
|
543
561
|
});
|
|
@@ -607,10 +625,18 @@ export function createContextEngine({
|
|
|
607
625
|
|
|
608
626
|
return {
|
|
609
627
|
atlas,
|
|
610
|
-
/**
|
|
628
|
+
/**
|
|
629
|
+
* A folder's files as last indexed ([{ path, blob }]); [] outside a repository, or before its first index, which
|
|
630
|
+
* this starts. A task frame asks without waiting, and a run's pack has indexed its folder just before.
|
|
631
|
+
*/
|
|
611
632
|
scope: (folder) => {
|
|
612
|
-
const place = folder &&
|
|
613
|
-
|
|
633
|
+
const place = folder && keptPlaces.get(folder)?.place;
|
|
634
|
+
const rows = place && scopes.get(place.top);
|
|
635
|
+
if (!rows && folder)
|
|
636
|
+
void keptPlaceOf(folder)
|
|
637
|
+
.then((found) => found && indexTree(found.top))
|
|
638
|
+
.catch((error) => log(`context engine: ${error.message}`));
|
|
639
|
+
return rows ?? [];
|
|
614
640
|
},
|
|
615
641
|
pack,
|
|
616
642
|
search,
|
package/lib/coordinator.mjs
CHANGED
|
@@ -764,6 +764,16 @@ export function createCoordinator({
|
|
|
764
764
|
kind: "open_item",
|
|
765
765
|
text: String(decision.openItem).slice(0, 2_000),
|
|
766
766
|
});
|
|
767
|
+
// Harness's part waiting for a window is never a reason to pause: it starts by itself once it can, and a paused
|
|
768
|
+
// thread is skipped when it is offered again (owner, October 28: a thread paused for it stayed paused for hours
|
|
769
|
+
// after the person let Harness Relay use their endpoint). The round waits instead; the thread already says why.
|
|
770
|
+
if (
|
|
771
|
+
decision.action === "pause" &&
|
|
772
|
+
store.threadRuns(thread.id).some(waitsForWindow)
|
|
773
|
+
) {
|
|
774
|
+
notifyBoard(store.getTask(thread.id));
|
|
775
|
+
return;
|
|
776
|
+
}
|
|
767
777
|
if (decision.action === "pause") {
|
|
768
778
|
pauseGoal(
|
|
769
779
|
thread.id,
|
package/lib/daemon.mjs
CHANGED
|
@@ -17,6 +17,11 @@ import { underperforming } from "./record.mjs";
|
|
|
17
17
|
import { createSubagents } from "./subagents.mjs";
|
|
18
18
|
import { createEnvironments } from "./environments.mjs";
|
|
19
19
|
import { createGuestRequests } from "./guestRequests.mjs";
|
|
20
|
+
import {
|
|
21
|
+
createLiveViews,
|
|
22
|
+
createViewCopies,
|
|
23
|
+
viewRequest,
|
|
24
|
+
} from "./liveViews.mjs";
|
|
20
25
|
import { createBoardWaits } from "./boardWaits.mjs";
|
|
21
26
|
import { createCoordinator } from "./coordinator.mjs";
|
|
22
27
|
import { createServerWords } from "./serverWords.mjs";
|
|
@@ -233,6 +238,7 @@ export async function startDaemon() {
|
|
|
233
238
|
// Which of the person's computers this is; ownMachines (created once this computer's key is read) keeps it.
|
|
234
239
|
const machine = loadMachine(paths.machine);
|
|
235
240
|
let ownMachines = null;
|
|
241
|
+
let viewCopies = null;
|
|
236
242
|
// Where an agent calling from a computer is, and the name beside its words ("Codex (linux)") once the person has more
|
|
237
243
|
// than one computer (ownMachines.mjs); before ownMachines starts, this computer alone.
|
|
238
244
|
const whereIs = (id) =>
|
|
@@ -262,6 +268,8 @@ export async function startDaemon() {
|
|
|
262
268
|
after,
|
|
263
269
|
feed,
|
|
264
270
|
standIn,
|
|
271
|
+
// This computer applies a row's changed columns (boardMirror.mjs changedSince).
|
|
272
|
+
diffs: true,
|
|
265
273
|
})) ?? null,
|
|
266
274
|
pushPage: async (page, { bring = false } = {}) =>
|
|
267
275
|
(await ownMachines?.mirrorCall({
|
|
@@ -1106,6 +1114,7 @@ export async function startDaemon() {
|
|
|
1106
1114
|
homeOf: (...args) => homeOf(...args),
|
|
1107
1115
|
placeElsewhere: (agent, workspace) =>
|
|
1108
1116
|
ownMachines?.placeElsewhere(agent, workspace) ?? null,
|
|
1117
|
+
threadFiles: (thread) => boardFiles.threadFiles(thread).files,
|
|
1109
1118
|
// Read when a turn starts: the person's computers are known once ownMachines is made, after this.
|
|
1110
1119
|
computers: {
|
|
1111
1120
|
self: () => ownMachines?.self() ?? null,
|
|
@@ -2076,13 +2085,22 @@ export async function startDaemon() {
|
|
|
2076
2085
|
return result;
|
|
2077
2086
|
}
|
|
2078
2087
|
return {
|
|
2079
|
-
pending:
|
|
2088
|
+
pending: (async () => {
|
|
2089
|
+
// A read this computer keeps a copy of is answered here (liveViews.mjs), and asks home for nothing more.
|
|
2090
|
+
const kept = await viewCopies?.read(request);
|
|
2091
|
+
if (kept) return kept;
|
|
2092
|
+
const answer = await ownMachines.callHome(request);
|
|
2080
2093
|
if (answer?.code === "home_offline" && readsFromCopy(request.command))
|
|
2081
2094
|
return fromCopy(request, answer);
|
|
2082
|
-
|
|
2095
|
+
// A write that reached home brings this computer's copy of the board up to date; a read changes nothing.
|
|
2096
|
+
if (
|
|
2097
|
+
answer?.ok !== false &&
|
|
2098
|
+
!readsFromCopy(request.command) &&
|
|
2099
|
+
!viewRequest(request)
|
|
2100
|
+
)
|
|
2083
2101
|
void boardMirror.sync();
|
|
2084
2102
|
return answer;
|
|
2085
|
-
}),
|
|
2103
|
+
})(),
|
|
2086
2104
|
};
|
|
2087
2105
|
};
|
|
2088
2106
|
const fromCopy = async (request, offline) => {
|
|
@@ -2302,6 +2320,7 @@ export async function startDaemon() {
|
|
|
2302
2320
|
};
|
|
2303
2321
|
const phones = await createPhoneDesk({
|
|
2304
2322
|
store,
|
|
2323
|
+
log,
|
|
2305
2324
|
identityFile: paths.phoneIdentity,
|
|
2306
2325
|
setupOfferFile: paths.browserOffer,
|
|
2307
2326
|
// A window linked to a worker works the swarm's board, which is home's, not one of its own.
|
|
@@ -2379,7 +2398,21 @@ export async function startDaemon() {
|
|
|
2379
2398
|
onRequest: (peerId, body) =>
|
|
2380
2399
|
ownMachines?.fromLink(peerId, body) ?? { ok: false },
|
|
2381
2400
|
});
|
|
2401
|
+
// The board as another computer shows it, answered there (liveViews.mjs): on home, the views it keeps for them; on
|
|
2402
|
+
// a worker, its copies, which a waiting surface here hears move.
|
|
2403
|
+
const liveViews = createLiveViews({
|
|
2404
|
+
answer: (workerId, request) => ownMachines.answerAs(workerId, request),
|
|
2405
|
+
push: (workerId, body) => ownMachines.pushViews(workerId, body),
|
|
2406
|
+
onMove: (listener) => boardWaits.onChange(listener),
|
|
2407
|
+
log,
|
|
2408
|
+
});
|
|
2409
|
+
viewCopies = createViewCopies({
|
|
2410
|
+
callHome: (request) => ownMachines.callHome(request),
|
|
2411
|
+
changed: () => boardWaits.bump(),
|
|
2412
|
+
});
|
|
2382
2413
|
ownMachines = createOwnMachines({
|
|
2414
|
+
liveViews: () => liveViews,
|
|
2415
|
+
viewCopies: () => viewCopies,
|
|
2383
2416
|
directLink,
|
|
2384
2417
|
store,
|
|
2385
2418
|
log,
|
|
@@ -2394,6 +2427,7 @@ export async function startDaemon() {
|
|
|
2394
2427
|
folders: () => machineFolders(),
|
|
2395
2428
|
repositoryOf: (folder) => repositoryName(folder),
|
|
2396
2429
|
homeBack: () => {
|
|
2430
|
+
viewCopies.reset();
|
|
2397
2431
|
void machineRuns.check();
|
|
2398
2432
|
void boardMirror.sync();
|
|
2399
2433
|
void swarmDevices.report();
|
|
@@ -2445,6 +2479,15 @@ export async function startDaemon() {
|
|
|
2445
2479
|
agents: installedAgents,
|
|
2446
2480
|
// Only in a folder this computer's own agents' sessions or T3 Code worked in, never just one home names.
|
|
2447
2481
|
knownFolders: () => folders.known({ own: true }).map((place) => place.path),
|
|
2482
|
+
// A folder-less thread's files, as this computer's copy of the board lists them (boardMirror.mjs).
|
|
2483
|
+
threadFileIds: (threadId) => {
|
|
2484
|
+
const thread = store.getTask(threadId);
|
|
2485
|
+
return thread
|
|
2486
|
+
? [thread, ...store.children(thread.id)].flatMap((task) =>
|
|
2487
|
+
store.taskFiles(task.id).map((file) => file.id)
|
|
2488
|
+
)
|
|
2489
|
+
: [];
|
|
2490
|
+
},
|
|
2448
2491
|
log,
|
|
2449
2492
|
modelFor: modelForRun,
|
|
2450
2493
|
});
|
package/lib/directLink.mjs
CHANGED
|
@@ -14,8 +14,9 @@ import { ASK_TIMEOUT_MS } from "./bridge.mjs";
|
|
|
14
14
|
// for windows, pages of the board, every one still sealed with the pair's key. The link is a second road, not a
|
|
15
15
|
// second set of rules.
|
|
16
16
|
// - A worker dials home. A request with no link, or with no answer within the time it has through Harness, gets null
|
|
17
|
-
// and goes through Harness;
|
|
18
|
-
// can freeze for good on some networks:
|
|
17
|
+
// and goes through Harness; the link itself stays. It closes when HyperDHT's keep-alive finds the other side gone,
|
|
18
|
+
// and is dialed again when it is next needed (a connection can freeze for good on some networks:
|
|
19
|
+
// holepunchto/libudx#333). A new link counts once one round trip over it answers.
|
|
19
20
|
|
|
20
21
|
// The public HyperDHT network's own starting points unless the computer is told others (a local test network).
|
|
21
22
|
const bootstrapOf = (value) =>
|
|
@@ -27,6 +28,11 @@ const bootstrapOf = (value) =>
|
|
|
27
28
|
: undefined;
|
|
28
29
|
// A frame larger than this is not one the other computer sent (a sealed message is at most 1.4 MB).
|
|
29
30
|
const MAX_FRAME_BYTES = 4 * 1024 * 1024;
|
|
31
|
+
// How long a new link has to answer one round trip before it is dropped and dialed again (owner's value, October 28).
|
|
32
|
+
// Measured: 2 to 11 ms over 7 links in the direct-link smoke (one Mac, a local HyperDHT network), about 45 ms over the
|
|
33
|
+
// internet in the October 21 trial (11 hole-punched links). A half-open first link stayed silent until HyperDHT dropped
|
|
34
|
+
// it about 24 s later.
|
|
35
|
+
const HELLO_MS = 2_000;
|
|
30
36
|
|
|
31
37
|
export function createDirectLink({
|
|
32
38
|
keyFile,
|
|
@@ -119,12 +125,12 @@ export function createDirectLink({
|
|
|
119
125
|
|
|
120
126
|
// One link: newline-framed JSON, requests and answers matched by number. The peer is looked up when something
|
|
121
127
|
// happens, never kept: a key swap updates it in place (a copied peer once kept a dead link "linked").
|
|
122
|
-
const attach = (id, socket) => {
|
|
128
|
+
const attach = (id, socket, { verified = true } = {}) => {
|
|
123
129
|
const peer = peers.get(id);
|
|
124
130
|
if (!peer) return socket.destroy();
|
|
125
131
|
const before = peer.socket;
|
|
126
132
|
peer.socket = socket;
|
|
127
|
-
peer.
|
|
133
|
+
peer.verified = verified;
|
|
128
134
|
if (before && before !== socket) {
|
|
129
135
|
release(peer, before);
|
|
130
136
|
before.destroy();
|
|
@@ -134,7 +140,6 @@ export function createDirectLink({
|
|
|
134
140
|
let buffered = "";
|
|
135
141
|
socket.on("data", (chunk) => {
|
|
136
142
|
const current = peers.get(id);
|
|
137
|
-
if (current?.socket === socket) current.heardAt = Date.now();
|
|
138
143
|
buffered += chunk.toString("utf8");
|
|
139
144
|
if (buffered.length > MAX_FRAME_BYTES) return socket.destroy();
|
|
140
145
|
let at;
|
|
@@ -196,7 +201,6 @@ export function createDirectLink({
|
|
|
196
201
|
socket: null,
|
|
197
202
|
pending: new Map(),
|
|
198
203
|
dialing: null,
|
|
199
|
-
heardAt: 0,
|
|
200
204
|
});
|
|
201
205
|
return;
|
|
202
206
|
}
|
|
@@ -240,10 +244,40 @@ export function createDirectLink({
|
|
|
240
244
|
socket.once("error", failed);
|
|
241
245
|
socket.once("open", () => {
|
|
242
246
|
socket.off("error", failed);
|
|
243
|
-
attach(id, socket);
|
|
247
|
+
attach(id, socket, { verified: false });
|
|
244
248
|
resolve();
|
|
245
249
|
});
|
|
246
250
|
});
|
|
251
|
+
// Open on this side is not yet a link: the other computer sometimes never saw the connection, and every call
|
|
252
|
+
// over it waited out the whole timeout before going through Harness (smoke direct-link, one run in two). It counts
|
|
253
|
+
// once the other computer answers over it; until then calls go through Harness.
|
|
254
|
+
const current = peers.get(id);
|
|
255
|
+
if (current?.socket && !current.verified) {
|
|
256
|
+
const socket = current.socket;
|
|
257
|
+
const openedAt = Date.now();
|
|
258
|
+
const heard = await send(current, socket, { kind: "hello" }, HELLO_MS);
|
|
259
|
+
const still = peers.get(id);
|
|
260
|
+
// Gone, or a newer link took its place; a link the timeout already dropped is still dialed again below.
|
|
261
|
+
if (!still || (still.socket && still.socket !== socket)) return;
|
|
262
|
+
if (heard === null) {
|
|
263
|
+
log(
|
|
264
|
+
`the direct link with ${id} did not answer; going through Harness and dialing again`
|
|
265
|
+
);
|
|
266
|
+
release(still, socket);
|
|
267
|
+
still.socket = null;
|
|
268
|
+
socket.destroy();
|
|
269
|
+
// A half-open link (this side open, the other never saw it) is dialed again at once rather than when the next
|
|
270
|
+
// call needs it.
|
|
271
|
+
still.redials = (still.redials ?? 0) + 1;
|
|
272
|
+
if (still.redials <= 3) setImmediate(() => void dial(id));
|
|
273
|
+
} else {
|
|
274
|
+
still.verified = true;
|
|
275
|
+
still.redials = 0;
|
|
276
|
+
log(
|
|
277
|
+
`the direct link with ${id} answered in ${Date.now() - openedAt} ms`
|
|
278
|
+
);
|
|
279
|
+
}
|
|
280
|
+
}
|
|
247
281
|
})().finally(() => {
|
|
248
282
|
peer.dialing = null;
|
|
249
283
|
});
|
|
@@ -253,24 +287,23 @@ export function createDirectLink({
|
|
|
253
287
|
/** A request to that computer over the link; null when there is none or it did not answer in time. */
|
|
254
288
|
const request = (id, body) => {
|
|
255
289
|
const peer = peers.get(id);
|
|
256
|
-
if (!peer?.socket) {
|
|
290
|
+
if (!peer?.socket || !peer.verified) {
|
|
257
291
|
if (peer?.dialer) void dial(id);
|
|
258
292
|
return Promise.resolve(null);
|
|
259
293
|
}
|
|
260
|
-
|
|
294
|
+
return send(peer, peer.socket, body);
|
|
295
|
+
};
|
|
296
|
+
// One request over the link and its answer; null when none came in time. A slow answer is not a dead link: that one
|
|
297
|
+
// call goes through Harness instead and the link stays (owner, October 28: links to pop-os and MBP-2 were torn down
|
|
298
|
+
// and dialed again about 20 times in 10 minutes, each time one push to a busy computer went 15 s unanswered). The
|
|
299
|
+
// link closes when HyperDHT's own keep-alive finds the other side gone (`close` and `error` in attach).
|
|
300
|
+
const send = (peer, socket, body, waitMs = timeoutMs) => {
|
|
261
301
|
const i = ++sequence;
|
|
262
|
-
const sentAt = Date.now();
|
|
263
302
|
return new Promise((resolve) => {
|
|
264
303
|
const timer = setTimeout(() => {
|
|
265
304
|
peer.pending.delete(i);
|
|
266
|
-
// Nothing at all heard from the other computer for as long as Harness would have taken: the link is dropped and
|
|
267
|
-
// dialed again when next needed. Heard from meanwhile, only this request was slow and the link stays.
|
|
268
|
-
if (peer.socket === socket && peer.heardAt < sentAt) {
|
|
269
|
-
log(`the direct link with ${id} went quiet; going through Harness`);
|
|
270
|
-
socket.destroy();
|
|
271
|
-
}
|
|
272
305
|
resolve(null);
|
|
273
|
-
},
|
|
306
|
+
}, waitMs);
|
|
274
307
|
peer.pending.set(i, { resolve, timer, socket });
|
|
275
308
|
socket.write(JSON.stringify({ i, q: body }) + "\n");
|
|
276
309
|
});
|
|
@@ -280,15 +313,37 @@ export function createDirectLink({
|
|
|
280
313
|
const status = () =>
|
|
281
314
|
[...peers.entries()].map(([id, peer]) => ({
|
|
282
315
|
id,
|
|
283
|
-
linked: Boolean(peer.socket),
|
|
316
|
+
linked: Boolean(peer.socket && peer.verified),
|
|
284
317
|
at: peer.socket?.rawStream?.remoteHost ?? null,
|
|
285
318
|
}));
|
|
286
319
|
|
|
320
|
+
/**
|
|
321
|
+
* The listener started afresh, the person's computers' keys kept: on home it once stayed unreachable for 30 minutes
|
|
322
|
+
* after a restart (every dial from pop-os HOLEPUNCH_ABORTED) until the daemon restarted again, when both workers
|
|
323
|
+
* linked within seconds (owner, October 28: home restarts its listener). Every link drops and is made again.
|
|
324
|
+
*/
|
|
325
|
+
const restart = async () => {
|
|
326
|
+
for (const peer of peers.values())
|
|
327
|
+
if (peer.socket) {
|
|
328
|
+
const socket = peer.socket;
|
|
329
|
+
release(peer, socket);
|
|
330
|
+
peer.socket = null;
|
|
331
|
+
socket.destroy();
|
|
332
|
+
}
|
|
333
|
+
await server?.close().catch(() => {});
|
|
334
|
+
await node?.destroy().catch(() => {});
|
|
335
|
+
server = null;
|
|
336
|
+
node = null;
|
|
337
|
+
starting = null;
|
|
338
|
+
log("direct links: started the listener again");
|
|
339
|
+
return start();
|
|
340
|
+
};
|
|
341
|
+
|
|
287
342
|
const stop = async () => {
|
|
288
343
|
for (const peer of peers.values()) peer.socket?.destroy();
|
|
289
344
|
await server?.close().catch(() => {});
|
|
290
345
|
await node?.destroy().catch(() => {});
|
|
291
346
|
};
|
|
292
347
|
|
|
293
|
-
return { key, allow, disallow, dial, request, status, stop };
|
|
348
|
+
return { key, allow, disallow, dial, request, status, restart, stop };
|
|
294
349
|
}
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
// What changed between two answers of the same read, small enough to send as it is, and that change applied to the
|
|
2
|
+
// earlier answer to give the later one exactly (liveViews.mjs: home pushes each view's change to the computers that
|
|
3
|
+
// read it; the thread list alone was 362 KB a read, mostly the same each time).
|
|
4
|
+
//
|
|
5
|
+
// A change is one of:
|
|
6
|
+
// - { v: value }: the whole value, for a value of another kind, a list without ids, or anything short;
|
|
7
|
+
// - { o: { key: change }, x: [key] }: an object, with the changes to its keys and the keys it lost;
|
|
8
|
+
// - { l: [id], c: { id: change } }: a list whose items each have their own string id, in their new order, with the
|
|
9
|
+
// changes to the items that changed or came new (a new one as { v: item }).
|
|
10
|
+
// Two equal values have no change (undefined).
|
|
11
|
+
|
|
12
|
+
const isObject = (value) =>
|
|
13
|
+
value !== null && typeof value === "object" && !Array.isArray(value);
|
|
14
|
+
|
|
15
|
+
// A list whose every item is an object with its own string id, none twice.
|
|
16
|
+
const idsOf = (list) => {
|
|
17
|
+
if (!Array.isArray(list) || !list.length) return null;
|
|
18
|
+
const ids = [];
|
|
19
|
+
const seen = new Set();
|
|
20
|
+
for (const item of list) {
|
|
21
|
+
if (!isObject(item) || typeof item.id !== "string" || seen.has(item.id))
|
|
22
|
+
return null;
|
|
23
|
+
seen.add(item.id);
|
|
24
|
+
ids.push(item.id);
|
|
25
|
+
}
|
|
26
|
+
return ids;
|
|
27
|
+
};
|
|
28
|
+
|
|
29
|
+
const same = (before, after) =>
|
|
30
|
+
before === after || JSON.stringify(before) === JSON.stringify(after);
|
|
31
|
+
|
|
32
|
+
/** The change from `before` to `after`, or undefined when there is none. */
|
|
33
|
+
export function diff(before, after) {
|
|
34
|
+
if (same(before, after)) return undefined;
|
|
35
|
+
if (isObject(before) && isObject(after)) {
|
|
36
|
+
const o = {};
|
|
37
|
+
for (const [key, value] of Object.entries(after)) {
|
|
38
|
+
const change = diff(before[key], value);
|
|
39
|
+
if (change !== undefined) o[key] = change;
|
|
40
|
+
}
|
|
41
|
+
const x = Object.keys(before).filter((key) => !(key in after));
|
|
42
|
+
return { o, ...(x.length ? { x } : {}) };
|
|
43
|
+
}
|
|
44
|
+
const beforeIds = idsOf(before);
|
|
45
|
+
const afterIds = idsOf(after);
|
|
46
|
+
if (beforeIds && afterIds) {
|
|
47
|
+
const was = new Map(before.map((item) => [item.id, item]));
|
|
48
|
+
const c = {};
|
|
49
|
+
for (const item of after) {
|
|
50
|
+
const change = was.has(item.id)
|
|
51
|
+
? diff(was.get(item.id), item)
|
|
52
|
+
: { v: item };
|
|
53
|
+
if (change !== undefined) c[item.id] = change;
|
|
54
|
+
}
|
|
55
|
+
return { l: afterIds, c };
|
|
56
|
+
}
|
|
57
|
+
return { v: after };
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/** `before` with `change` applied: the later answer. Throws on a change that does not fit `before`. */
|
|
61
|
+
export function apply(before, change) {
|
|
62
|
+
if (change === undefined) return before;
|
|
63
|
+
if ("v" in change) return change.v;
|
|
64
|
+
if (change.o) {
|
|
65
|
+
if (!isObject(before))
|
|
66
|
+
throw new Error("an object's change on a non-object");
|
|
67
|
+
const after = { ...before };
|
|
68
|
+
for (const key of change.x ?? []) delete after[key];
|
|
69
|
+
for (const [key, sub] of Object.entries(change.o))
|
|
70
|
+
after[key] = apply(before[key], sub);
|
|
71
|
+
return after;
|
|
72
|
+
}
|
|
73
|
+
if (change.l) {
|
|
74
|
+
const was = new Map(
|
|
75
|
+
(Array.isArray(before) ? before : []).map((item) => [item?.id, item])
|
|
76
|
+
);
|
|
77
|
+
return change.l.map((id) => {
|
|
78
|
+
const sub = change.c?.[id];
|
|
79
|
+
if (sub !== undefined) return apply(was.get(id), sub);
|
|
80
|
+
if (!was.has(id))
|
|
81
|
+
throw new Error(`a list's change names ${id}, not in it`);
|
|
82
|
+
return was.get(id);
|
|
83
|
+
});
|
|
84
|
+
}
|
|
85
|
+
throw new Error("not a change");
|
|
86
|
+
}
|