tryharness 0.7.44 → 0.7.53
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +232 -12
- package/cli.mjs +406 -184
- package/lib/actorRoster.mjs +28 -5
- package/lib/adapters.mjs +9 -1
- package/lib/agentRuntimeConfig.mjs +36 -0
- package/lib/agentRuntimes.mjs +560 -0
- package/lib/atlas.mjs +7 -2
- package/lib/boardCommands.mjs +5 -10
- package/lib/boardFileCommands.mjs +2 -1
- package/lib/boardMirror.mjs +459 -0
- package/lib/boardProtocol.json +259 -6
- package/lib/boardTools.mjs +2 -6
- package/lib/bridge.mjs +27 -3
- package/lib/browserWake.mjs +60 -0
- package/lib/cards.mjs +35 -2
- package/lib/commands.mjs +20 -5
- package/lib/connectionTools.d.ts +13 -0
- package/lib/connectionTools.js +45 -0
- package/lib/connectorAuth.mjs +309 -0
- package/lib/connectorCatalog.json +41 -0
- package/lib/connectorCommands.mjs +121 -0
- package/lib/connectorGoogle.mjs +513 -0
- package/lib/connectors.mjs +805 -0
- package/lib/coordinator.mjs +16 -2
- package/lib/coreModels.mjs +78 -0
- package/lib/daemon.mjs +775 -46
- package/lib/desktop.mjs +7 -2
- package/lib/digest.mjs +32 -0
- package/lib/directLink.mjs +283 -0
- package/lib/environments.mjs +595 -0
- package/lib/folders.mjs +2 -2
- package/lib/goalCommands.mjs +61 -12
- package/lib/inference.mjs +263 -152
- package/lib/lessons.mjs +6 -0
- package/lib/lineup.mjs +324 -0
- package/lib/localmodels.mjs +10 -0
- package/lib/machineIdentity.mjs +3 -0
- package/lib/machineRoutes.mjs +40 -2
- package/lib/machineRuns.mjs +30 -3
- package/lib/machineSeal.mjs +69 -10
- package/lib/mcp-stdio.mjs +180 -98
- package/lib/mcpRelay.mjs +119 -0
- package/lib/members.json +12 -0
- package/lib/members.mjs +33 -0
- package/lib/modelMemory.mjs +100 -0
- package/lib/modelTiers.json +1183 -0
- package/lib/modelTiers.mjs +133 -0
- package/lib/models.mjs +41 -8
- package/lib/museSession.mjs +119 -0
- package/lib/nativeHost.mjs +203 -0
- package/lib/network.mjs +38 -1
- package/lib/nudge.mjs +10 -2
- package/lib/onboard.mjs +430 -1
- package/lib/oneclawLocal.mjs +150 -0
- package/lib/ownMachines.mjs +521 -36
- package/lib/pairing.mjs +165 -0
- package/lib/paths.mjs +4 -0
- package/lib/phones.mjs +61 -0
- package/lib/placeCommands.mjs +27 -0
- package/lib/places.mjs +10 -2
- package/lib/plans.mjs +151 -27
- package/lib/profile.mjs +152 -0
- package/lib/projectGit.mjs +328 -0
- package/lib/projectSettings.mjs +120 -0
- package/lib/projects.mjs +2539 -0
- package/lib/readiness.mjs +283 -0
- package/lib/repoLinks.mjs +354 -0
- package/lib/roomChannels.mjs +641 -0
- package/lib/runBrief.mjs +32 -2
- package/lib/runRequests.mjs +157 -0
- package/lib/runWorker.mjs +20 -7
- package/lib/runner.mjs +67 -8
- package/lib/secretPatterns.json +55 -0
- package/lib/secrets.mjs +10 -27
- package/lib/seen.mjs +102 -0
- package/lib/service.mjs +11 -6
- package/lib/sessionControl.mjs +10 -8
- package/lib/sessionIndex.mjs +5 -3
- package/lib/store.mjs +559 -30
- package/lib/subagents.mjs +8 -4
- package/lib/swarm.mjs +379 -38
- package/lib/swarmMemory.mjs +947 -0
- package/lib/t3.mjs +43 -13
- package/lib/taskCommands.mjs +58 -5
- package/lib/taskRuns.mjs +61 -11
- package/lib/team.json +17 -15
- package/lib/team.mjs +68 -18
- package/lib/threadRuns.mjs +50 -7
- package/lib/threadView.mjs +68 -15
- package/lib/tokenizer.mjs +405 -0
- package/lib/ui.mjs +16 -2
- package/lib/usageTotals.mjs +148 -0
- package/lib/vault.mjs +1144 -0
- package/lib/vaultCloud.mjs +117 -0
- package/lib/vaultCommands.mjs +201 -0
- package/lib/vaultTools.d.ts +12 -0
- package/lib/vaultTools.js +42 -0
- package/lib/windowAsks.mjs +42 -0
- package/lib/workFocus.mjs +59 -0
- package/lib/workLinks.mjs +90 -0
- package/package.json +9 -4
package/lib/atlas.mjs
CHANGED
|
@@ -503,6 +503,11 @@ const git = (cwd, args, input) =>
|
|
|
503
503
|
maxBuffer: 256 * 2 ** 20,
|
|
504
504
|
});
|
|
505
505
|
|
|
506
|
+
// A part of a larger string is kept by V8 as a slice that holds the whole string alive. The rows below outlive git's
|
|
507
|
+
// output (a commit's file list is kept while history is read), so each path and blob is its own copy, or every list
|
|
508
|
+
// kept holds that commit's entire `git ls-tree` output.
|
|
509
|
+
const own = (text) => structuredClone(text);
|
|
510
|
+
|
|
506
511
|
/** The repository's files at a commit (or the working tree when commit is null): [{ path, blob, size }]. */
|
|
507
512
|
export function treeFiles(root, commit = null) {
|
|
508
513
|
if (commit) {
|
|
@@ -515,7 +520,7 @@ export function treeFiles(root, commit = null) {
|
|
|
515
520
|
const [meta, rel] = line.split("\t");
|
|
516
521
|
const [, type, blob, size] = meta.split(/\s+/);
|
|
517
522
|
return type === "blob"
|
|
518
|
-
? { path: rel, blob, size: Number(size) || 0 }
|
|
523
|
+
? { path: own(rel), blob: own(blob), size: Number(size) || 0 }
|
|
519
524
|
: null;
|
|
520
525
|
})
|
|
521
526
|
.filter((row) => row && indexable(row.path, row.size));
|
|
@@ -529,7 +534,7 @@ export function treeFiles(root, commit = null) {
|
|
|
529
534
|
const tab = entry.indexOf("\t");
|
|
530
535
|
if (tab < 0) continue;
|
|
531
536
|
const [, blob, stage] = entry.slice(0, tab).split(" ");
|
|
532
|
-
if (stage === "0") blobs.set(entry.slice(tab + 1), blob);
|
|
537
|
+
if (stage === "0") blobs.set(own(entry.slice(tab + 1)), own(blob));
|
|
533
538
|
}
|
|
534
539
|
const changed = git(root, ["diff", "--name-only", "-z"])
|
|
535
540
|
.stdout.split("\0")
|
package/lib/boardCommands.mjs
CHANGED
|
@@ -15,20 +15,15 @@ export function createBoardCommands({
|
|
|
15
15
|
return { ok: false, error: "board requests need an agent name" };
|
|
16
16
|
if (request.command === "board.post") touchListener();
|
|
17
17
|
const agent = String(request.agent);
|
|
18
|
-
|
|
19
|
-
|
|
18
|
+
// The computer the call came from: this one, or a worker whose call home opened (ownMachines.mjs).
|
|
19
|
+
const machine = machineOf(request);
|
|
20
|
+
const lastLooked = store.lastSeenAt(agent, machine);
|
|
21
|
+
store.touchAgent(agent, machine);
|
|
20
22
|
const id = String(request.id || "");
|
|
21
23
|
const expected = Number.isInteger(request.expectedVersion)
|
|
22
24
|
? request.expectedVersion
|
|
23
25
|
: null;
|
|
24
|
-
|
|
25
|
-
const context = {
|
|
26
|
-
agent,
|
|
27
|
-
lastLooked,
|
|
28
|
-
id,
|
|
29
|
-
expected,
|
|
30
|
-
machine: machineOf(request),
|
|
31
|
-
};
|
|
26
|
+
const context = { agent, lastLooked, id, expected, machine };
|
|
32
27
|
const owner = boardCommandOwner(String(request.command));
|
|
33
28
|
const handler = Object.hasOwn(handlers, owner) ? handlers[owner] : task;
|
|
34
29
|
return handler(request, context);
|
|
@@ -23,7 +23,7 @@ export function createBoardFileCommands({
|
|
|
23
23
|
tooBig,
|
|
24
24
|
visibleFile,
|
|
25
25
|
}) {
|
|
26
|
-
return (request, { agent, id }) => {
|
|
26
|
+
return (request, { agent, id, machine }) => {
|
|
27
27
|
if (request.command === "board.file.add") {
|
|
28
28
|
const task = store.getTask(id);
|
|
29
29
|
if (!task) return { ok: false, error: "task not found" };
|
|
@@ -122,6 +122,7 @@ export function createBoardFileCommands({
|
|
|
122
122
|
author: agent,
|
|
123
123
|
text: `${agent} attached ${fileLine(file)}`,
|
|
124
124
|
source: `file:${file.id}`,
|
|
125
|
+
machine,
|
|
125
126
|
});
|
|
126
127
|
notifyBoard(store.getTask(place));
|
|
127
128
|
readFilesSoon();
|
|
@@ -0,0 +1,459 @@
|
|
|
1
|
+
import fs from "node:fs";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { randomUUID } from "node:crypto";
|
|
4
|
+
|
|
5
|
+
// Every computer of the person's swarm keeps the board. Home's is the board; each worker keeps a copy of it in its own
|
|
6
|
+
// board tables, so its agents still read the board when home is out of reach, and a worker standing in for home
|
|
7
|
+
// already holds it (owner, October 21: "all nodes and home in a swarm stay in sync as much as possible", "home node is
|
|
8
|
+
// canonical", "entire swarm works off shared board").
|
|
9
|
+
//
|
|
10
|
+
// - Every computer numbers the changes to its board tables (triggers into board_changes: the latest change per row, so
|
|
11
|
+
// the log is never larger than the board). Its feed id names that numbering.
|
|
12
|
+
// - A worker asks home for the changes after the last one it applied (`since`, a sealed call through ownMachines.mjs)
|
|
13
|
+
// and applies each page in one transaction: a row as home has it now, or its removal. When home's feed is not the one
|
|
14
|
+
// it followed (the first time, or home moved), it starts over and, at the end, takes out the rows home does not have,
|
|
15
|
+
// writing them to a file first: a computer that was home before keeps its old board there.
|
|
16
|
+
// - Home tells connected workers when its board changed; a worker also catches up after each write it sent home and
|
|
17
|
+
// when home is back.
|
|
18
|
+
|
|
19
|
+
/** A worker's call for home's changes (ownMachines.mjs answers it on home, never through the board's own commands). */
|
|
20
|
+
export const MIRROR_COMMAND = "board.mirror.since";
|
|
21
|
+
/** A computer that stood in as home giving home back what it wrote meanwhile (answered on home the same way). */
|
|
22
|
+
export const MERGE_COMMAND = "board.mirror.merge";
|
|
23
|
+
|
|
24
|
+
/** The board tables every computer keeps the same as home's, by their primary key. */
|
|
25
|
+
export const MIRRORED = {
|
|
26
|
+
board_tasks: ["id"],
|
|
27
|
+
board_goals: ["goal_id"],
|
|
28
|
+
board_threads: ["task_id"],
|
|
29
|
+
board_messages: ["id"],
|
|
30
|
+
board_contributions: ["id"],
|
|
31
|
+
board_reactions: ["post_id", "agent"],
|
|
32
|
+
board_standing: ["agent"],
|
|
33
|
+
board_lessons: ["id"],
|
|
34
|
+
// What the person had forgotten or replaced, and what the dream keeps (swarmMemory.mjs), so every computer's agents
|
|
35
|
+
// recall the same.
|
|
36
|
+
board_memory_marks: ["contribution_id"],
|
|
37
|
+
board_observations: ["id"],
|
|
38
|
+
team_notes: ["id"],
|
|
39
|
+
board_files: ["id"],
|
|
40
|
+
};
|
|
41
|
+
// A page stays well under what one sealed call carries (MAX_CT, src/shared/bridge/sealedWire.ts).
|
|
42
|
+
const PAGE_BYTES = 512 * 1024;
|
|
43
|
+
const FEED = "mirror-feed";
|
|
44
|
+
const CURSOR = "mirror-cursor";
|
|
45
|
+
// On a computer standing in as home: whose board it holds and where its own changes began.
|
|
46
|
+
const STAND_IN = "mirror-stand-in";
|
|
47
|
+
|
|
48
|
+
const keyJson = (prefix, columns) =>
|
|
49
|
+
`json_array(${columns.map((column) => `${prefix}.${column}`).join(", ")})`;
|
|
50
|
+
|
|
51
|
+
/** The change log and its triggers, made once on every computer, and every row already there counted as changed. */
|
|
52
|
+
function install(db) {
|
|
53
|
+
db.exec(`create table if not exists board_changes (
|
|
54
|
+
seq integer primary key autoincrement,
|
|
55
|
+
tbl text not null,
|
|
56
|
+
pk text not null,
|
|
57
|
+
gone integer not null default 0,
|
|
58
|
+
unique(tbl, pk)
|
|
59
|
+
)`);
|
|
60
|
+
for (const [table, columns] of Object.entries(MIRRORED)) {
|
|
61
|
+
// A delete then an insert, never "insert or replace": the conflict rule of the statement that fired the trigger
|
|
62
|
+
// (a store's "insert or ignore") would override the trigger's own and drop the change (review, October 21).
|
|
63
|
+
const changed = (prefix, gone) =>
|
|
64
|
+
`delete from board_changes where tbl = '${table}' and pk = ${keyJson(prefix, columns)};
|
|
65
|
+
insert into board_changes(tbl, pk, gone) values('${table}', ${keyJson(prefix, columns)}, ${gone});`;
|
|
66
|
+
// Made again on every start, so a computer that has older triggers gets these.
|
|
67
|
+
db.exec(`
|
|
68
|
+
drop trigger if exists ${table}_mirror_insert;
|
|
69
|
+
drop trigger if exists ${table}_mirror_update;
|
|
70
|
+
drop trigger if exists ${table}_mirror_delete;
|
|
71
|
+
create trigger ${table}_mirror_insert after insert on ${table} begin ${changed("new", 0)} end;
|
|
72
|
+
create trigger ${table}_mirror_update after update on ${table} begin ${changed("new", 0)} end;
|
|
73
|
+
create trigger ${table}_mirror_delete after delete on ${table} begin ${changed("old", 1)} end;
|
|
74
|
+
`);
|
|
75
|
+
}
|
|
76
|
+
if (!db.get("select value from board_meta where key = ?", [FEED])) {
|
|
77
|
+
db.exec("begin immediate");
|
|
78
|
+
try {
|
|
79
|
+
for (const [table, columns] of Object.entries(MIRRORED))
|
|
80
|
+
db.run(
|
|
81
|
+
`insert or ignore into board_changes(tbl, pk, gone) select '${table}', ${keyJson(table, columns)}, 0 from ${table}`
|
|
82
|
+
);
|
|
83
|
+
db.run("insert into board_meta(key, value) values(?, ?)", [
|
|
84
|
+
FEED,
|
|
85
|
+
randomUUID(),
|
|
86
|
+
]);
|
|
87
|
+
db.exec("commit");
|
|
88
|
+
} catch (error) {
|
|
89
|
+
if (db.inTransaction) db.exec("rollback");
|
|
90
|
+
throw error;
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
const whereKey = (columns) =>
|
|
96
|
+
columns.map((column) => `${column} = ?`).join(" and ");
|
|
97
|
+
|
|
98
|
+
// A row as the wire carries it (JSON): bytes (a lesson's embedding) as base64, read back as bytes.
|
|
99
|
+
const onWire = (row) =>
|
|
100
|
+
Object.fromEntries(
|
|
101
|
+
Object.entries(row).map(([name, value]) => [
|
|
102
|
+
name,
|
|
103
|
+
value instanceof Uint8Array
|
|
104
|
+
? { $bytes: Buffer.from(value).toString("base64") }
|
|
105
|
+
: value,
|
|
106
|
+
])
|
|
107
|
+
);
|
|
108
|
+
const offWire = (value) =>
|
|
109
|
+
value && typeof value === "object" && typeof value.$bytes === "string"
|
|
110
|
+
? new Uint8Array(Buffer.from(value.$bytes, "base64"))
|
|
111
|
+
: value;
|
|
112
|
+
// The words search reads for the rows that have them (store.mjs keeps these beside the rows it writes).
|
|
113
|
+
const SEARCHED = {
|
|
114
|
+
board_contributions: {
|
|
115
|
+
table: "board_contributions_fts",
|
|
116
|
+
columns: (row) => ({ id: row.id, task_id: row.task_id, text: row.text }),
|
|
117
|
+
},
|
|
118
|
+
board_lessons: {
|
|
119
|
+
table: "board_lessons_fts",
|
|
120
|
+
columns: (row) => ({ id: row.id, text: row.text }),
|
|
121
|
+
},
|
|
122
|
+
};
|
|
123
|
+
|
|
124
|
+
export function createBoardMirror({
|
|
125
|
+
store,
|
|
126
|
+
log = () => {},
|
|
127
|
+
backups,
|
|
128
|
+
// On a worker: home's next page after a change (a sealed call through ownMachines.mjs), or null when unreachable.
|
|
129
|
+
fetchPage = async () => null,
|
|
130
|
+
// On a worker that stood in: one page of what it wrote, merged on home; the answer says whether home took it.
|
|
131
|
+
pushPage = async () => null,
|
|
132
|
+
// Where this computer is: "home", "follow" (a worker), or "return" (home, while another computer stands in for it).
|
|
133
|
+
situation = () => "home",
|
|
134
|
+
// On a computer someone stood in for, once it has the stand-in's changes: Harness gives it home back.
|
|
135
|
+
reclaim = async () => false,
|
|
136
|
+
// Whether the computer standing in for this one can be reached (Harness has it connected, or a direct link does).
|
|
137
|
+
standInReachable = () => true,
|
|
138
|
+
// Rows another computer's changes brought in: what acts on the board hears of them.
|
|
139
|
+
merged = () => {},
|
|
140
|
+
}) {
|
|
141
|
+
const db = store.db;
|
|
142
|
+
install(db);
|
|
143
|
+
const meta = (key) =>
|
|
144
|
+
db.get("select value from board_meta where key = ?", [key])?.value ?? null;
|
|
145
|
+
const setMeta = (key, value) =>
|
|
146
|
+
db.run(
|
|
147
|
+
"insert into board_meta(key, value) values(?, ?) on conflict(key) do update set value = excluded.value",
|
|
148
|
+
[key, value]
|
|
149
|
+
);
|
|
150
|
+
const feed = () => meta(FEED);
|
|
151
|
+
// One row as the other computer has it: only the columns this board has (a newer version may have more), bytes read
|
|
152
|
+
// back from the wire, and its words kept where search reads them.
|
|
153
|
+
const writeRow = (table, row) => {
|
|
154
|
+
const known = columnsOf.get(table);
|
|
155
|
+
const names = Object.keys(row).filter((name) => known.has(name));
|
|
156
|
+
db.run(
|
|
157
|
+
`insert or replace into ${table}(${names.join(", ")}) values(${names.map(() => "?").join(", ")})`,
|
|
158
|
+
names.map((name) => offWire(row[name]))
|
|
159
|
+
);
|
|
160
|
+
const searched = SEARCHED[table];
|
|
161
|
+
if (!searched) return;
|
|
162
|
+
const words = searched.columns(row);
|
|
163
|
+
db.run(`delete from ${searched.table} where id = ?`, [words.id]);
|
|
164
|
+
db.run(
|
|
165
|
+
`insert into ${searched.table}(${Object.keys(words).join(", ")}) values(${Object.keys(
|
|
166
|
+
words
|
|
167
|
+
)
|
|
168
|
+
.map(() => "?")
|
|
169
|
+
.join(", ")})`,
|
|
170
|
+
Object.values(words)
|
|
171
|
+
);
|
|
172
|
+
};
|
|
173
|
+
const removeRow = (table, key) => {
|
|
174
|
+
const columns = MIRRORED[table];
|
|
175
|
+
db.run(`delete from ${table} where ${whereKey(columns)}`, key);
|
|
176
|
+
if (SEARCHED[table])
|
|
177
|
+
db.run(`delete from ${SEARCHED[table].table} where id = ?`, [key[0]]);
|
|
178
|
+
};
|
|
179
|
+
const head = () =>
|
|
180
|
+
Number(db.get("select max(seq) as seq from board_changes")?.seq ?? 0);
|
|
181
|
+
const columnsOf = new Map(
|
|
182
|
+
Object.keys(MIRRORED).map((table) => [
|
|
183
|
+
table,
|
|
184
|
+
new Set(db.all(`pragma table_info(${table})`).map((row) => row.name)),
|
|
185
|
+
])
|
|
186
|
+
);
|
|
187
|
+
|
|
188
|
+
/**
|
|
189
|
+
* On home: the changes after `after` in its feed, a page at a time, each a row as it is now or its removal. Asked for
|
|
190
|
+
* `standIn`, a stand-in's own changes since it stood in, for the computer it stood in for.
|
|
191
|
+
*/
|
|
192
|
+
const since = ({
|
|
193
|
+
after = 0,
|
|
194
|
+
feed: followed = null,
|
|
195
|
+
standIn = false,
|
|
196
|
+
} = {}) => {
|
|
197
|
+
const standing = standInRecord();
|
|
198
|
+
if (standIn && !standing)
|
|
199
|
+
return { ok: false, error: "this computer does not stand in" };
|
|
200
|
+
const asked = Math.max(0, Number(after) || 0);
|
|
201
|
+
// A feed numbered lower than a worker has followed (this board restored from a backup) is listed again whole.
|
|
202
|
+
const from =
|
|
203
|
+
followed === feed() && asked <= head()
|
|
204
|
+
? asked
|
|
205
|
+
: standIn
|
|
206
|
+
? standing.from
|
|
207
|
+
: 0;
|
|
208
|
+
const changes = [];
|
|
209
|
+
let bytes = 0;
|
|
210
|
+
let last = from;
|
|
211
|
+
for (const change of db.all(
|
|
212
|
+
"select seq, tbl, pk, gone from board_changes where seq > ? order by seq",
|
|
213
|
+
[from]
|
|
214
|
+
)) {
|
|
215
|
+
const columns = MIRRORED[change.tbl];
|
|
216
|
+
if (!columns) {
|
|
217
|
+
last = change.seq;
|
|
218
|
+
continue;
|
|
219
|
+
}
|
|
220
|
+
const key = JSON.parse(change.pk);
|
|
221
|
+
const row = change.gone
|
|
222
|
+
? null
|
|
223
|
+
: db.get(`select * from ${change.tbl} where ${whereKey(columns)}`, key);
|
|
224
|
+
const item = row
|
|
225
|
+
? { seq: change.seq, tbl: change.tbl, row: onWire(row) }
|
|
226
|
+
: { seq: change.seq, tbl: change.tbl, gone: key };
|
|
227
|
+
const size = Buffer.byteLength(JSON.stringify(item));
|
|
228
|
+
if (changes.length && bytes + size > PAGE_BYTES) break;
|
|
229
|
+
changes.push(item);
|
|
230
|
+
bytes += size;
|
|
231
|
+
last = change.seq;
|
|
232
|
+
}
|
|
233
|
+
return {
|
|
234
|
+
ok: true,
|
|
235
|
+
feed: feed(),
|
|
236
|
+
from,
|
|
237
|
+
to: last,
|
|
238
|
+
head: head(),
|
|
239
|
+
changes,
|
|
240
|
+
};
|
|
241
|
+
};
|
|
242
|
+
|
|
243
|
+
// Keys seen during a start-over, to take out what home does not have once it is complete.
|
|
244
|
+
let starting = null;
|
|
245
|
+
|
|
246
|
+
/** On a worker: one page from home, applied in one transaction. Returns whether more remain. */
|
|
247
|
+
const apply = (page) => {
|
|
248
|
+
// A page from the start of home's feed lists the whole board: what is not in it goes once the listing ends.
|
|
249
|
+
if (page.from === 0)
|
|
250
|
+
starting ??= new Map(
|
|
251
|
+
Object.keys(MIRRORED).map((table) => [table, new Set()])
|
|
252
|
+
);
|
|
253
|
+
db.exec("begin immediate");
|
|
254
|
+
try {
|
|
255
|
+
for (const change of page.changes) {
|
|
256
|
+
const columns = MIRRORED[change.tbl];
|
|
257
|
+
if (!columns) continue;
|
|
258
|
+
if (change.gone) {
|
|
259
|
+
removeRow(change.tbl, change.gone);
|
|
260
|
+
continue;
|
|
261
|
+
}
|
|
262
|
+
writeRow(change.tbl, change.row);
|
|
263
|
+
starting
|
|
264
|
+
?.get(change.tbl)
|
|
265
|
+
?.add(JSON.stringify(columns.map((column) => change.row[column])));
|
|
266
|
+
}
|
|
267
|
+
const done = page.to >= page.head;
|
|
268
|
+
if (done && starting) sweep(starting);
|
|
269
|
+
setMeta(
|
|
270
|
+
CURSOR,
|
|
271
|
+
JSON.stringify({
|
|
272
|
+
feed: page.feed,
|
|
273
|
+
seq: page.to,
|
|
274
|
+
at: new Date().toISOString(),
|
|
275
|
+
// A listing cut off by a restart starts over: the keys it had seen were in memory.
|
|
276
|
+
...(starting && !done ? { listing: true } : {}),
|
|
277
|
+
})
|
|
278
|
+
);
|
|
279
|
+
db.exec("commit");
|
|
280
|
+
if (done) starting = null;
|
|
281
|
+
return !done;
|
|
282
|
+
} catch (error) {
|
|
283
|
+
if (db.inTransaction) db.exec("rollback");
|
|
284
|
+
throw error;
|
|
285
|
+
}
|
|
286
|
+
};
|
|
287
|
+
|
|
288
|
+
// After a start-over: the rows home does not have go, written first to a file in Harness's folder.
|
|
289
|
+
const sweep = (seen) => {
|
|
290
|
+
const extra = [];
|
|
291
|
+
for (const [table, columns] of Object.entries(MIRRORED))
|
|
292
|
+
for (const row of db.all(`select * from ${table}`)) {
|
|
293
|
+
const key = JSON.stringify(columns.map((column) => row[column]));
|
|
294
|
+
if (!seen.get(table).has(key)) extra.push({ table, row });
|
|
295
|
+
}
|
|
296
|
+
if (!extra.length) return;
|
|
297
|
+
fs.mkdirSync(backups, { recursive: true });
|
|
298
|
+
const file = path.join(
|
|
299
|
+
backups,
|
|
300
|
+
`board-before-mirror-${new Date().toISOString().replace(/[:.]/g, "-")}.jsonl`
|
|
301
|
+
);
|
|
302
|
+
fs.writeFileSync(
|
|
303
|
+
file,
|
|
304
|
+
extra
|
|
305
|
+
.map(({ table, row }) => JSON.stringify({ table, row: onWire(row) }))
|
|
306
|
+
.join("\n") + "\n"
|
|
307
|
+
);
|
|
308
|
+
for (const { table, row } of extra)
|
|
309
|
+
removeRow(
|
|
310
|
+
table,
|
|
311
|
+
MIRRORED[table].map((column) => row[column])
|
|
312
|
+
);
|
|
313
|
+
log(
|
|
314
|
+
`the board here now follows home's: ${extra.length} row(s) it did not have are kept in ${file}`
|
|
315
|
+
);
|
|
316
|
+
};
|
|
317
|
+
|
|
318
|
+
/** The last change applied from home, and when: what a read from the copy says it is as of. */
|
|
319
|
+
const cursor = () => JSON.parse(meta(CURSOR) || "null");
|
|
320
|
+
const standInRecord = () => JSON.parse(meta(STAND_IN) || "null");
|
|
321
|
+
|
|
322
|
+
/** This computer stands in as home for another: its own changes from here on go back to that computer. */
|
|
323
|
+
const startStandIn = (forId) =>
|
|
324
|
+
setMeta(STAND_IN, JSON.stringify({ for: forId, from: head() }));
|
|
325
|
+
|
|
326
|
+
/**
|
|
327
|
+
* Another computer's changes merged into this board, never overwriting newer work: a task by its version (every
|
|
328
|
+
* write bumps it), a lesson by when it changed, and any other row as the computer that changed it while standing in
|
|
329
|
+
* has it. Nothing here is taken out for being absent from the page.
|
|
330
|
+
*/
|
|
331
|
+
const merge = (page) => {
|
|
332
|
+
db.exec("begin immediate");
|
|
333
|
+
try {
|
|
334
|
+
for (const change of page.changes ?? []) {
|
|
335
|
+
const columns = MIRRORED[change.tbl];
|
|
336
|
+
if (!columns) continue;
|
|
337
|
+
if (change.gone) {
|
|
338
|
+
removeRow(change.tbl, change.gone);
|
|
339
|
+
continue;
|
|
340
|
+
}
|
|
341
|
+
const here = db.get(
|
|
342
|
+
`select * from ${change.tbl} where ${whereKey(columns)}`,
|
|
343
|
+
columns.map((column) => change.row[column])
|
|
344
|
+
);
|
|
345
|
+
if (here && !newer(change.row, here)) continue;
|
|
346
|
+
writeRow(change.tbl, change.row);
|
|
347
|
+
}
|
|
348
|
+
db.exec("commit");
|
|
349
|
+
merged();
|
|
350
|
+
return { ok: true };
|
|
351
|
+
} catch (error) {
|
|
352
|
+
if (db.inTransaction) db.exec("rollback");
|
|
353
|
+
throw error;
|
|
354
|
+
}
|
|
355
|
+
};
|
|
356
|
+
// A row with no version or time of change (a thread's state, a goal's sessions, a reaction) is taken from the
|
|
357
|
+
// computer that sent it: only rows it changed while standing in come in a merge (review, October 21: these edits were
|
|
358
|
+
// dropped).
|
|
359
|
+
const newer = (incoming, here) => {
|
|
360
|
+
if ("version" in incoming && "version" in here) {
|
|
361
|
+
if (Number(incoming.version) !== Number(here.version))
|
|
362
|
+
return Number(incoming.version) > Number(here.version);
|
|
363
|
+
return String(incoming.updated_at) > String(here.updated_at);
|
|
364
|
+
}
|
|
365
|
+
if ("updated_at" in incoming && "updated_at" in here)
|
|
366
|
+
return String(incoming.updated_at) > String(here.updated_at);
|
|
367
|
+
return true;
|
|
368
|
+
};
|
|
369
|
+
|
|
370
|
+
// A stand-in that is a worker again: home gets everything it wrote while standing in, before it follows home again.
|
|
371
|
+
const handOver = async () => {
|
|
372
|
+
const standing = standInRecord();
|
|
373
|
+
if (!standing) return true;
|
|
374
|
+
let after = standing.from;
|
|
375
|
+
for (;;) {
|
|
376
|
+
const page = since({ after, feed: feed() });
|
|
377
|
+
if (!page.changes.length) break;
|
|
378
|
+
const answer = await pushPage(page);
|
|
379
|
+
if (answer?.ok !== true) return false;
|
|
380
|
+
after = page.to;
|
|
381
|
+
if (page.to >= page.head) break;
|
|
382
|
+
}
|
|
383
|
+
db.run("delete from board_meta where key = ?", [STAND_IN]);
|
|
384
|
+
log(`gave home what this computer wrote while it stood in`);
|
|
385
|
+
return true;
|
|
386
|
+
};
|
|
387
|
+
|
|
388
|
+
// Home again after another computer stood in for it: that computer's changes merged here, then home taken back.
|
|
389
|
+
const takeBack = async () => {
|
|
390
|
+
let after = null;
|
|
391
|
+
let followed = null;
|
|
392
|
+
for (;;) {
|
|
393
|
+
const page = await fetchPage({ standIn: true, after, feed: followed });
|
|
394
|
+
if (!page?.ok || !Array.isArray(page.changes)) {
|
|
395
|
+
// The stand-in is away too, or kept no record of standing in: home is taken back now, and the stand-in hands
|
|
396
|
+
// its changes over when it is back (handOver), rather than the swarm having no home (review, October 21).
|
|
397
|
+
if (page?.ok === false || !standInReachable()) break;
|
|
398
|
+
return;
|
|
399
|
+
}
|
|
400
|
+
merge(page);
|
|
401
|
+
after = page.to;
|
|
402
|
+
followed = page.feed;
|
|
403
|
+
if (page.to >= page.head) break;
|
|
404
|
+
}
|
|
405
|
+
if (await reclaim()) log("took home back with the stand-in's changes");
|
|
406
|
+
};
|
|
407
|
+
|
|
408
|
+
// One catch-up at a time; asked again while it runs, it runs once more after.
|
|
409
|
+
let running = null;
|
|
410
|
+
let again = false;
|
|
411
|
+
const catchUp = async () => {
|
|
412
|
+
for (;;) {
|
|
413
|
+
const followed = cursor();
|
|
414
|
+
const fresh = followed && !(followed.listing && !starting);
|
|
415
|
+
const page = await fetchPage({
|
|
416
|
+
after: fresh ? followed.seq : 0,
|
|
417
|
+
feed: fresh ? followed.feed : null,
|
|
418
|
+
});
|
|
419
|
+
if (!page?.ok || !Array.isArray(page.changes)) return;
|
|
420
|
+
if (!apply(page)) return;
|
|
421
|
+
}
|
|
422
|
+
};
|
|
423
|
+
/** On a worker: take home's changes until this copy is up to date. */
|
|
424
|
+
const sync = () => {
|
|
425
|
+
if (running) {
|
|
426
|
+
again = true;
|
|
427
|
+
return running;
|
|
428
|
+
}
|
|
429
|
+
running = (async () => {
|
|
430
|
+
try {
|
|
431
|
+
do {
|
|
432
|
+
again = false;
|
|
433
|
+
const where = situation();
|
|
434
|
+
if (where === "home") continue;
|
|
435
|
+
if (!(await handOver())) continue;
|
|
436
|
+
if (where === "return") await takeBack();
|
|
437
|
+
else await catchUp();
|
|
438
|
+
} while (again);
|
|
439
|
+
} catch (error) {
|
|
440
|
+
log(`the board copy could not catch up with home: ${error.message}`);
|
|
441
|
+
} finally {
|
|
442
|
+
running = null;
|
|
443
|
+
}
|
|
444
|
+
})();
|
|
445
|
+
return running;
|
|
446
|
+
};
|
|
447
|
+
|
|
448
|
+
return {
|
|
449
|
+
since,
|
|
450
|
+
apply,
|
|
451
|
+
merge,
|
|
452
|
+
sync,
|
|
453
|
+
cursor,
|
|
454
|
+
feed,
|
|
455
|
+
head,
|
|
456
|
+
startStandIn,
|
|
457
|
+
standIn: standInRecord,
|
|
458
|
+
};
|
|
459
|
+
}
|