tryharness 0.7.45 → 0.7.53
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +232 -12
- package/cli.mjs +406 -184
- package/lib/actorRoster.mjs +28 -5
- package/lib/adapters.mjs +9 -1
- package/lib/agentRuntimeConfig.mjs +36 -0
- package/lib/agentRuntimes.mjs +560 -0
- package/lib/atlas.mjs +7 -2
- package/lib/boardCommands.mjs +5 -10
- package/lib/boardFileCommands.mjs +2 -1
- package/lib/boardMirror.mjs +459 -0
- package/lib/boardProtocol.json +206 -6
- package/lib/boardTools.mjs +2 -6
- package/lib/bridge.mjs +25 -3
- package/lib/browserWake.mjs +60 -0
- package/lib/cards.mjs +35 -2
- package/lib/commands.mjs +20 -5
- package/lib/connectionTools.d.ts +13 -0
- package/lib/connectionTools.js +45 -0
- package/lib/connectorAuth.mjs +309 -0
- package/lib/connectorCatalog.json +41 -0
- package/lib/connectorCommands.mjs +121 -0
- package/lib/connectorGoogle.mjs +513 -0
- package/lib/connectors.mjs +805 -0
- package/lib/coordinator.mjs +8 -1
- package/lib/coreModels.mjs +78 -0
- package/lib/daemon.mjs +643 -45
- package/lib/desktop.mjs +7 -2
- package/lib/digest.mjs +32 -0
- package/lib/directLink.mjs +283 -0
- package/lib/environments.mjs +595 -0
- package/lib/folders.mjs +2 -2
- package/lib/goalCommands.mjs +58 -14
- package/lib/inference.mjs +263 -152
- package/lib/lessons.mjs +6 -0
- package/lib/lineup.mjs +324 -0
- package/lib/localmodels.mjs +10 -0
- package/lib/machineIdentity.mjs +3 -0
- package/lib/machineRoutes.mjs +40 -2
- package/lib/machineRuns.mjs +30 -3
- package/lib/machineSeal.mjs +69 -10
- package/lib/mcp-stdio.mjs +121 -118
- package/lib/mcpRelay.mjs +119 -0
- package/lib/members.json +12 -0
- package/lib/members.mjs +33 -0
- package/lib/modelMemory.mjs +100 -0
- package/lib/modelTiers.json +1183 -0
- package/lib/modelTiers.mjs +133 -0
- package/lib/models.mjs +41 -8
- package/lib/museSession.mjs +119 -0
- package/lib/nativeHost.mjs +203 -0
- package/lib/network.mjs +7 -2
- package/lib/nudge.mjs +10 -2
- package/lib/onboard.mjs +430 -1
- package/lib/oneclawLocal.mjs +150 -0
- package/lib/ownMachines.mjs +521 -36
- package/lib/pairing.mjs +165 -0
- package/lib/paths.mjs +4 -0
- package/lib/phones.mjs +61 -0
- package/lib/placeCommands.mjs +27 -0
- package/lib/places.mjs +10 -2
- package/lib/plans.mjs +151 -27
- package/lib/profile.mjs +152 -0
- package/lib/projects.mjs +624 -137
- package/lib/readiness.mjs +283 -0
- package/lib/repoLinks.mjs +354 -0
- package/lib/roomChannels.mjs +641 -0
- package/lib/runBrief.mjs +32 -2
- package/lib/runRequests.mjs +157 -0
- package/lib/runWorker.mjs +20 -7
- package/lib/runner.mjs +67 -8
- package/lib/secretPatterns.json +55 -0
- package/lib/secrets.mjs +10 -27
- package/lib/seen.mjs +102 -0
- package/lib/service.mjs +11 -6
- package/lib/sessionControl.mjs +10 -8
- package/lib/sessionIndex.mjs +5 -3
- package/lib/store.mjs +546 -29
- package/lib/subagents.mjs +8 -4
- package/lib/swarm.mjs +367 -37
- package/lib/swarmMemory.mjs +947 -0
- package/lib/t3.mjs +43 -13
- package/lib/taskCommands.mjs +56 -5
- package/lib/taskRuns.mjs +61 -11
- package/lib/team.json +17 -15
- package/lib/team.mjs +68 -18
- package/lib/threadRuns.mjs +3 -1
- package/lib/threadView.mjs +51 -14
- package/lib/tokenizer.mjs +405 -0
- package/lib/ui.mjs +16 -2
- package/lib/usageTotals.mjs +148 -0
- package/lib/vault.mjs +1144 -0
- package/lib/vaultCloud.mjs +117 -0
- package/lib/vaultCommands.mjs +201 -0
- package/lib/vaultTools.d.ts +12 -0
- package/lib/vaultTools.js +42 -0
- package/lib/windowAsks.mjs +42 -0
- package/lib/workFocus.mjs +59 -0
- package/lib/workLinks.mjs +90 -0
- package/package.json +9 -4
package/lib/threadView.mjs
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { nameOf } from "./members.mjs";
|
|
2
|
+
import { onMachine, withComputer } from "./machineRoutes.mjs";
|
|
2
3
|
|
|
3
4
|
// A thread as the person reads it, worked out once here so the buddy, the phone, the PiP, the dashboard and the
|
|
4
5
|
// notices all say the same thing: its title, its state in one word, one line on where it stands, what needs the
|
|
@@ -72,6 +73,9 @@ function headlineOf(text) {
|
|
|
72
73
|
return shorten(sentence ?? line, 160);
|
|
73
74
|
}
|
|
74
75
|
|
|
76
|
+
// An allowed task the browser has not taken by then is the person's again: Allow once more, or open Harness.
|
|
77
|
+
const BROWSER_START_MS = 2 * 60_000;
|
|
78
|
+
|
|
75
79
|
export function createThreadView({
|
|
76
80
|
store,
|
|
77
81
|
goals = null,
|
|
@@ -79,7 +83,19 @@ export function createThreadView({
|
|
|
79
83
|
now = Date.now,
|
|
80
84
|
// What a shared project's room waits on the person for (projects.mjs needsOf): an OK, a post to share.
|
|
81
85
|
projectNeeds = () => [],
|
|
86
|
+
nativeNeeds = () => [],
|
|
87
|
+
// A run's agent asking for one of the person's secrets (vault.mjs).
|
|
88
|
+
vaultNeeds = () => [],
|
|
89
|
+
// The name of the computer an agent wrote from, once the person has more than one (ownMachines.mjs shownLabel).
|
|
90
|
+
computerOf = () => null,
|
|
91
|
+
// Whether the person's browser is listening for this computer (browserWake.mjs), so Harness can use it now.
|
|
92
|
+
browserListening = () => false,
|
|
82
93
|
}) {
|
|
94
|
+
// Words from an agent on the person's computers say which one: the buddy reads it "Codex (linux)".
|
|
95
|
+
const writtenOn = (post) => {
|
|
96
|
+
const { computer } = withComputer(post, computerOf);
|
|
97
|
+
return computer ? { computer } : {};
|
|
98
|
+
};
|
|
83
99
|
const coordinated = (post) =>
|
|
84
100
|
Boolean(store.meta(`coordinator-post:${post.id}`));
|
|
85
101
|
const filing = (threadId) => {
|
|
@@ -133,6 +149,8 @@ export function createThreadView({
|
|
|
133
149
|
function needsYouOf(thread, posts, runs) {
|
|
134
150
|
const items = [];
|
|
135
151
|
for (const run of runs) {
|
|
152
|
+
items.push(...nativeNeeds(run.id));
|
|
153
|
+
items.push(...vaultNeeds(run.id));
|
|
136
154
|
if (run.status === "needs_input" && run.holder === "user")
|
|
137
155
|
items.push({
|
|
138
156
|
id: `answer:${run.id}`,
|
|
@@ -201,21 +219,35 @@ export function createThreadView({
|
|
|
201
219
|
at: thread.updated_at,
|
|
202
220
|
});
|
|
203
221
|
// What Harness's server left for the browser waits on the person opening their Harness window (October 18: one
|
|
204
|
-
// such part waited 9 hours with nothing saying so)
|
|
205
|
-
|
|
222
|
+
// such part waited 9 hours with nothing saying so), or on their Allow when their browser listens for this computer
|
|
223
|
+
// (October 22: "right away, even if away"). Once allowed it waits on no one, unless the browser has not taken it.
|
|
224
|
+
for (const run of runs) {
|
|
206
225
|
if (
|
|
207
|
-
run.target_agent
|
|
208
|
-
run.status
|
|
209
|
-
store.meta(`harness-window:${run.id}`)
|
|
226
|
+
run.target_agent !== "harness" ||
|
|
227
|
+
run.status !== "open" ||
|
|
228
|
+
!store.meta(`harness-window:${run.id}`)
|
|
210
229
|
)
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
230
|
+
continue;
|
|
231
|
+
const allowedAt = Date.parse(
|
|
232
|
+
store.meta(`harness-browser-allowed:${run.id}`) || ""
|
|
233
|
+
);
|
|
234
|
+
if (now() - allowedAt < BROWSER_START_MS) continue;
|
|
235
|
+
const stopped = store.meta(`harness-browser-stopped:${run.id}`);
|
|
236
|
+
const canAllow = !stopped && browserListening();
|
|
237
|
+
items.push({
|
|
238
|
+
id: `window:${run.id}`,
|
|
239
|
+
kind: "window",
|
|
240
|
+
agent: "harness",
|
|
241
|
+
text: stopped
|
|
242
|
+
? `Harness stopped in your browser to ask you: ${stopped}. Open Harness on your computer to answer.`
|
|
243
|
+
: canAllow
|
|
244
|
+
? "Harness needs your browser to finish this. Allow it, and Harness carries on in your browser now."
|
|
245
|
+
: "Harness needs your browser to finish this: open Harness on your computer.",
|
|
246
|
+
runId: run.id,
|
|
247
|
+
...(canAllow ? { browser: "can-allow" } : {}),
|
|
248
|
+
at: run.updated_at,
|
|
249
|
+
});
|
|
250
|
+
}
|
|
219
251
|
for (const held of store.heldPosts(thread.id))
|
|
220
252
|
if (!store.meta(`held-replaced:${held.id}`))
|
|
221
253
|
items.push({
|
|
@@ -276,7 +308,9 @@ export function createThreadView({
|
|
|
276
308
|
: thread.goal.paused_reason || "Stopped";
|
|
277
309
|
const last = lastWords(posts);
|
|
278
310
|
if (!last) return "Nothing yet";
|
|
279
|
-
const
|
|
311
|
+
const { computer } = writtenOn(last);
|
|
312
|
+
const who =
|
|
313
|
+
last.author === "user" ? "You" : onMachine(whoIs(last.author), computer);
|
|
280
314
|
return `${who}: ${shorten(withoutMentions(firstLine(last.text)), 140)}`;
|
|
281
315
|
}
|
|
282
316
|
|
|
@@ -360,6 +394,7 @@ export function createThreadView({
|
|
|
360
394
|
type: "file",
|
|
361
395
|
seq: post.seq,
|
|
362
396
|
author: post.author,
|
|
397
|
+
...writtenOn(post),
|
|
363
398
|
file: post.file,
|
|
364
399
|
at: post.created_at,
|
|
365
400
|
};
|
|
@@ -377,6 +412,7 @@ export function createThreadView({
|
|
|
377
412
|
type: "message",
|
|
378
413
|
seq: post.seq,
|
|
379
414
|
author: post.author,
|
|
415
|
+
...writtenOn(post),
|
|
380
416
|
text: post.text,
|
|
381
417
|
result: false,
|
|
382
418
|
asks: false,
|
|
@@ -397,6 +433,7 @@ export function createThreadView({
|
|
|
397
433
|
type: "message",
|
|
398
434
|
seq: post.seq,
|
|
399
435
|
author: post.author,
|
|
436
|
+
...writtenOn(post),
|
|
400
437
|
text: post.text,
|
|
401
438
|
result: post.kind === "result" || post.kind === "answer",
|
|
402
439
|
asks: post.kind === "question",
|
|
@@ -0,0 +1,405 @@
|
|
|
1
|
+
import fs from "node:fs";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
|
|
4
|
+
// A compact BPE tokenizer for the guards' vocabulary (Gemma's: 256,000 tokens, 580,604 merges). @huggingface/tokenizers
|
|
5
|
+
// keeps the parsed tokenizer.json and a string-keyed map of every merge, about 300 MB for this vocabulary; here the merges
|
|
6
|
+
// are a table of integer pairs, about 21 MB in all. It reproduces that library's ids exactly, including where it departs
|
|
7
|
+
// from the Rust tokenizer (its Metaspace step does not split a section into words, so a whole section is one BPE word),
|
|
8
|
+
// because the guards' thresholds were calibrated on its ids. Any tokenizer.json of another shape gets null, and the
|
|
9
|
+
// caller uses the library instead. Parsing the 34 MB file leaves about 200 MB for the garbage collector, so the first
|
|
10
|
+
// build saves the tables (about 7 MB) and later builds read those.
|
|
11
|
+
|
|
12
|
+
const REPLACEMENT = "\u2581";
|
|
13
|
+
// The saved tables' format; a file of another format is rebuilt.
|
|
14
|
+
const MAGIC = "HTK1";
|
|
15
|
+
|
|
16
|
+
/** The tokenizer for a tokenizer.json file, or null when it is not the shape this reproduces. */
|
|
17
|
+
export function loadCompactTokenizer(file, tablesFile) {
|
|
18
|
+
const saved = readTables(tablesFile);
|
|
19
|
+
if (saved) return encoderFrom(saved);
|
|
20
|
+
const json = JSON.parse(fs.readFileSync(file, "utf8"));
|
|
21
|
+
if (!supported(json)) return null;
|
|
22
|
+
const tables = tablesOf(json);
|
|
23
|
+
writeTables(tablesFile, tables);
|
|
24
|
+
return encoderFrom(tables);
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/** The tokenizer for a parsed tokenizer.json, or null when it is not the shape this reproduces. */
|
|
28
|
+
export function compactTokenizer(json) {
|
|
29
|
+
if (!supported(json)) return null;
|
|
30
|
+
return encoderFrom(tablesOf(json));
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/** Everything encoding needs, as plain data that holds nothing of the parsed file. */
|
|
34
|
+
function tablesOf(json) {
|
|
35
|
+
const { model } = json;
|
|
36
|
+
const addedIds = new Map(
|
|
37
|
+
json.added_tokens.map((token) => [token.content, token.id])
|
|
38
|
+
);
|
|
39
|
+
const idOf = (content) => addedIds.get(content) ?? model.vocab[content];
|
|
40
|
+
const characters = Object.keys(model.vocab)
|
|
41
|
+
.filter((content) => [...content].length === 1)
|
|
42
|
+
.map((content) => [content, idOf(content)]);
|
|
43
|
+
return {
|
|
44
|
+
added: json.added_tokens.map(({ content, id, lstrip, rstrip }) => ({
|
|
45
|
+
content,
|
|
46
|
+
id,
|
|
47
|
+
lstrip: Boolean(lstrip),
|
|
48
|
+
rstrip: Boolean(rstrip),
|
|
49
|
+
})),
|
|
50
|
+
characters,
|
|
51
|
+
byteIds: Array.from({ length: 256 }, (_, byte) =>
|
|
52
|
+
idOf(`<0x${byte.toString(16).toUpperCase().padStart(2, "0")}>`)
|
|
53
|
+
),
|
|
54
|
+
unknownId: idOf(model.unk_token),
|
|
55
|
+
template: templateOf(json.post_processor, idOf),
|
|
56
|
+
merges: mergeIds(model.merges, idOf),
|
|
57
|
+
};
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/** The tables as one file: the magic, the plain parts as JSON, then the merges' three columns of ids. */
|
|
61
|
+
function writeTables(file, { merges, ...plain }) {
|
|
62
|
+
const header = Buffer.from(
|
|
63
|
+
JSON.stringify({ ...plain, mergeCount: merges.results.length })
|
|
64
|
+
);
|
|
65
|
+
const padding = (4 - ((MAGIC.length + 4 + header.length) % 4)) % 4;
|
|
66
|
+
const length = Buffer.alloc(4);
|
|
67
|
+
length.writeUInt32LE(header.length);
|
|
68
|
+
const columns = [merges.lefts, merges.rights, merges.results].map((column) =>
|
|
69
|
+
Buffer.from(column.buffer, column.byteOffset, column.byteLength)
|
|
70
|
+
);
|
|
71
|
+
fs.mkdirSync(path.dirname(file), { recursive: true });
|
|
72
|
+
const partial = `${file}.partial`;
|
|
73
|
+
fs.writeFileSync(
|
|
74
|
+
partial,
|
|
75
|
+
Buffer.concat([
|
|
76
|
+
Buffer.from(MAGIC),
|
|
77
|
+
length,
|
|
78
|
+
header,
|
|
79
|
+
Buffer.alloc(padding),
|
|
80
|
+
...columns,
|
|
81
|
+
])
|
|
82
|
+
);
|
|
83
|
+
fs.renameSync(partial, file);
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
function readTables(file) {
|
|
87
|
+
let bytes;
|
|
88
|
+
try {
|
|
89
|
+
bytes = fs.readFileSync(file);
|
|
90
|
+
} catch {
|
|
91
|
+
return null;
|
|
92
|
+
}
|
|
93
|
+
try {
|
|
94
|
+
if (bytes.subarray(0, MAGIC.length).toString() !== MAGIC) return null;
|
|
95
|
+
const headerLength = bytes.readUInt32LE(MAGIC.length);
|
|
96
|
+
const headerStart = MAGIC.length + 4;
|
|
97
|
+
const { mergeCount, ...plain } = JSON.parse(
|
|
98
|
+
bytes.subarray(headerStart, headerStart + headerLength).toString()
|
|
99
|
+
);
|
|
100
|
+
const columnsStart =
|
|
101
|
+
headerStart +
|
|
102
|
+
headerLength +
|
|
103
|
+
((4 - ((headerStart + headerLength) % 4)) % 4);
|
|
104
|
+
const column = (index) => {
|
|
105
|
+
const start = columnsStart + index * mergeCount * 4;
|
|
106
|
+
return new Int32Array(
|
|
107
|
+
bytes.buffer.slice(
|
|
108
|
+
bytes.byteOffset + start,
|
|
109
|
+
bytes.byteOffset + start + mergeCount * 4
|
|
110
|
+
)
|
|
111
|
+
);
|
|
112
|
+
};
|
|
113
|
+
if (bytes.length !== columnsStart + 3 * mergeCount * 4) return null;
|
|
114
|
+
return {
|
|
115
|
+
...plain,
|
|
116
|
+
merges: { lefts: column(0), rights: column(1), results: column(2) },
|
|
117
|
+
};
|
|
118
|
+
} catch {
|
|
119
|
+
return null;
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
function encoderFrom({
|
|
124
|
+
added: addedTokens,
|
|
125
|
+
characters,
|
|
126
|
+
byteIds,
|
|
127
|
+
unknownId,
|
|
128
|
+
template,
|
|
129
|
+
merges: mergeColumns,
|
|
130
|
+
}) {
|
|
131
|
+
const addedIds = new Map(
|
|
132
|
+
addedTokens.map((token) => [token.content, token.id])
|
|
133
|
+
);
|
|
134
|
+
const added = addedTokenSplitter(addedTokens);
|
|
135
|
+
const characterIds = new Map(characters);
|
|
136
|
+
const merges = mergeTable(mergeColumns);
|
|
137
|
+
const encoder = new TextEncoder();
|
|
138
|
+
const fallbackIds = (character) => {
|
|
139
|
+
const ids = Array.from(encoder.encode(character), (byte) => byteIds[byte]);
|
|
140
|
+
return ids.every((id) => id !== undefined && id !== null)
|
|
141
|
+
? ids
|
|
142
|
+
: [unknownId];
|
|
143
|
+
};
|
|
144
|
+
const wordIds = (word) =>
|
|
145
|
+
bpe(word, characterIds, merges).flatMap((symbol) =>
|
|
146
|
+
typeof symbol === "number" ? [symbol] : fallbackIds(symbol)
|
|
147
|
+
);
|
|
148
|
+
|
|
149
|
+
const sectionIds = (section) => {
|
|
150
|
+
if (addedIds.has(section)) return [addedIds.get(section)];
|
|
151
|
+
const normalized = section.replaceAll(" ", REPLACEMENT);
|
|
152
|
+
if (addedIds.has(normalized)) return [addedIds.get(normalized)];
|
|
153
|
+
const word = normalized.startsWith(REPLACEMENT)
|
|
154
|
+
? normalized
|
|
155
|
+
: REPLACEMENT + normalized;
|
|
156
|
+
return wordIds(word);
|
|
157
|
+
};
|
|
158
|
+
const textIds = (text) =>
|
|
159
|
+
added
|
|
160
|
+
.split(text)
|
|
161
|
+
.filter((section) => section.length > 0)
|
|
162
|
+
.flatMap(sectionIds);
|
|
163
|
+
|
|
164
|
+
return {
|
|
165
|
+
/** { ids } as @huggingface/tokenizers' Tokenizer.encode gives them. */
|
|
166
|
+
encode(text, { text_pair = null, add_special_tokens = true } = {}) {
|
|
167
|
+
const first = textIds(text);
|
|
168
|
+
const second = text_pair === null ? null : textIds(text_pair);
|
|
169
|
+
const steps = second === null ? template.single : template.pair;
|
|
170
|
+
const ids = steps.flatMap((step) => {
|
|
171
|
+
if (step.special !== undefined)
|
|
172
|
+
return add_special_tokens ? [step.special] : [];
|
|
173
|
+
return step.sequence === "A" ? first : second;
|
|
174
|
+
});
|
|
175
|
+
return { ids };
|
|
176
|
+
},
|
|
177
|
+
};
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
/** The post-processor's templates with their special tokens as ids. */
|
|
181
|
+
function templateOf(postProcessor, idOf) {
|
|
182
|
+
const steps = (template) =>
|
|
183
|
+
template.map((step) =>
|
|
184
|
+
"SpecialToken" in step
|
|
185
|
+
? { special: idOf(step.SpecialToken.id) }
|
|
186
|
+
: { sequence: step.Sequence.id }
|
|
187
|
+
);
|
|
188
|
+
return {
|
|
189
|
+
single: steps(postProcessor.single),
|
|
190
|
+
pair: steps(postProcessor.pair),
|
|
191
|
+
};
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
function supported(json) {
|
|
195
|
+
const { model, normalizer, pre_tokenizer, post_processor } = json ?? {};
|
|
196
|
+
return (
|
|
197
|
+
model?.type === "BPE" &&
|
|
198
|
+
model.byte_fallback === true &&
|
|
199
|
+
!model.dropout &&
|
|
200
|
+
!model.end_of_word_suffix &&
|
|
201
|
+
!model.continuing_subword_suffix &&
|
|
202
|
+
!model.ignore_merges &&
|
|
203
|
+
Array.isArray(model.merges?.[0]) &&
|
|
204
|
+
normalizer?.type === "Replace" &&
|
|
205
|
+
normalizer.pattern?.String === " " &&
|
|
206
|
+
normalizer.content === REPLACEMENT &&
|
|
207
|
+
pre_tokenizer?.type === "Metaspace" &&
|
|
208
|
+
pre_tokenizer.replacement === REPLACEMENT &&
|
|
209
|
+
pre_tokenizer.prepend_scheme === "always" &&
|
|
210
|
+
!pre_tokenizer.str_rep &&
|
|
211
|
+
post_processor?.type === "TemplateProcessing" &&
|
|
212
|
+
json.added_tokens.every((token) => token.normalized === false)
|
|
213
|
+
);
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
/** Each merge's two parts and its result as ids, in rank order. */
|
|
217
|
+
function mergeIds(merges, idOf) {
|
|
218
|
+
const lefts = new Int32Array(merges.length);
|
|
219
|
+
const rights = new Int32Array(merges.length);
|
|
220
|
+
const results = new Int32Array(merges.length);
|
|
221
|
+
merges.forEach(([left, right], rank) => {
|
|
222
|
+
lefts[rank] = idOf(left);
|
|
223
|
+
rights[rank] = idOf(right);
|
|
224
|
+
results[rank] = idOf(left + right);
|
|
225
|
+
});
|
|
226
|
+
return { lefts, rights, results };
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
/**
|
|
230
|
+
* The merges as an open-addressing table keyed by the pair's two ids: rank(left, right) is the merge's priority (lower
|
|
231
|
+
* merges first) or -1, and result(rank) the merged token's id. Built from ids alone, so it holds nothing of the file.
|
|
232
|
+
*/
|
|
233
|
+
function mergeTable({ lefts, rights, results }) {
|
|
234
|
+
let capacity = 1;
|
|
235
|
+
while (capacity < results.length * 1.8) capacity *= 2;
|
|
236
|
+
const mask = capacity - 1;
|
|
237
|
+
const slotLefts = new Int32Array(capacity);
|
|
238
|
+
const slotRights = new Int32Array(capacity);
|
|
239
|
+
const slotRanks = new Int32Array(capacity).fill(-1);
|
|
240
|
+
const slotOf = (left, right) =>
|
|
241
|
+
(Math.imul(left, 0x9e3779b1) ^ Math.imul(right, 0x85ebca77)) & mask;
|
|
242
|
+
const find = (left, right) => {
|
|
243
|
+
let slot = slotOf(left, right);
|
|
244
|
+
while (
|
|
245
|
+
slotRanks[slot] !== -1 &&
|
|
246
|
+
!(slotLefts[slot] === left && slotRights[slot] === right)
|
|
247
|
+
)
|
|
248
|
+
slot = (slot + 1) & mask;
|
|
249
|
+
return slot;
|
|
250
|
+
};
|
|
251
|
+
|
|
252
|
+
// A pair listed twice keeps its later rank, as the library's Map built from the list does.
|
|
253
|
+
for (let rank = 0; rank < results.length; rank += 1) {
|
|
254
|
+
const slot = find(lefts[rank], rights[rank]);
|
|
255
|
+
slotLefts[slot] = lefts[rank];
|
|
256
|
+
slotRights[slot] = rights[rank];
|
|
257
|
+
slotRanks[slot] = rank;
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
return {
|
|
261
|
+
rank: (left, right) => slotRanks[find(left, right)],
|
|
262
|
+
result: (rank) => results[rank],
|
|
263
|
+
};
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
/**
|
|
267
|
+
* One word's symbols after every merge: ids, and the characters outside the vocabulary as strings (they never merge). The
|
|
268
|
+
* lowest-ranked pair merges first, the leftmost of equal pairs first, as the library's queue orders them.
|
|
269
|
+
*/
|
|
270
|
+
function bpe(word, characterIds, merges) {
|
|
271
|
+
const symbols = Array.from(word, (character) =>
|
|
272
|
+
characterIds.has(character) ? characterIds.get(character) : character
|
|
273
|
+
);
|
|
274
|
+
const next = symbols.map((_, i) => (i + 1 < symbols.length ? i + 1 : -1));
|
|
275
|
+
const previous = symbols.map((_, i) => i - 1);
|
|
276
|
+
const alive = symbols.map(() => true);
|
|
277
|
+
const queue = new PairQueue();
|
|
278
|
+
|
|
279
|
+
const consider = (left) => {
|
|
280
|
+
const right = next[left];
|
|
281
|
+
if (right === -1) return;
|
|
282
|
+
const [leftId, rightId] = [symbols[left], symbols[right]];
|
|
283
|
+
if (typeof leftId !== "number" || typeof rightId !== "number") return;
|
|
284
|
+
const rank = merges.rank(leftId, rightId);
|
|
285
|
+
if (rank !== -1) queue.push({ rank, left, right, leftId, rightId });
|
|
286
|
+
};
|
|
287
|
+
const stillThere = (pair) =>
|
|
288
|
+
alive[pair.left] &&
|
|
289
|
+
alive[pair.right] &&
|
|
290
|
+
next[pair.left] === pair.right &&
|
|
291
|
+
symbols[pair.left] === pair.leftId &&
|
|
292
|
+
symbols[pair.right] === pair.rightId;
|
|
293
|
+
|
|
294
|
+
for (let i = 0; i < symbols.length; i += 1) consider(i);
|
|
295
|
+
while (queue.size > 0) {
|
|
296
|
+
const pair = queue.pop();
|
|
297
|
+
if (!stillThere(pair)) continue;
|
|
298
|
+
symbols[pair.left] = merges.result(pair.rank);
|
|
299
|
+
alive[pair.right] = false;
|
|
300
|
+
next[pair.left] = next[pair.right];
|
|
301
|
+
if (next[pair.left] !== -1) previous[next[pair.left]] = pair.left;
|
|
302
|
+
if (previous[pair.left] !== -1) consider(previous[pair.left]);
|
|
303
|
+
consider(pair.left);
|
|
304
|
+
}
|
|
305
|
+
return symbols.filter((_, i) => alive[i]);
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
/** A binary min-heap of candidate merges, by rank and then by the left symbol's place in the word. */
|
|
309
|
+
class PairQueue {
|
|
310
|
+
#heap = [];
|
|
311
|
+
get size() {
|
|
312
|
+
return this.#heap.length;
|
|
313
|
+
}
|
|
314
|
+
#before(a, b) {
|
|
315
|
+
return a.rank < b.rank || (a.rank === b.rank && a.left < b.left);
|
|
316
|
+
}
|
|
317
|
+
push(pair) {
|
|
318
|
+
const heap = this.#heap;
|
|
319
|
+
heap.push(pair);
|
|
320
|
+
let i = heap.length - 1;
|
|
321
|
+
while (i > 0) {
|
|
322
|
+
const parent = (i - 1) >> 1;
|
|
323
|
+
if (!this.#before(heap[i], heap[parent])) break;
|
|
324
|
+
[heap[i], heap[parent]] = [heap[parent], heap[i]];
|
|
325
|
+
i = parent;
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
pop() {
|
|
329
|
+
const heap = this.#heap;
|
|
330
|
+
const top = heap[0];
|
|
331
|
+
const last = heap.pop();
|
|
332
|
+
if (heap.length > 0) {
|
|
333
|
+
heap[0] = last;
|
|
334
|
+
let i = 0;
|
|
335
|
+
for (;;) {
|
|
336
|
+
const left = 2 * i + 1;
|
|
337
|
+
const right = left + 1;
|
|
338
|
+
let smallest = i;
|
|
339
|
+
if (left < heap.length && this.#before(heap[left], heap[smallest]))
|
|
340
|
+
smallest = left;
|
|
341
|
+
if (right < heap.length && this.#before(heap[right], heap[smallest]))
|
|
342
|
+
smallest = right;
|
|
343
|
+
if (smallest === i) break;
|
|
344
|
+
[heap[i], heap[smallest]] = [heap[smallest], heap[i]];
|
|
345
|
+
i = smallest;
|
|
346
|
+
}
|
|
347
|
+
}
|
|
348
|
+
return top;
|
|
349
|
+
}
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
/**
|
|
353
|
+
* Splits text around added tokens the way the library's DictionarySplitter does (the longest token starting at each
|
|
354
|
+
* place, scanning left to right, by UTF-16 unit), and applies their lstrip and rstrip to the neighbouring text.
|
|
355
|
+
*/
|
|
356
|
+
function addedTokenSplitter(addedTokens) {
|
|
357
|
+
const byContent = new Map(addedTokens.map((token) => [token.content, token]));
|
|
358
|
+
const trie = new Map();
|
|
359
|
+
for (const token of addedTokens) {
|
|
360
|
+
let node = trie;
|
|
361
|
+
for (const unit of token.content) {
|
|
362
|
+
if (!node.has(unit)) node.set(unit, new Map());
|
|
363
|
+
node = node.get(unit);
|
|
364
|
+
}
|
|
365
|
+
node.token = token;
|
|
366
|
+
}
|
|
367
|
+
const longestAt = (text, start) => {
|
|
368
|
+
let node = trie;
|
|
369
|
+
let match = null;
|
|
370
|
+
for (let i = start; i < text.length; i += 1) {
|
|
371
|
+
node = node.get(text[i]);
|
|
372
|
+
if (!node) break;
|
|
373
|
+
if (node.token) match = node.token;
|
|
374
|
+
}
|
|
375
|
+
return match;
|
|
376
|
+
};
|
|
377
|
+
return {
|
|
378
|
+
split(text) {
|
|
379
|
+
const pieces = [];
|
|
380
|
+
let start = 0;
|
|
381
|
+
let i = 0;
|
|
382
|
+
while (i < text.length) {
|
|
383
|
+
const token = longestAt(text, i);
|
|
384
|
+
if (!token) {
|
|
385
|
+
i += 1;
|
|
386
|
+
continue;
|
|
387
|
+
}
|
|
388
|
+
if (i > start) pieces.push(text.slice(start, i));
|
|
389
|
+
pieces.push(token.content);
|
|
390
|
+
i += token.content.length;
|
|
391
|
+
start = i;
|
|
392
|
+
}
|
|
393
|
+
if (start < text.length) pieces.push(text.slice(start));
|
|
394
|
+
// As the library does, a piece is a token by its text, and strips its neighbours in order.
|
|
395
|
+
pieces.forEach((piece, index) => {
|
|
396
|
+
const token = byContent.get(piece);
|
|
397
|
+
if (token?.lstrip && index > 0)
|
|
398
|
+
pieces[index - 1] = pieces[index - 1].trimEnd();
|
|
399
|
+
if (token?.rstrip && index < pieces.length - 1)
|
|
400
|
+
pieces[index + 1] = pieces[index + 1].trimStart();
|
|
401
|
+
});
|
|
402
|
+
return pieces;
|
|
403
|
+
},
|
|
404
|
+
};
|
|
405
|
+
}
|
package/lib/ui.mjs
CHANGED
|
@@ -87,8 +87,12 @@ export function createUi({
|
|
|
87
87
|
info: (text) => p.log.info(fit(text)),
|
|
88
88
|
warn: (text) => p.log.warn(fit(text)),
|
|
89
89
|
message: (text) => p.log.message(fit(text)),
|
|
90
|
-
note
|
|
90
|
+
// A boxed note draws its own border, so its lines wrap a little narrower than the rail's.
|
|
91
|
+
note: (text, title) =>
|
|
92
|
+
p.note(wrap(text, Math.max(40, columns() - 10)).join("\n"), title),
|
|
91
93
|
select: async (opts) => answer(await p.select(opts)),
|
|
94
|
+
multiselect: async (opts) =>
|
|
95
|
+
answer(await p.multiselect({ required: false, ...opts })),
|
|
92
96
|
text: async (opts) => answer(await p.text(opts)),
|
|
93
97
|
password: async (opts) => answer(await p.password(opts)),
|
|
94
98
|
confirm: async (opts) => answer(await p.confirm(opts)),
|
|
@@ -157,6 +161,16 @@ export function scriptedUi(answers = [], lines = []) {
|
|
|
157
161
|
);
|
|
158
162
|
return next(opts.initialValue ?? opts.options[0].value);
|
|
159
163
|
},
|
|
164
|
+
multiselect: async (opts) => {
|
|
165
|
+
lines.push(
|
|
166
|
+
`? ${opts.message}`,
|
|
167
|
+
...opts.options.map(
|
|
168
|
+
(option) =>
|
|
169
|
+
` [${opts.initialValues?.includes(option.value) ? "x" : " "}] ${option.label}${option.hint ? ` (${option.hint})` : ""}`
|
|
170
|
+
)
|
|
171
|
+
);
|
|
172
|
+
return next(opts.initialValues ?? []);
|
|
173
|
+
},
|
|
160
174
|
text: ask,
|
|
161
175
|
password: async (opts) => {
|
|
162
176
|
lines.push(`? ${opts.message}`);
|
|
@@ -168,7 +182,7 @@ export function scriptedUi(answers = [], lines = []) {
|
|
|
168
182
|
},
|
|
169
183
|
spinner: () => ({
|
|
170
184
|
start: say("◒"),
|
|
171
|
-
message: ()
|
|
185
|
+
message: say("◒"),
|
|
172
186
|
stop: (text, code = 0) => say(code ? "▲" : "◇")(text),
|
|
173
187
|
}),
|
|
174
188
|
waitOrType: async ({ message, until }) => {
|