tryharness 0.7.44 → 0.7.53
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +232 -12
- package/cli.mjs +406 -184
- package/lib/actorRoster.mjs +28 -5
- package/lib/adapters.mjs +9 -1
- package/lib/agentRuntimeConfig.mjs +36 -0
- package/lib/agentRuntimes.mjs +560 -0
- package/lib/atlas.mjs +7 -2
- package/lib/boardCommands.mjs +5 -10
- package/lib/boardFileCommands.mjs +2 -1
- package/lib/boardMirror.mjs +459 -0
- package/lib/boardProtocol.json +259 -6
- package/lib/boardTools.mjs +2 -6
- package/lib/bridge.mjs +27 -3
- package/lib/browserWake.mjs +60 -0
- package/lib/cards.mjs +35 -2
- package/lib/commands.mjs +20 -5
- package/lib/connectionTools.d.ts +13 -0
- package/lib/connectionTools.js +45 -0
- package/lib/connectorAuth.mjs +309 -0
- package/lib/connectorCatalog.json +41 -0
- package/lib/connectorCommands.mjs +121 -0
- package/lib/connectorGoogle.mjs +513 -0
- package/lib/connectors.mjs +805 -0
- package/lib/coordinator.mjs +16 -2
- package/lib/coreModels.mjs +78 -0
- package/lib/daemon.mjs +775 -46
- package/lib/desktop.mjs +7 -2
- package/lib/digest.mjs +32 -0
- package/lib/directLink.mjs +283 -0
- package/lib/environments.mjs +595 -0
- package/lib/folders.mjs +2 -2
- package/lib/goalCommands.mjs +61 -12
- package/lib/inference.mjs +263 -152
- package/lib/lessons.mjs +6 -0
- package/lib/lineup.mjs +324 -0
- package/lib/localmodels.mjs +10 -0
- package/lib/machineIdentity.mjs +3 -0
- package/lib/machineRoutes.mjs +40 -2
- package/lib/machineRuns.mjs +30 -3
- package/lib/machineSeal.mjs +69 -10
- package/lib/mcp-stdio.mjs +180 -98
- package/lib/mcpRelay.mjs +119 -0
- package/lib/members.json +12 -0
- package/lib/members.mjs +33 -0
- package/lib/modelMemory.mjs +100 -0
- package/lib/modelTiers.json +1183 -0
- package/lib/modelTiers.mjs +133 -0
- package/lib/models.mjs +41 -8
- package/lib/museSession.mjs +119 -0
- package/lib/nativeHost.mjs +203 -0
- package/lib/network.mjs +38 -1
- package/lib/nudge.mjs +10 -2
- package/lib/onboard.mjs +430 -1
- package/lib/oneclawLocal.mjs +150 -0
- package/lib/ownMachines.mjs +521 -36
- package/lib/pairing.mjs +165 -0
- package/lib/paths.mjs +4 -0
- package/lib/phones.mjs +61 -0
- package/lib/placeCommands.mjs +27 -0
- package/lib/places.mjs +10 -2
- package/lib/plans.mjs +151 -27
- package/lib/profile.mjs +152 -0
- package/lib/projectGit.mjs +328 -0
- package/lib/projectSettings.mjs +120 -0
- package/lib/projects.mjs +2539 -0
- package/lib/readiness.mjs +283 -0
- package/lib/repoLinks.mjs +354 -0
- package/lib/roomChannels.mjs +641 -0
- package/lib/runBrief.mjs +32 -2
- package/lib/runRequests.mjs +157 -0
- package/lib/runWorker.mjs +20 -7
- package/lib/runner.mjs +67 -8
- package/lib/secretPatterns.json +55 -0
- package/lib/secrets.mjs +10 -27
- package/lib/seen.mjs +102 -0
- package/lib/service.mjs +11 -6
- package/lib/sessionControl.mjs +10 -8
- package/lib/sessionIndex.mjs +5 -3
- package/lib/store.mjs +559 -30
- package/lib/subagents.mjs +8 -4
- package/lib/swarm.mjs +379 -38
- package/lib/swarmMemory.mjs +947 -0
- package/lib/t3.mjs +43 -13
- package/lib/taskCommands.mjs +58 -5
- package/lib/taskRuns.mjs +61 -11
- package/lib/team.json +17 -15
- package/lib/team.mjs +68 -18
- package/lib/threadRuns.mjs +50 -7
- package/lib/threadView.mjs +68 -15
- package/lib/tokenizer.mjs +405 -0
- package/lib/ui.mjs +16 -2
- package/lib/usageTotals.mjs +148 -0
- package/lib/vault.mjs +1144 -0
- package/lib/vaultCloud.mjs +117 -0
- package/lib/vaultCommands.mjs +201 -0
- package/lib/vaultTools.d.ts +12 -0
- package/lib/vaultTools.js +42 -0
- package/lib/windowAsks.mjs +42 -0
- package/lib/workFocus.mjs +59 -0
- package/lib/workLinks.mjs +90 -0
- package/package.json +9 -4
package/lib/threadRuns.mjs
CHANGED
|
@@ -1,12 +1,17 @@
|
|
|
1
1
|
import fs from "node:fs";
|
|
2
2
|
import path from "node:path";
|
|
3
3
|
import { randomUUID } from "node:crypto";
|
|
4
|
-
import { nameOf } from "./members.mjs";
|
|
4
|
+
import { MEMBERS, nameOf } from "./members.mjs";
|
|
5
5
|
import { oneLine } from "./oneLine.mjs";
|
|
6
6
|
import { setRunPick } from "./models.mjs";
|
|
7
7
|
import { t3Access, startT3SignIn, steerT3Run } from "./t3.mjs";
|
|
8
8
|
import { t3Model, t3Open, t3Project, t3Provider } from "./places.mjs";
|
|
9
9
|
|
|
10
|
+
// The agents with a read-only switch (runner.mjs): only they talk out a clash in a shared project without waiting.
|
|
11
|
+
const REVIEWERS = new Set(
|
|
12
|
+
MEMBERS.filter((entry) => entry.review).map((entry) => entry.id)
|
|
13
|
+
);
|
|
14
|
+
|
|
10
15
|
/** Thread execution owns queued calls, fresh briefs, worktree settlement and T3 continuity. */
|
|
11
16
|
export function createThreadRuns({
|
|
12
17
|
store,
|
|
@@ -24,6 +29,8 @@ export function createThreadRuns({
|
|
|
24
29
|
// A coding agent not installed here but on another of the person's computers that has the thread's repository:
|
|
25
30
|
// { machine, folder } there (ownMachines.mjs), or null.
|
|
26
31
|
placeElsewhere = () => null,
|
|
32
|
+
// Projects shared with another person's swarm (projects.mjs): their check before work, and each agent's own branch.
|
|
33
|
+
projects = () => null,
|
|
27
34
|
}) {
|
|
28
35
|
const { RUNNERS } = authority;
|
|
29
36
|
const inT3 = (thread, agent, review) => {
|
|
@@ -232,6 +239,9 @@ export function createThreadRuns({
|
|
|
232
239
|
held = null,
|
|
233
240
|
// The asker's turn on its own task's answer (daemon.mjs threadMayRun).
|
|
234
241
|
answer = false,
|
|
242
|
+
// Past a shared project's check before work (projects.mjs gate); `wait` holds the run for the person's Start.
|
|
243
|
+
checked = false,
|
|
244
|
+
wait = false,
|
|
235
245
|
} = {}
|
|
236
246
|
) => {
|
|
237
247
|
const task = store.getTask(goalId);
|
|
@@ -289,10 +299,33 @@ export function createThreadRuns({
|
|
|
289
299
|
pauseGoal(goalId, `${agent} was woken ${goal.max_wakes} times`);
|
|
290
300
|
return;
|
|
291
301
|
}
|
|
302
|
+
// In a shared project, new work is checked against both sides' work and decisions first; a clash holds it, and the
|
|
303
|
+
// agent only talks it out (read-only) until a person OKs the fix.
|
|
304
|
+
const shared = projects()?.isProjectRoom(goalId) ? projects() : null;
|
|
305
|
+
if (shared && !checked && !recovering)
|
|
306
|
+
return (
|
|
307
|
+
shared.gate(
|
|
308
|
+
{ task, agent, reason, asker, go, mode: null, words },
|
|
309
|
+
(mode, override = null) =>
|
|
310
|
+
wakeAgent(goalId, agent, override ?? reason, {
|
|
311
|
+
review: review || (mode === "review" && REVIEWERS.has(agent)),
|
|
312
|
+
go: mode === "review" ? REVIEWERS.has(agent) : go,
|
|
313
|
+
wait:
|
|
314
|
+
mode === "wait" || (mode === "review" && !REVIEWERS.has(agent)),
|
|
315
|
+
asker,
|
|
316
|
+
pick,
|
|
317
|
+
words,
|
|
318
|
+
held,
|
|
319
|
+
answer,
|
|
320
|
+
checked: true,
|
|
321
|
+
})
|
|
322
|
+
) ?? "checking"
|
|
323
|
+
);
|
|
292
324
|
const brief = swarmBrief(task, agent, reason, { review });
|
|
293
325
|
// A coding thread in a repository works in its own worktree, so threads never share a checkout; one whose agents the
|
|
294
|
-
// person uses in T3 Code works in T3 (inT3), where T3 makes and keeps the checkout.
|
|
295
|
-
|
|
326
|
+
// person uses in T3 Code works in T3 (inT3), where T3 makes and keeps the checkout. In a shared project each agent
|
|
327
|
+
// works on its own branch here, never in T3's checkout.
|
|
328
|
+
const t3 = !elsewhere && !shared && inT3(task, agent, review);
|
|
296
329
|
// The person uses this agent in T3 Code but this turn runs here: the thread says why, once for each reason.
|
|
297
330
|
const gap = t3 || review || elsewhere ? null : t3Gap(agent, goal.workspace);
|
|
298
331
|
if (gap && !store.meta(`t3-gap-note:${goalId}:${agent}:${gap}`)) {
|
|
@@ -307,7 +340,9 @@ export function createThreadRuns({
|
|
|
307
340
|
let folder = elsewhere?.folder ?? goal.workspace;
|
|
308
341
|
if (!t3 && !elsewhere)
|
|
309
342
|
try {
|
|
310
|
-
folder =
|
|
343
|
+
folder =
|
|
344
|
+
(shared ? shared.folderFor(task, agent, { review }) : null) ??
|
|
345
|
+
worktrees.folderFor(task);
|
|
311
346
|
} catch (error) {
|
|
312
347
|
store.roomPost({
|
|
313
348
|
goalId,
|
|
@@ -344,7 +379,9 @@ export function createThreadRuns({
|
|
|
344
379
|
if (pick) setRunPick(store, wake.id, pick, asker);
|
|
345
380
|
// A review runs with the agent's own read-only switch (runner.mjs); only Claude Code and Codex have one.
|
|
346
381
|
if (review) store.meta(`review:${wake.id}`, "1");
|
|
347
|
-
if (
|
|
382
|
+
if (wait) {
|
|
383
|
+
// Waits for the person's Start; the project already said why in the thread.
|
|
384
|
+
} else if (!recovering && threadMayRun(task, { answer })) {
|
|
348
385
|
if (go) {
|
|
349
386
|
store.meta(`person-wake:${wake.id}`, "1");
|
|
350
387
|
store.authorizeGoalRun(wake.id);
|
|
@@ -417,12 +454,16 @@ export function createThreadRuns({
|
|
|
417
454
|
}
|
|
418
455
|
};
|
|
419
456
|
// The turn's changes are committed in its thread's copy as soon as it ends, before another run can start there.
|
|
420
|
-
|
|
421
|
-
|
|
457
|
+
// A shared project's agent commits on its own branch (projects.mjs); it is undefined for any other thread.
|
|
458
|
+
const commitWake = (task) => {
|
|
459
|
+
const onBranch = projects()?.committed(task);
|
|
460
|
+
if (onBranch !== undefined) return onBranch;
|
|
461
|
+
return worktrees.commitTurn(
|
|
422
462
|
task.parent_id,
|
|
423
463
|
task.id,
|
|
424
464
|
`${task.target_agent}: ${task.objective.replace(/^Room: /, "")}`
|
|
425
465
|
);
|
|
466
|
+
};
|
|
426
467
|
const settleWake = (task, kind, text, startedAt) => {
|
|
427
468
|
const goalId = task.parent_id;
|
|
428
469
|
try {
|
|
@@ -443,6 +484,8 @@ export function createThreadRuns({
|
|
|
443
484
|
author: task.target_agent,
|
|
444
485
|
kind: kind === "done" ? "post" : "failed",
|
|
445
486
|
text: kind === "done" ? text : `run failed: ${text}`,
|
|
487
|
+
// The computer the turn ran on (runWorker.mjs claims it there).
|
|
488
|
+
machine: store.getTask(task.id)?.claimed_machine ?? null,
|
|
446
489
|
});
|
|
447
490
|
const key = `${goalId}:${task.target_agent}`;
|
|
448
491
|
const pending = pendingWakes.get(key);
|
package/lib/threadView.mjs
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { nameOf } from "./members.mjs";
|
|
2
|
+
import { onMachine, withComputer } from "./machineRoutes.mjs";
|
|
2
3
|
|
|
3
4
|
// A thread as the person reads it, worked out once here so the buddy, the phone, the PiP, the dashboard and the
|
|
4
5
|
// notices all say the same thing: its title, its state in one word, one line on where it stands, what needs the
|
|
@@ -35,6 +36,13 @@ export function shorten(text, limit) {
|
|
|
35
36
|
|
|
36
37
|
const capitalized = (text) => text.charAt(0).toUpperCase() + text.slice(1);
|
|
37
38
|
|
|
39
|
+
// Who said something, as the person reads it: another person's agent in a shared project is "igor's Codex".
|
|
40
|
+
const whoIs = (name) => {
|
|
41
|
+
const [handle, agent] = String(name).split("/");
|
|
42
|
+
if (!agent) return nameOf(name);
|
|
43
|
+
return agent === "person" ? handle : `${handle}'s ${nameOf(agent)}`;
|
|
44
|
+
};
|
|
45
|
+
|
|
38
46
|
const withoutMentions = (text) =>
|
|
39
47
|
text.replace(/(^|\s)@[a-z][a-z0-9:./-]*/gi, "$1").trim();
|
|
40
48
|
|
|
@@ -65,12 +73,29 @@ function headlineOf(text) {
|
|
|
65
73
|
return shorten(sentence ?? line, 160);
|
|
66
74
|
}
|
|
67
75
|
|
|
76
|
+
// An allowed task the browser has not taken by then is the person's again: Allow once more, or open Harness.
|
|
77
|
+
const BROWSER_START_MS = 2 * 60_000;
|
|
78
|
+
|
|
68
79
|
export function createThreadView({
|
|
69
80
|
store,
|
|
70
81
|
goals = null,
|
|
71
82
|
handoffs = null,
|
|
72
83
|
now = Date.now,
|
|
84
|
+
// What a shared project's room waits on the person for (projects.mjs needsOf): an OK, a post to share.
|
|
85
|
+
projectNeeds = () => [],
|
|
86
|
+
nativeNeeds = () => [],
|
|
87
|
+
// A run's agent asking for one of the person's secrets (vault.mjs).
|
|
88
|
+
vaultNeeds = () => [],
|
|
89
|
+
// The name of the computer an agent wrote from, once the person has more than one (ownMachines.mjs shownLabel).
|
|
90
|
+
computerOf = () => null,
|
|
91
|
+
// Whether the person's browser is listening for this computer (browserWake.mjs), so Harness can use it now.
|
|
92
|
+
browserListening = () => false,
|
|
73
93
|
}) {
|
|
94
|
+
// Words from an agent on the person's computers say which one: the buddy reads it "Codex (linux)".
|
|
95
|
+
const writtenOn = (post) => {
|
|
96
|
+
const { computer } = withComputer(post, computerOf);
|
|
97
|
+
return computer ? { computer } : {};
|
|
98
|
+
};
|
|
74
99
|
const coordinated = (post) =>
|
|
75
100
|
Boolean(store.meta(`coordinator-post:${post.id}`));
|
|
76
101
|
const filing = (threadId) => {
|
|
@@ -93,6 +118,9 @@ export function createThreadView({
|
|
|
93
118
|
};
|
|
94
119
|
|
|
95
120
|
function titleOf(thread, posts) {
|
|
121
|
+
// A shared project's room keeps the project's name, the same on both people's computers.
|
|
122
|
+
if (store.meta(`project-room:${thread.id}`))
|
|
123
|
+
return shorten(thread.objective, TITLE_LIMIT);
|
|
96
124
|
const written = goals?.record(thread.id)?.title;
|
|
97
125
|
if (written) return written;
|
|
98
126
|
// A task's own thread is named by its task, not by what the person writes in it later.
|
|
@@ -121,6 +149,8 @@ export function createThreadView({
|
|
|
121
149
|
function needsYouOf(thread, posts, runs) {
|
|
122
150
|
const items = [];
|
|
123
151
|
for (const run of runs) {
|
|
152
|
+
items.push(...nativeNeeds(run.id));
|
|
153
|
+
items.push(...vaultNeeds(run.id));
|
|
124
154
|
if (run.status === "needs_input" && run.holder === "user")
|
|
125
155
|
items.push({
|
|
126
156
|
id: `answer:${run.id}`,
|
|
@@ -189,21 +219,35 @@ export function createThreadView({
|
|
|
189
219
|
at: thread.updated_at,
|
|
190
220
|
});
|
|
191
221
|
// What Harness's server left for the browser waits on the person opening their Harness window (October 18: one
|
|
192
|
-
// such part waited 9 hours with nothing saying so)
|
|
193
|
-
|
|
222
|
+
// such part waited 9 hours with nothing saying so), or on their Allow when their browser listens for this computer
|
|
223
|
+
// (October 22: "right away, even if away"). Once allowed it waits on no one, unless the browser has not taken it.
|
|
224
|
+
for (const run of runs) {
|
|
194
225
|
if (
|
|
195
|
-
run.target_agent
|
|
196
|
-
run.status
|
|
197
|
-
store.meta(`harness-window:${run.id}`)
|
|
226
|
+
run.target_agent !== "harness" ||
|
|
227
|
+
run.status !== "open" ||
|
|
228
|
+
!store.meta(`harness-window:${run.id}`)
|
|
198
229
|
)
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
230
|
+
continue;
|
|
231
|
+
const allowedAt = Date.parse(
|
|
232
|
+
store.meta(`harness-browser-allowed:${run.id}`) || ""
|
|
233
|
+
);
|
|
234
|
+
if (now() - allowedAt < BROWSER_START_MS) continue;
|
|
235
|
+
const stopped = store.meta(`harness-browser-stopped:${run.id}`);
|
|
236
|
+
const canAllow = !stopped && browserListening();
|
|
237
|
+
items.push({
|
|
238
|
+
id: `window:${run.id}`,
|
|
239
|
+
kind: "window",
|
|
240
|
+
agent: "harness",
|
|
241
|
+
text: stopped
|
|
242
|
+
? `Harness stopped in your browser to ask you: ${stopped}. Open Harness on your computer to answer.`
|
|
243
|
+
: canAllow
|
|
244
|
+
? "Harness needs your browser to finish this. Allow it, and Harness carries on in your browser now."
|
|
245
|
+
: "Harness needs your browser to finish this: open Harness on your computer.",
|
|
246
|
+
runId: run.id,
|
|
247
|
+
...(canAllow ? { browser: "can-allow" } : {}),
|
|
248
|
+
at: run.updated_at,
|
|
249
|
+
});
|
|
250
|
+
}
|
|
207
251
|
for (const held of store.heldPosts(thread.id))
|
|
208
252
|
if (!store.meta(`held-replaced:${held.id}`))
|
|
209
253
|
items.push({
|
|
@@ -214,6 +258,7 @@ export function createThreadView({
|
|
|
214
258
|
seq: held.seq,
|
|
215
259
|
at: held.created_at,
|
|
216
260
|
});
|
|
261
|
+
items.push(...projectNeeds(thread.id));
|
|
217
262
|
return items.sort((a, b) => String(a.at).localeCompare(String(b.at)));
|
|
218
263
|
}
|
|
219
264
|
|
|
@@ -263,12 +308,17 @@ export function createThreadView({
|
|
|
263
308
|
: thread.goal.paused_reason || "Stopped";
|
|
264
309
|
const last = lastWords(posts);
|
|
265
310
|
if (!last) return "Nothing yet";
|
|
266
|
-
const
|
|
311
|
+
const { computer } = writtenOn(last);
|
|
312
|
+
const who =
|
|
313
|
+
last.author === "user" ? "You" : onMachine(whoIs(last.author), computer);
|
|
267
314
|
return `${who}: ${shorten(withoutMentions(firstLine(last.text)), 140)}`;
|
|
268
315
|
}
|
|
269
316
|
|
|
270
317
|
function itemLine(item) {
|
|
271
|
-
const who =
|
|
318
|
+
const who = whoIs(item.agent);
|
|
319
|
+
if (item.kind === "agree")
|
|
320
|
+
return `${item.who ?? who} asks for your OK: ${shorten(firstLine(item.text), 140)}`;
|
|
321
|
+
if (item.kind === "share") return item.text;
|
|
272
322
|
if (item.kind === "answer")
|
|
273
323
|
return `${who} asks you: ${shorten(firstLine(item.text), 140)}`;
|
|
274
324
|
if (item.kind === "start") return `${who} waits for your Start`;
|
|
@@ -344,6 +394,7 @@ export function createThreadView({
|
|
|
344
394
|
type: "file",
|
|
345
395
|
seq: post.seq,
|
|
346
396
|
author: post.author,
|
|
397
|
+
...writtenOn(post),
|
|
347
398
|
file: post.file,
|
|
348
399
|
at: post.created_at,
|
|
349
400
|
};
|
|
@@ -361,6 +412,7 @@ export function createThreadView({
|
|
|
361
412
|
type: "message",
|
|
362
413
|
seq: post.seq,
|
|
363
414
|
author: post.author,
|
|
415
|
+
...writtenOn(post),
|
|
364
416
|
text: post.text,
|
|
365
417
|
result: false,
|
|
366
418
|
asks: false,
|
|
@@ -381,6 +433,7 @@ export function createThreadView({
|
|
|
381
433
|
type: "message",
|
|
382
434
|
seq: post.seq,
|
|
383
435
|
author: post.author,
|
|
436
|
+
...writtenOn(post),
|
|
384
437
|
text: post.text,
|
|
385
438
|
result: post.kind === "result" || post.kind === "answer",
|
|
386
439
|
asks: post.kind === "question",
|
|
@@ -0,0 +1,405 @@
|
|
|
1
|
+
import fs from "node:fs";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
|
|
4
|
+
// A compact BPE tokenizer for the guards' vocabulary (Gemma's: 256,000 tokens, 580,604 merges). @huggingface/tokenizers
|
|
5
|
+
// keeps the parsed tokenizer.json and a string-keyed map of every merge, about 300 MB for this vocabulary; here the merges
|
|
6
|
+
// are a table of integer pairs, about 21 MB in all. It reproduces that library's ids exactly, including where it departs
|
|
7
|
+
// from the Rust tokenizer (its Metaspace step does not split a section into words, so a whole section is one BPE word),
|
|
8
|
+
// because the guards' thresholds were calibrated on its ids. Any tokenizer.json of another shape gets null, and the
|
|
9
|
+
// caller uses the library instead. Parsing the 34 MB file leaves about 200 MB for the garbage collector, so the first
|
|
10
|
+
// build saves the tables (about 7 MB) and later builds read those.
|
|
11
|
+
|
|
12
|
+
const REPLACEMENT = "\u2581";
|
|
13
|
+
// The saved tables' format; a file of another format is rebuilt.
|
|
14
|
+
const MAGIC = "HTK1";
|
|
15
|
+
|
|
16
|
+
/** The tokenizer for a tokenizer.json file, or null when it is not the shape this reproduces. */
|
|
17
|
+
export function loadCompactTokenizer(file, tablesFile) {
|
|
18
|
+
const saved = readTables(tablesFile);
|
|
19
|
+
if (saved) return encoderFrom(saved);
|
|
20
|
+
const json = JSON.parse(fs.readFileSync(file, "utf8"));
|
|
21
|
+
if (!supported(json)) return null;
|
|
22
|
+
const tables = tablesOf(json);
|
|
23
|
+
writeTables(tablesFile, tables);
|
|
24
|
+
return encoderFrom(tables);
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/** The tokenizer for a parsed tokenizer.json, or null when it is not the shape this reproduces. */
|
|
28
|
+
export function compactTokenizer(json) {
|
|
29
|
+
if (!supported(json)) return null;
|
|
30
|
+
return encoderFrom(tablesOf(json));
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/** Everything encoding needs, as plain data that holds nothing of the parsed file. */
|
|
34
|
+
function tablesOf(json) {
|
|
35
|
+
const { model } = json;
|
|
36
|
+
const addedIds = new Map(
|
|
37
|
+
json.added_tokens.map((token) => [token.content, token.id])
|
|
38
|
+
);
|
|
39
|
+
const idOf = (content) => addedIds.get(content) ?? model.vocab[content];
|
|
40
|
+
const characters = Object.keys(model.vocab)
|
|
41
|
+
.filter((content) => [...content].length === 1)
|
|
42
|
+
.map((content) => [content, idOf(content)]);
|
|
43
|
+
return {
|
|
44
|
+
added: json.added_tokens.map(({ content, id, lstrip, rstrip }) => ({
|
|
45
|
+
content,
|
|
46
|
+
id,
|
|
47
|
+
lstrip: Boolean(lstrip),
|
|
48
|
+
rstrip: Boolean(rstrip),
|
|
49
|
+
})),
|
|
50
|
+
characters,
|
|
51
|
+
byteIds: Array.from({ length: 256 }, (_, byte) =>
|
|
52
|
+
idOf(`<0x${byte.toString(16).toUpperCase().padStart(2, "0")}>`)
|
|
53
|
+
),
|
|
54
|
+
unknownId: idOf(model.unk_token),
|
|
55
|
+
template: templateOf(json.post_processor, idOf),
|
|
56
|
+
merges: mergeIds(model.merges, idOf),
|
|
57
|
+
};
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/** The tables as one file: the magic, the plain parts as JSON, then the merges' three columns of ids. */
|
|
61
|
+
function writeTables(file, { merges, ...plain }) {
|
|
62
|
+
const header = Buffer.from(
|
|
63
|
+
JSON.stringify({ ...plain, mergeCount: merges.results.length })
|
|
64
|
+
);
|
|
65
|
+
const padding = (4 - ((MAGIC.length + 4 + header.length) % 4)) % 4;
|
|
66
|
+
const length = Buffer.alloc(4);
|
|
67
|
+
length.writeUInt32LE(header.length);
|
|
68
|
+
const columns = [merges.lefts, merges.rights, merges.results].map((column) =>
|
|
69
|
+
Buffer.from(column.buffer, column.byteOffset, column.byteLength)
|
|
70
|
+
);
|
|
71
|
+
fs.mkdirSync(path.dirname(file), { recursive: true });
|
|
72
|
+
const partial = `${file}.partial`;
|
|
73
|
+
fs.writeFileSync(
|
|
74
|
+
partial,
|
|
75
|
+
Buffer.concat([
|
|
76
|
+
Buffer.from(MAGIC),
|
|
77
|
+
length,
|
|
78
|
+
header,
|
|
79
|
+
Buffer.alloc(padding),
|
|
80
|
+
...columns,
|
|
81
|
+
])
|
|
82
|
+
);
|
|
83
|
+
fs.renameSync(partial, file);
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
function readTables(file) {
|
|
87
|
+
let bytes;
|
|
88
|
+
try {
|
|
89
|
+
bytes = fs.readFileSync(file);
|
|
90
|
+
} catch {
|
|
91
|
+
return null;
|
|
92
|
+
}
|
|
93
|
+
try {
|
|
94
|
+
if (bytes.subarray(0, MAGIC.length).toString() !== MAGIC) return null;
|
|
95
|
+
const headerLength = bytes.readUInt32LE(MAGIC.length);
|
|
96
|
+
const headerStart = MAGIC.length + 4;
|
|
97
|
+
const { mergeCount, ...plain } = JSON.parse(
|
|
98
|
+
bytes.subarray(headerStart, headerStart + headerLength).toString()
|
|
99
|
+
);
|
|
100
|
+
const columnsStart =
|
|
101
|
+
headerStart +
|
|
102
|
+
headerLength +
|
|
103
|
+
((4 - ((headerStart + headerLength) % 4)) % 4);
|
|
104
|
+
const column = (index) => {
|
|
105
|
+
const start = columnsStart + index * mergeCount * 4;
|
|
106
|
+
return new Int32Array(
|
|
107
|
+
bytes.buffer.slice(
|
|
108
|
+
bytes.byteOffset + start,
|
|
109
|
+
bytes.byteOffset + start + mergeCount * 4
|
|
110
|
+
)
|
|
111
|
+
);
|
|
112
|
+
};
|
|
113
|
+
if (bytes.length !== columnsStart + 3 * mergeCount * 4) return null;
|
|
114
|
+
return {
|
|
115
|
+
...plain,
|
|
116
|
+
merges: { lefts: column(0), rights: column(1), results: column(2) },
|
|
117
|
+
};
|
|
118
|
+
} catch {
|
|
119
|
+
return null;
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
function encoderFrom({
|
|
124
|
+
added: addedTokens,
|
|
125
|
+
characters,
|
|
126
|
+
byteIds,
|
|
127
|
+
unknownId,
|
|
128
|
+
template,
|
|
129
|
+
merges: mergeColumns,
|
|
130
|
+
}) {
|
|
131
|
+
const addedIds = new Map(
|
|
132
|
+
addedTokens.map((token) => [token.content, token.id])
|
|
133
|
+
);
|
|
134
|
+
const added = addedTokenSplitter(addedTokens);
|
|
135
|
+
const characterIds = new Map(characters);
|
|
136
|
+
const merges = mergeTable(mergeColumns);
|
|
137
|
+
const encoder = new TextEncoder();
|
|
138
|
+
const fallbackIds = (character) => {
|
|
139
|
+
const ids = Array.from(encoder.encode(character), (byte) => byteIds[byte]);
|
|
140
|
+
return ids.every((id) => id !== undefined && id !== null)
|
|
141
|
+
? ids
|
|
142
|
+
: [unknownId];
|
|
143
|
+
};
|
|
144
|
+
const wordIds = (word) =>
|
|
145
|
+
bpe(word, characterIds, merges).flatMap((symbol) =>
|
|
146
|
+
typeof symbol === "number" ? [symbol] : fallbackIds(symbol)
|
|
147
|
+
);
|
|
148
|
+
|
|
149
|
+
const sectionIds = (section) => {
|
|
150
|
+
if (addedIds.has(section)) return [addedIds.get(section)];
|
|
151
|
+
const normalized = section.replaceAll(" ", REPLACEMENT);
|
|
152
|
+
if (addedIds.has(normalized)) return [addedIds.get(normalized)];
|
|
153
|
+
const word = normalized.startsWith(REPLACEMENT)
|
|
154
|
+
? normalized
|
|
155
|
+
: REPLACEMENT + normalized;
|
|
156
|
+
return wordIds(word);
|
|
157
|
+
};
|
|
158
|
+
const textIds = (text) =>
|
|
159
|
+
added
|
|
160
|
+
.split(text)
|
|
161
|
+
.filter((section) => section.length > 0)
|
|
162
|
+
.flatMap(sectionIds);
|
|
163
|
+
|
|
164
|
+
return {
|
|
165
|
+
/** { ids } as @huggingface/tokenizers' Tokenizer.encode gives them. */
|
|
166
|
+
encode(text, { text_pair = null, add_special_tokens = true } = {}) {
|
|
167
|
+
const first = textIds(text);
|
|
168
|
+
const second = text_pair === null ? null : textIds(text_pair);
|
|
169
|
+
const steps = second === null ? template.single : template.pair;
|
|
170
|
+
const ids = steps.flatMap((step) => {
|
|
171
|
+
if (step.special !== undefined)
|
|
172
|
+
return add_special_tokens ? [step.special] : [];
|
|
173
|
+
return step.sequence === "A" ? first : second;
|
|
174
|
+
});
|
|
175
|
+
return { ids };
|
|
176
|
+
},
|
|
177
|
+
};
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
/** The post-processor's templates with their special tokens as ids. */
|
|
181
|
+
function templateOf(postProcessor, idOf) {
|
|
182
|
+
const steps = (template) =>
|
|
183
|
+
template.map((step) =>
|
|
184
|
+
"SpecialToken" in step
|
|
185
|
+
? { special: idOf(step.SpecialToken.id) }
|
|
186
|
+
: { sequence: step.Sequence.id }
|
|
187
|
+
);
|
|
188
|
+
return {
|
|
189
|
+
single: steps(postProcessor.single),
|
|
190
|
+
pair: steps(postProcessor.pair),
|
|
191
|
+
};
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
function supported(json) {
|
|
195
|
+
const { model, normalizer, pre_tokenizer, post_processor } = json ?? {};
|
|
196
|
+
return (
|
|
197
|
+
model?.type === "BPE" &&
|
|
198
|
+
model.byte_fallback === true &&
|
|
199
|
+
!model.dropout &&
|
|
200
|
+
!model.end_of_word_suffix &&
|
|
201
|
+
!model.continuing_subword_suffix &&
|
|
202
|
+
!model.ignore_merges &&
|
|
203
|
+
Array.isArray(model.merges?.[0]) &&
|
|
204
|
+
normalizer?.type === "Replace" &&
|
|
205
|
+
normalizer.pattern?.String === " " &&
|
|
206
|
+
normalizer.content === REPLACEMENT &&
|
|
207
|
+
pre_tokenizer?.type === "Metaspace" &&
|
|
208
|
+
pre_tokenizer.replacement === REPLACEMENT &&
|
|
209
|
+
pre_tokenizer.prepend_scheme === "always" &&
|
|
210
|
+
!pre_tokenizer.str_rep &&
|
|
211
|
+
post_processor?.type === "TemplateProcessing" &&
|
|
212
|
+
json.added_tokens.every((token) => token.normalized === false)
|
|
213
|
+
);
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
/** Each merge's two parts and its result as ids, in rank order. */
|
|
217
|
+
function mergeIds(merges, idOf) {
|
|
218
|
+
const lefts = new Int32Array(merges.length);
|
|
219
|
+
const rights = new Int32Array(merges.length);
|
|
220
|
+
const results = new Int32Array(merges.length);
|
|
221
|
+
merges.forEach(([left, right], rank) => {
|
|
222
|
+
lefts[rank] = idOf(left);
|
|
223
|
+
rights[rank] = idOf(right);
|
|
224
|
+
results[rank] = idOf(left + right);
|
|
225
|
+
});
|
|
226
|
+
return { lefts, rights, results };
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
/**
|
|
230
|
+
* The merges as an open-addressing table keyed by the pair's two ids: rank(left, right) is the merge's priority (lower
|
|
231
|
+
* merges first) or -1, and result(rank) the merged token's id. Built from ids alone, so it holds nothing of the file.
|
|
232
|
+
*/
|
|
233
|
+
function mergeTable({ lefts, rights, results }) {
|
|
234
|
+
let capacity = 1;
|
|
235
|
+
while (capacity < results.length * 1.8) capacity *= 2;
|
|
236
|
+
const mask = capacity - 1;
|
|
237
|
+
const slotLefts = new Int32Array(capacity);
|
|
238
|
+
const slotRights = new Int32Array(capacity);
|
|
239
|
+
const slotRanks = new Int32Array(capacity).fill(-1);
|
|
240
|
+
const slotOf = (left, right) =>
|
|
241
|
+
(Math.imul(left, 0x9e3779b1) ^ Math.imul(right, 0x85ebca77)) & mask;
|
|
242
|
+
const find = (left, right) => {
|
|
243
|
+
let slot = slotOf(left, right);
|
|
244
|
+
while (
|
|
245
|
+
slotRanks[slot] !== -1 &&
|
|
246
|
+
!(slotLefts[slot] === left && slotRights[slot] === right)
|
|
247
|
+
)
|
|
248
|
+
slot = (slot + 1) & mask;
|
|
249
|
+
return slot;
|
|
250
|
+
};
|
|
251
|
+
|
|
252
|
+
// A pair listed twice keeps its later rank, as the library's Map built from the list does.
|
|
253
|
+
for (let rank = 0; rank < results.length; rank += 1) {
|
|
254
|
+
const slot = find(lefts[rank], rights[rank]);
|
|
255
|
+
slotLefts[slot] = lefts[rank];
|
|
256
|
+
slotRights[slot] = rights[rank];
|
|
257
|
+
slotRanks[slot] = rank;
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
return {
|
|
261
|
+
rank: (left, right) => slotRanks[find(left, right)],
|
|
262
|
+
result: (rank) => results[rank],
|
|
263
|
+
};
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
/**
|
|
267
|
+
* One word's symbols after every merge: ids, and the characters outside the vocabulary as strings (they never merge). The
|
|
268
|
+
* lowest-ranked pair merges first, the leftmost of equal pairs first, as the library's queue orders them.
|
|
269
|
+
*/
|
|
270
|
+
function bpe(word, characterIds, merges) {
|
|
271
|
+
const symbols = Array.from(word, (character) =>
|
|
272
|
+
characterIds.has(character) ? characterIds.get(character) : character
|
|
273
|
+
);
|
|
274
|
+
const next = symbols.map((_, i) => (i + 1 < symbols.length ? i + 1 : -1));
|
|
275
|
+
const previous = symbols.map((_, i) => i - 1);
|
|
276
|
+
const alive = symbols.map(() => true);
|
|
277
|
+
const queue = new PairQueue();
|
|
278
|
+
|
|
279
|
+
const consider = (left) => {
|
|
280
|
+
const right = next[left];
|
|
281
|
+
if (right === -1) return;
|
|
282
|
+
const [leftId, rightId] = [symbols[left], symbols[right]];
|
|
283
|
+
if (typeof leftId !== "number" || typeof rightId !== "number") return;
|
|
284
|
+
const rank = merges.rank(leftId, rightId);
|
|
285
|
+
if (rank !== -1) queue.push({ rank, left, right, leftId, rightId });
|
|
286
|
+
};
|
|
287
|
+
const stillThere = (pair) =>
|
|
288
|
+
alive[pair.left] &&
|
|
289
|
+
alive[pair.right] &&
|
|
290
|
+
next[pair.left] === pair.right &&
|
|
291
|
+
symbols[pair.left] === pair.leftId &&
|
|
292
|
+
symbols[pair.right] === pair.rightId;
|
|
293
|
+
|
|
294
|
+
for (let i = 0; i < symbols.length; i += 1) consider(i);
|
|
295
|
+
while (queue.size > 0) {
|
|
296
|
+
const pair = queue.pop();
|
|
297
|
+
if (!stillThere(pair)) continue;
|
|
298
|
+
symbols[pair.left] = merges.result(pair.rank);
|
|
299
|
+
alive[pair.right] = false;
|
|
300
|
+
next[pair.left] = next[pair.right];
|
|
301
|
+
if (next[pair.left] !== -1) previous[next[pair.left]] = pair.left;
|
|
302
|
+
if (previous[pair.left] !== -1) consider(previous[pair.left]);
|
|
303
|
+
consider(pair.left);
|
|
304
|
+
}
|
|
305
|
+
return symbols.filter((_, i) => alive[i]);
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
/** A binary min-heap of candidate merges, by rank and then by the left symbol's place in the word. */
|
|
309
|
+
class PairQueue {
|
|
310
|
+
#heap = [];
|
|
311
|
+
get size() {
|
|
312
|
+
return this.#heap.length;
|
|
313
|
+
}
|
|
314
|
+
#before(a, b) {
|
|
315
|
+
return a.rank < b.rank || (a.rank === b.rank && a.left < b.left);
|
|
316
|
+
}
|
|
317
|
+
push(pair) {
|
|
318
|
+
const heap = this.#heap;
|
|
319
|
+
heap.push(pair);
|
|
320
|
+
let i = heap.length - 1;
|
|
321
|
+
while (i > 0) {
|
|
322
|
+
const parent = (i - 1) >> 1;
|
|
323
|
+
if (!this.#before(heap[i], heap[parent])) break;
|
|
324
|
+
[heap[i], heap[parent]] = [heap[parent], heap[i]];
|
|
325
|
+
i = parent;
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
pop() {
|
|
329
|
+
const heap = this.#heap;
|
|
330
|
+
const top = heap[0];
|
|
331
|
+
const last = heap.pop();
|
|
332
|
+
if (heap.length > 0) {
|
|
333
|
+
heap[0] = last;
|
|
334
|
+
let i = 0;
|
|
335
|
+
for (;;) {
|
|
336
|
+
const left = 2 * i + 1;
|
|
337
|
+
const right = left + 1;
|
|
338
|
+
let smallest = i;
|
|
339
|
+
if (left < heap.length && this.#before(heap[left], heap[smallest]))
|
|
340
|
+
smallest = left;
|
|
341
|
+
if (right < heap.length && this.#before(heap[right], heap[smallest]))
|
|
342
|
+
smallest = right;
|
|
343
|
+
if (smallest === i) break;
|
|
344
|
+
[heap[i], heap[smallest]] = [heap[smallest], heap[i]];
|
|
345
|
+
i = smallest;
|
|
346
|
+
}
|
|
347
|
+
}
|
|
348
|
+
return top;
|
|
349
|
+
}
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
/**
|
|
353
|
+
* Splits text around added tokens the way the library's DictionarySplitter does (the longest token starting at each
|
|
354
|
+
* place, scanning left to right, by UTF-16 unit), and applies their lstrip and rstrip to the neighbouring text.
|
|
355
|
+
*/
|
|
356
|
+
function addedTokenSplitter(addedTokens) {
|
|
357
|
+
const byContent = new Map(addedTokens.map((token) => [token.content, token]));
|
|
358
|
+
const trie = new Map();
|
|
359
|
+
for (const token of addedTokens) {
|
|
360
|
+
let node = trie;
|
|
361
|
+
for (const unit of token.content) {
|
|
362
|
+
if (!node.has(unit)) node.set(unit, new Map());
|
|
363
|
+
node = node.get(unit);
|
|
364
|
+
}
|
|
365
|
+
node.token = token;
|
|
366
|
+
}
|
|
367
|
+
const longestAt = (text, start) => {
|
|
368
|
+
let node = trie;
|
|
369
|
+
let match = null;
|
|
370
|
+
for (let i = start; i < text.length; i += 1) {
|
|
371
|
+
node = node.get(text[i]);
|
|
372
|
+
if (!node) break;
|
|
373
|
+
if (node.token) match = node.token;
|
|
374
|
+
}
|
|
375
|
+
return match;
|
|
376
|
+
};
|
|
377
|
+
return {
|
|
378
|
+
split(text) {
|
|
379
|
+
const pieces = [];
|
|
380
|
+
let start = 0;
|
|
381
|
+
let i = 0;
|
|
382
|
+
while (i < text.length) {
|
|
383
|
+
const token = longestAt(text, i);
|
|
384
|
+
if (!token) {
|
|
385
|
+
i += 1;
|
|
386
|
+
continue;
|
|
387
|
+
}
|
|
388
|
+
if (i > start) pieces.push(text.slice(start, i));
|
|
389
|
+
pieces.push(token.content);
|
|
390
|
+
i += token.content.length;
|
|
391
|
+
start = i;
|
|
392
|
+
}
|
|
393
|
+
if (start < text.length) pieces.push(text.slice(start));
|
|
394
|
+
// As the library does, a piece is a token by its text, and strips its neighbours in order.
|
|
395
|
+
pieces.forEach((piece, index) => {
|
|
396
|
+
const token = byContent.get(piece);
|
|
397
|
+
if (token?.lstrip && index > 0)
|
|
398
|
+
pieces[index - 1] = pieces[index - 1].trimEnd();
|
|
399
|
+
if (token?.rstrip && index < pieces.length - 1)
|
|
400
|
+
pieces[index + 1] = pieces[index + 1].trimStart();
|
|
401
|
+
});
|
|
402
|
+
return pieces;
|
|
403
|
+
},
|
|
404
|
+
};
|
|
405
|
+
}
|