tryharness 0.7.44 → 0.7.53

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (101) hide show
  1. package/README.md +232 -12
  2. package/cli.mjs +406 -184
  3. package/lib/actorRoster.mjs +28 -5
  4. package/lib/adapters.mjs +9 -1
  5. package/lib/agentRuntimeConfig.mjs +36 -0
  6. package/lib/agentRuntimes.mjs +560 -0
  7. package/lib/atlas.mjs +7 -2
  8. package/lib/boardCommands.mjs +5 -10
  9. package/lib/boardFileCommands.mjs +2 -1
  10. package/lib/boardMirror.mjs +459 -0
  11. package/lib/boardProtocol.json +259 -6
  12. package/lib/boardTools.mjs +2 -6
  13. package/lib/bridge.mjs +27 -3
  14. package/lib/browserWake.mjs +60 -0
  15. package/lib/cards.mjs +35 -2
  16. package/lib/commands.mjs +20 -5
  17. package/lib/connectionTools.d.ts +13 -0
  18. package/lib/connectionTools.js +45 -0
  19. package/lib/connectorAuth.mjs +309 -0
  20. package/lib/connectorCatalog.json +41 -0
  21. package/lib/connectorCommands.mjs +121 -0
  22. package/lib/connectorGoogle.mjs +513 -0
  23. package/lib/connectors.mjs +805 -0
  24. package/lib/coordinator.mjs +16 -2
  25. package/lib/coreModels.mjs +78 -0
  26. package/lib/daemon.mjs +775 -46
  27. package/lib/desktop.mjs +7 -2
  28. package/lib/digest.mjs +32 -0
  29. package/lib/directLink.mjs +283 -0
  30. package/lib/environments.mjs +595 -0
  31. package/lib/folders.mjs +2 -2
  32. package/lib/goalCommands.mjs +61 -12
  33. package/lib/inference.mjs +263 -152
  34. package/lib/lessons.mjs +6 -0
  35. package/lib/lineup.mjs +324 -0
  36. package/lib/localmodels.mjs +10 -0
  37. package/lib/machineIdentity.mjs +3 -0
  38. package/lib/machineRoutes.mjs +40 -2
  39. package/lib/machineRuns.mjs +30 -3
  40. package/lib/machineSeal.mjs +69 -10
  41. package/lib/mcp-stdio.mjs +180 -98
  42. package/lib/mcpRelay.mjs +119 -0
  43. package/lib/members.json +12 -0
  44. package/lib/members.mjs +33 -0
  45. package/lib/modelMemory.mjs +100 -0
  46. package/lib/modelTiers.json +1183 -0
  47. package/lib/modelTiers.mjs +133 -0
  48. package/lib/models.mjs +41 -8
  49. package/lib/museSession.mjs +119 -0
  50. package/lib/nativeHost.mjs +203 -0
  51. package/lib/network.mjs +38 -1
  52. package/lib/nudge.mjs +10 -2
  53. package/lib/onboard.mjs +430 -1
  54. package/lib/oneclawLocal.mjs +150 -0
  55. package/lib/ownMachines.mjs +521 -36
  56. package/lib/pairing.mjs +165 -0
  57. package/lib/paths.mjs +4 -0
  58. package/lib/phones.mjs +61 -0
  59. package/lib/placeCommands.mjs +27 -0
  60. package/lib/places.mjs +10 -2
  61. package/lib/plans.mjs +151 -27
  62. package/lib/profile.mjs +152 -0
  63. package/lib/projectGit.mjs +328 -0
  64. package/lib/projectSettings.mjs +120 -0
  65. package/lib/projects.mjs +2539 -0
  66. package/lib/readiness.mjs +283 -0
  67. package/lib/repoLinks.mjs +354 -0
  68. package/lib/roomChannels.mjs +641 -0
  69. package/lib/runBrief.mjs +32 -2
  70. package/lib/runRequests.mjs +157 -0
  71. package/lib/runWorker.mjs +20 -7
  72. package/lib/runner.mjs +67 -8
  73. package/lib/secretPatterns.json +55 -0
  74. package/lib/secrets.mjs +10 -27
  75. package/lib/seen.mjs +102 -0
  76. package/lib/service.mjs +11 -6
  77. package/lib/sessionControl.mjs +10 -8
  78. package/lib/sessionIndex.mjs +5 -3
  79. package/lib/store.mjs +559 -30
  80. package/lib/subagents.mjs +8 -4
  81. package/lib/swarm.mjs +379 -38
  82. package/lib/swarmMemory.mjs +947 -0
  83. package/lib/t3.mjs +43 -13
  84. package/lib/taskCommands.mjs +58 -5
  85. package/lib/taskRuns.mjs +61 -11
  86. package/lib/team.json +17 -15
  87. package/lib/team.mjs +68 -18
  88. package/lib/threadRuns.mjs +50 -7
  89. package/lib/threadView.mjs +68 -15
  90. package/lib/tokenizer.mjs +405 -0
  91. package/lib/ui.mjs +16 -2
  92. package/lib/usageTotals.mjs +148 -0
  93. package/lib/vault.mjs +1144 -0
  94. package/lib/vaultCloud.mjs +117 -0
  95. package/lib/vaultCommands.mjs +201 -0
  96. package/lib/vaultTools.d.ts +12 -0
  97. package/lib/vaultTools.js +42 -0
  98. package/lib/windowAsks.mjs +42 -0
  99. package/lib/workFocus.mjs +59 -0
  100. package/lib/workLinks.mjs +90 -0
  101. package/package.json +9 -4
@@ -1,12 +1,17 @@
1
1
  import fs from "node:fs";
2
2
  import path from "node:path";
3
3
  import { randomUUID } from "node:crypto";
4
- import { nameOf } from "./members.mjs";
4
+ import { MEMBERS, nameOf } from "./members.mjs";
5
5
  import { oneLine } from "./oneLine.mjs";
6
6
  import { setRunPick } from "./models.mjs";
7
7
  import { t3Access, startT3SignIn, steerT3Run } from "./t3.mjs";
8
8
  import { t3Model, t3Open, t3Project, t3Provider } from "./places.mjs";
9
9
 
10
+ // The agents with a read-only switch (runner.mjs): only they talk out a clash in a shared project without waiting.
11
+ const REVIEWERS = new Set(
12
+ MEMBERS.filter((entry) => entry.review).map((entry) => entry.id)
13
+ );
14
+
10
15
  /** Thread execution owns queued calls, fresh briefs, worktree settlement and T3 continuity. */
11
16
  export function createThreadRuns({
12
17
  store,
@@ -24,6 +29,8 @@ export function createThreadRuns({
24
29
  // A coding agent not installed here but on another of the person's computers that has the thread's repository:
25
30
  // { machine, folder } there (ownMachines.mjs), or null.
26
31
  placeElsewhere = () => null,
32
+ // Projects shared with another person's swarm (projects.mjs): their check before work, and each agent's own branch.
33
+ projects = () => null,
27
34
  }) {
28
35
  const { RUNNERS } = authority;
29
36
  const inT3 = (thread, agent, review) => {
@@ -232,6 +239,9 @@ export function createThreadRuns({
232
239
  held = null,
233
240
  // The asker's turn on its own task's answer (daemon.mjs threadMayRun).
234
241
  answer = false,
242
+ // Past a shared project's check before work (projects.mjs gate); `wait` holds the run for the person's Start.
243
+ checked = false,
244
+ wait = false,
235
245
  } = {}
236
246
  ) => {
237
247
  const task = store.getTask(goalId);
@@ -289,10 +299,33 @@ export function createThreadRuns({
289
299
  pauseGoal(goalId, `${agent} was woken ${goal.max_wakes} times`);
290
300
  return;
291
301
  }
302
+ // In a shared project, new work is checked against both sides' work and decisions first; a clash holds it, and the
303
+ // agent only talks it out (read-only) until a person OKs the fix.
304
+ const shared = projects()?.isProjectRoom(goalId) ? projects() : null;
305
+ if (shared && !checked && !recovering)
306
+ return (
307
+ shared.gate(
308
+ { task, agent, reason, asker, go, mode: null, words },
309
+ (mode, override = null) =>
310
+ wakeAgent(goalId, agent, override ?? reason, {
311
+ review: review || (mode === "review" && REVIEWERS.has(agent)),
312
+ go: mode === "review" ? REVIEWERS.has(agent) : go,
313
+ wait:
314
+ mode === "wait" || (mode === "review" && !REVIEWERS.has(agent)),
315
+ asker,
316
+ pick,
317
+ words,
318
+ held,
319
+ answer,
320
+ checked: true,
321
+ })
322
+ ) ?? "checking"
323
+ );
292
324
  const brief = swarmBrief(task, agent, reason, { review });
293
325
  // A coding thread in a repository works in its own worktree, so threads never share a checkout; one whose agents the
294
- // person uses in T3 Code works in T3 (inT3), where T3 makes and keeps the checkout.
295
- const t3 = !elsewhere && inT3(task, agent, review);
326
+ // person uses in T3 Code works in T3 (inT3), where T3 makes and keeps the checkout. In a shared project each agent
327
+ // works on its own branch here, never in T3's checkout.
328
+ const t3 = !elsewhere && !shared && inT3(task, agent, review);
296
329
  // The person uses this agent in T3 Code but this turn runs here: the thread says why, once for each reason.
297
330
  const gap = t3 || review || elsewhere ? null : t3Gap(agent, goal.workspace);
298
331
  if (gap && !store.meta(`t3-gap-note:${goalId}:${agent}:${gap}`)) {
@@ -307,7 +340,9 @@ export function createThreadRuns({
307
340
  let folder = elsewhere?.folder ?? goal.workspace;
308
341
  if (!t3 && !elsewhere)
309
342
  try {
310
- folder = worktrees.folderFor(task);
343
+ folder =
344
+ (shared ? shared.folderFor(task, agent, { review }) : null) ??
345
+ worktrees.folderFor(task);
311
346
  } catch (error) {
312
347
  store.roomPost({
313
348
  goalId,
@@ -344,7 +379,9 @@ export function createThreadRuns({
344
379
  if (pick) setRunPick(store, wake.id, pick, asker);
345
380
  // A review runs with the agent's own read-only switch (runner.mjs); only Claude Code and Codex have one.
346
381
  if (review) store.meta(`review:${wake.id}`, "1");
347
- if (!recovering && threadMayRun(task, { answer })) {
382
+ if (wait) {
383
+ // Waits for the person's Start; the project already said why in the thread.
384
+ } else if (!recovering && threadMayRun(task, { answer })) {
348
385
  if (go) {
349
386
  store.meta(`person-wake:${wake.id}`, "1");
350
387
  store.authorizeGoalRun(wake.id);
@@ -417,12 +454,16 @@ export function createThreadRuns({
417
454
  }
418
455
  };
419
456
  // The turn's changes are committed in its thread's copy as soon as it ends, before another run can start there.
420
- const commitWake = (task) =>
421
- worktrees.commitTurn(
457
+ // A shared project's agent commits on its own branch (projects.mjs); it is undefined for any other thread.
458
+ const commitWake = (task) => {
459
+ const onBranch = projects()?.committed(task);
460
+ if (onBranch !== undefined) return onBranch;
461
+ return worktrees.commitTurn(
422
462
  task.parent_id,
423
463
  task.id,
424
464
  `${task.target_agent}: ${task.objective.replace(/^Room: /, "")}`
425
465
  );
466
+ };
426
467
  const settleWake = (task, kind, text, startedAt) => {
427
468
  const goalId = task.parent_id;
428
469
  try {
@@ -443,6 +484,8 @@ export function createThreadRuns({
443
484
  author: task.target_agent,
444
485
  kind: kind === "done" ? "post" : "failed",
445
486
  text: kind === "done" ? text : `run failed: ${text}`,
487
+ // The computer the turn ran on (runWorker.mjs claims it there).
488
+ machine: store.getTask(task.id)?.claimed_machine ?? null,
446
489
  });
447
490
  const key = `${goalId}:${task.target_agent}`;
448
491
  const pending = pendingWakes.get(key);
@@ -1,4 +1,5 @@
1
1
  import { nameOf } from "./members.mjs";
2
+ import { onMachine, withComputer } from "./machineRoutes.mjs";
2
3
 
3
4
  // A thread as the person reads it, worked out once here so the buddy, the phone, the PiP, the dashboard and the
4
5
  // notices all say the same thing: its title, its state in one word, one line on where it stands, what needs the
@@ -35,6 +36,13 @@ export function shorten(text, limit) {
35
36
 
36
37
  const capitalized = (text) => text.charAt(0).toUpperCase() + text.slice(1);
37
38
 
39
+ // Who said something, as the person reads it: another person's agent in a shared project is "igor's Codex".
40
+ const whoIs = (name) => {
41
+ const [handle, agent] = String(name).split("/");
42
+ if (!agent) return nameOf(name);
43
+ return agent === "person" ? handle : `${handle}'s ${nameOf(agent)}`;
44
+ };
45
+
38
46
  const withoutMentions = (text) =>
39
47
  text.replace(/(^|\s)@[a-z][a-z0-9:./-]*/gi, "$1").trim();
40
48
 
@@ -65,12 +73,29 @@ function headlineOf(text) {
65
73
  return shorten(sentence ?? line, 160);
66
74
  }
67
75
 
76
+ // An allowed task the browser has not taken by then is the person's again: Allow once more, or open Harness.
77
+ const BROWSER_START_MS = 2 * 60_000;
78
+
68
79
  export function createThreadView({
69
80
  store,
70
81
  goals = null,
71
82
  handoffs = null,
72
83
  now = Date.now,
84
+ // What a shared project's room waits on the person for (projects.mjs needsOf): an OK, a post to share.
85
+ projectNeeds = () => [],
86
+ nativeNeeds = () => [],
87
+ // A run's agent asking for one of the person's secrets (vault.mjs).
88
+ vaultNeeds = () => [],
89
+ // The name of the computer an agent wrote from, once the person has more than one (ownMachines.mjs shownLabel).
90
+ computerOf = () => null,
91
+ // Whether the person's browser is listening for this computer (browserWake.mjs), so Harness can use it now.
92
+ browserListening = () => false,
73
93
  }) {
94
+ // Words from an agent on the person's computers say which one: the buddy reads it "Codex (linux)".
95
+ const writtenOn = (post) => {
96
+ const { computer } = withComputer(post, computerOf);
97
+ return computer ? { computer } : {};
98
+ };
74
99
  const coordinated = (post) =>
75
100
  Boolean(store.meta(`coordinator-post:${post.id}`));
76
101
  const filing = (threadId) => {
@@ -93,6 +118,9 @@ export function createThreadView({
93
118
  };
94
119
 
95
120
  function titleOf(thread, posts) {
121
+ // A shared project's room keeps the project's name, the same on both people's computers.
122
+ if (store.meta(`project-room:${thread.id}`))
123
+ return shorten(thread.objective, TITLE_LIMIT);
96
124
  const written = goals?.record(thread.id)?.title;
97
125
  if (written) return written;
98
126
  // A task's own thread is named by its task, not by what the person writes in it later.
@@ -121,6 +149,8 @@ export function createThreadView({
121
149
  function needsYouOf(thread, posts, runs) {
122
150
  const items = [];
123
151
  for (const run of runs) {
152
+ items.push(...nativeNeeds(run.id));
153
+ items.push(...vaultNeeds(run.id));
124
154
  if (run.status === "needs_input" && run.holder === "user")
125
155
  items.push({
126
156
  id: `answer:${run.id}`,
@@ -189,21 +219,35 @@ export function createThreadView({
189
219
  at: thread.updated_at,
190
220
  });
191
221
  // What Harness's server left for the browser waits on the person opening their Harness window (October 18: one
192
- // such part waited 9 hours with nothing saying so).
193
- for (const run of runs)
222
+ // such part waited 9 hours with nothing saying so), or on their Allow when their browser listens for this computer
223
+ // (October 22: "right away, even if away"). Once allowed it waits on no one, unless the browser has not taken it.
224
+ for (const run of runs) {
194
225
  if (
195
- run.target_agent === "harness" &&
196
- run.status === "open" &&
197
- store.meta(`harness-window:${run.id}`)
226
+ run.target_agent !== "harness" ||
227
+ run.status !== "open" ||
228
+ !store.meta(`harness-window:${run.id}`)
198
229
  )
199
- items.push({
200
- id: `window:${run.id}`,
201
- kind: "window",
202
- agent: "harness",
203
- text: "Harness needs your browser to finish this: open Harness on your computer.",
204
- runId: run.id,
205
- at: run.updated_at,
206
- });
230
+ continue;
231
+ const allowedAt = Date.parse(
232
+ store.meta(`harness-browser-allowed:${run.id}`) || ""
233
+ );
234
+ if (now() - allowedAt < BROWSER_START_MS) continue;
235
+ const stopped = store.meta(`harness-browser-stopped:${run.id}`);
236
+ const canAllow = !stopped && browserListening();
237
+ items.push({
238
+ id: `window:${run.id}`,
239
+ kind: "window",
240
+ agent: "harness",
241
+ text: stopped
242
+ ? `Harness stopped in your browser to ask you: ${stopped}. Open Harness on your computer to answer.`
243
+ : canAllow
244
+ ? "Harness needs your browser to finish this. Allow it, and Harness carries on in your browser now."
245
+ : "Harness needs your browser to finish this: open Harness on your computer.",
246
+ runId: run.id,
247
+ ...(canAllow ? { browser: "can-allow" } : {}),
248
+ at: run.updated_at,
249
+ });
250
+ }
207
251
  for (const held of store.heldPosts(thread.id))
208
252
  if (!store.meta(`held-replaced:${held.id}`))
209
253
  items.push({
@@ -214,6 +258,7 @@ export function createThreadView({
214
258
  seq: held.seq,
215
259
  at: held.created_at,
216
260
  });
261
+ items.push(...projectNeeds(thread.id));
217
262
  return items.sort((a, b) => String(a.at).localeCompare(String(b.at)));
218
263
  }
219
264
 
@@ -263,12 +308,17 @@ export function createThreadView({
263
308
  : thread.goal.paused_reason || "Stopped";
264
309
  const last = lastWords(posts);
265
310
  if (!last) return "Nothing yet";
266
- const who = last.author === "user" ? "You" : nameOf(last.author);
311
+ const { computer } = writtenOn(last);
312
+ const who =
313
+ last.author === "user" ? "You" : onMachine(whoIs(last.author), computer);
267
314
  return `${who}: ${shorten(withoutMentions(firstLine(last.text)), 140)}`;
268
315
  }
269
316
 
270
317
  function itemLine(item) {
271
- const who = nameOf(item.agent);
318
+ const who = whoIs(item.agent);
319
+ if (item.kind === "agree")
320
+ return `${item.who ?? who} asks for your OK: ${shorten(firstLine(item.text), 140)}`;
321
+ if (item.kind === "share") return item.text;
272
322
  if (item.kind === "answer")
273
323
  return `${who} asks you: ${shorten(firstLine(item.text), 140)}`;
274
324
  if (item.kind === "start") return `${who} waits for your Start`;
@@ -344,6 +394,7 @@ export function createThreadView({
344
394
  type: "file",
345
395
  seq: post.seq,
346
396
  author: post.author,
397
+ ...writtenOn(post),
347
398
  file: post.file,
348
399
  at: post.created_at,
349
400
  };
@@ -361,6 +412,7 @@ export function createThreadView({
361
412
  type: "message",
362
413
  seq: post.seq,
363
414
  author: post.author,
415
+ ...writtenOn(post),
364
416
  text: post.text,
365
417
  result: false,
366
418
  asks: false,
@@ -381,6 +433,7 @@ export function createThreadView({
381
433
  type: "message",
382
434
  seq: post.seq,
383
435
  author: post.author,
436
+ ...writtenOn(post),
384
437
  text: post.text,
385
438
  result: post.kind === "result" || post.kind === "answer",
386
439
  asks: post.kind === "question",
@@ -0,0 +1,405 @@
1
+ import fs from "node:fs";
2
+ import path from "node:path";
3
+
4
+ // A compact BPE tokenizer for the guards' vocabulary (Gemma's: 256,000 tokens, 580,604 merges). @huggingface/tokenizers
5
+ // keeps the parsed tokenizer.json and a string-keyed map of every merge, about 300 MB for this vocabulary; here the merges
6
+ // are a table of integer pairs, about 21 MB in all. It reproduces that library's ids exactly, including where it departs
7
+ // from the Rust tokenizer (its Metaspace step does not split a section into words, so a whole section is one BPE word),
8
+ // because the guards' thresholds were calibrated on its ids. Any tokenizer.json of another shape gets null, and the
9
+ // caller uses the library instead. Parsing the 34 MB file leaves about 200 MB for the garbage collector, so the first
10
+ // build saves the tables (about 7 MB) and later builds read those.
11
+
12
+ const REPLACEMENT = "\u2581";
13
+ // The saved tables' format; a file of another format is rebuilt.
14
+ const MAGIC = "HTK1";
15
+
16
+ /** The tokenizer for a tokenizer.json file, or null when it is not the shape this reproduces. */
17
+ export function loadCompactTokenizer(file, tablesFile) {
18
+ const saved = readTables(tablesFile);
19
+ if (saved) return encoderFrom(saved);
20
+ const json = JSON.parse(fs.readFileSync(file, "utf8"));
21
+ if (!supported(json)) return null;
22
+ const tables = tablesOf(json);
23
+ writeTables(tablesFile, tables);
24
+ return encoderFrom(tables);
25
+ }
26
+
27
+ /** The tokenizer for a parsed tokenizer.json, or null when it is not the shape this reproduces. */
28
+ export function compactTokenizer(json) {
29
+ if (!supported(json)) return null;
30
+ return encoderFrom(tablesOf(json));
31
+ }
32
+
33
+ /** Everything encoding needs, as plain data that holds nothing of the parsed file. */
34
+ function tablesOf(json) {
35
+ const { model } = json;
36
+ const addedIds = new Map(
37
+ json.added_tokens.map((token) => [token.content, token.id])
38
+ );
39
+ const idOf = (content) => addedIds.get(content) ?? model.vocab[content];
40
+ const characters = Object.keys(model.vocab)
41
+ .filter((content) => [...content].length === 1)
42
+ .map((content) => [content, idOf(content)]);
43
+ return {
44
+ added: json.added_tokens.map(({ content, id, lstrip, rstrip }) => ({
45
+ content,
46
+ id,
47
+ lstrip: Boolean(lstrip),
48
+ rstrip: Boolean(rstrip),
49
+ })),
50
+ characters,
51
+ byteIds: Array.from({ length: 256 }, (_, byte) =>
52
+ idOf(`<0x${byte.toString(16).toUpperCase().padStart(2, "0")}>`)
53
+ ),
54
+ unknownId: idOf(model.unk_token),
55
+ template: templateOf(json.post_processor, idOf),
56
+ merges: mergeIds(model.merges, idOf),
57
+ };
58
+ }
59
+
60
+ /** The tables as one file: the magic, the plain parts as JSON, then the merges' three columns of ids. */
61
+ function writeTables(file, { merges, ...plain }) {
62
+ const header = Buffer.from(
63
+ JSON.stringify({ ...plain, mergeCount: merges.results.length })
64
+ );
65
+ const padding = (4 - ((MAGIC.length + 4 + header.length) % 4)) % 4;
66
+ const length = Buffer.alloc(4);
67
+ length.writeUInt32LE(header.length);
68
+ const columns = [merges.lefts, merges.rights, merges.results].map((column) =>
69
+ Buffer.from(column.buffer, column.byteOffset, column.byteLength)
70
+ );
71
+ fs.mkdirSync(path.dirname(file), { recursive: true });
72
+ const partial = `${file}.partial`;
73
+ fs.writeFileSync(
74
+ partial,
75
+ Buffer.concat([
76
+ Buffer.from(MAGIC),
77
+ length,
78
+ header,
79
+ Buffer.alloc(padding),
80
+ ...columns,
81
+ ])
82
+ );
83
+ fs.renameSync(partial, file);
84
+ }
85
+
86
+ function readTables(file) {
87
+ let bytes;
88
+ try {
89
+ bytes = fs.readFileSync(file);
90
+ } catch {
91
+ return null;
92
+ }
93
+ try {
94
+ if (bytes.subarray(0, MAGIC.length).toString() !== MAGIC) return null;
95
+ const headerLength = bytes.readUInt32LE(MAGIC.length);
96
+ const headerStart = MAGIC.length + 4;
97
+ const { mergeCount, ...plain } = JSON.parse(
98
+ bytes.subarray(headerStart, headerStart + headerLength).toString()
99
+ );
100
+ const columnsStart =
101
+ headerStart +
102
+ headerLength +
103
+ ((4 - ((headerStart + headerLength) % 4)) % 4);
104
+ const column = (index) => {
105
+ const start = columnsStart + index * mergeCount * 4;
106
+ return new Int32Array(
107
+ bytes.buffer.slice(
108
+ bytes.byteOffset + start,
109
+ bytes.byteOffset + start + mergeCount * 4
110
+ )
111
+ );
112
+ };
113
+ if (bytes.length !== columnsStart + 3 * mergeCount * 4) return null;
114
+ return {
115
+ ...plain,
116
+ merges: { lefts: column(0), rights: column(1), results: column(2) },
117
+ };
118
+ } catch {
119
+ return null;
120
+ }
121
+ }
122
+
123
+ function encoderFrom({
124
+ added: addedTokens,
125
+ characters,
126
+ byteIds,
127
+ unknownId,
128
+ template,
129
+ merges: mergeColumns,
130
+ }) {
131
+ const addedIds = new Map(
132
+ addedTokens.map((token) => [token.content, token.id])
133
+ );
134
+ const added = addedTokenSplitter(addedTokens);
135
+ const characterIds = new Map(characters);
136
+ const merges = mergeTable(mergeColumns);
137
+ const encoder = new TextEncoder();
138
+ const fallbackIds = (character) => {
139
+ const ids = Array.from(encoder.encode(character), (byte) => byteIds[byte]);
140
+ return ids.every((id) => id !== undefined && id !== null)
141
+ ? ids
142
+ : [unknownId];
143
+ };
144
+ const wordIds = (word) =>
145
+ bpe(word, characterIds, merges).flatMap((symbol) =>
146
+ typeof symbol === "number" ? [symbol] : fallbackIds(symbol)
147
+ );
148
+
149
+ const sectionIds = (section) => {
150
+ if (addedIds.has(section)) return [addedIds.get(section)];
151
+ const normalized = section.replaceAll(" ", REPLACEMENT);
152
+ if (addedIds.has(normalized)) return [addedIds.get(normalized)];
153
+ const word = normalized.startsWith(REPLACEMENT)
154
+ ? normalized
155
+ : REPLACEMENT + normalized;
156
+ return wordIds(word);
157
+ };
158
+ const textIds = (text) =>
159
+ added
160
+ .split(text)
161
+ .filter((section) => section.length > 0)
162
+ .flatMap(sectionIds);
163
+
164
+ return {
165
+ /** { ids } as @huggingface/tokenizers' Tokenizer.encode gives them. */
166
+ encode(text, { text_pair = null, add_special_tokens = true } = {}) {
167
+ const first = textIds(text);
168
+ const second = text_pair === null ? null : textIds(text_pair);
169
+ const steps = second === null ? template.single : template.pair;
170
+ const ids = steps.flatMap((step) => {
171
+ if (step.special !== undefined)
172
+ return add_special_tokens ? [step.special] : [];
173
+ return step.sequence === "A" ? first : second;
174
+ });
175
+ return { ids };
176
+ },
177
+ };
178
+ }
179
+
180
+ /** The post-processor's templates with their special tokens as ids. */
181
+ function templateOf(postProcessor, idOf) {
182
+ const steps = (template) =>
183
+ template.map((step) =>
184
+ "SpecialToken" in step
185
+ ? { special: idOf(step.SpecialToken.id) }
186
+ : { sequence: step.Sequence.id }
187
+ );
188
+ return {
189
+ single: steps(postProcessor.single),
190
+ pair: steps(postProcessor.pair),
191
+ };
192
+ }
193
+
194
+ function supported(json) {
195
+ const { model, normalizer, pre_tokenizer, post_processor } = json ?? {};
196
+ return (
197
+ model?.type === "BPE" &&
198
+ model.byte_fallback === true &&
199
+ !model.dropout &&
200
+ !model.end_of_word_suffix &&
201
+ !model.continuing_subword_suffix &&
202
+ !model.ignore_merges &&
203
+ Array.isArray(model.merges?.[0]) &&
204
+ normalizer?.type === "Replace" &&
205
+ normalizer.pattern?.String === " " &&
206
+ normalizer.content === REPLACEMENT &&
207
+ pre_tokenizer?.type === "Metaspace" &&
208
+ pre_tokenizer.replacement === REPLACEMENT &&
209
+ pre_tokenizer.prepend_scheme === "always" &&
210
+ !pre_tokenizer.str_rep &&
211
+ post_processor?.type === "TemplateProcessing" &&
212
+ json.added_tokens.every((token) => token.normalized === false)
213
+ );
214
+ }
215
+
216
+ /** Each merge's two parts and its result as ids, in rank order. */
217
+ function mergeIds(merges, idOf) {
218
+ const lefts = new Int32Array(merges.length);
219
+ const rights = new Int32Array(merges.length);
220
+ const results = new Int32Array(merges.length);
221
+ merges.forEach(([left, right], rank) => {
222
+ lefts[rank] = idOf(left);
223
+ rights[rank] = idOf(right);
224
+ results[rank] = idOf(left + right);
225
+ });
226
+ return { lefts, rights, results };
227
+ }
228
+
229
+ /**
230
+ * The merges as an open-addressing table keyed by the pair's two ids: rank(left, right) is the merge's priority (lower
231
+ * merges first) or -1, and result(rank) the merged token's id. Built from ids alone, so it holds nothing of the file.
232
+ */
233
+ function mergeTable({ lefts, rights, results }) {
234
+ let capacity = 1;
235
+ while (capacity < results.length * 1.8) capacity *= 2;
236
+ const mask = capacity - 1;
237
+ const slotLefts = new Int32Array(capacity);
238
+ const slotRights = new Int32Array(capacity);
239
+ const slotRanks = new Int32Array(capacity).fill(-1);
240
+ const slotOf = (left, right) =>
241
+ (Math.imul(left, 0x9e3779b1) ^ Math.imul(right, 0x85ebca77)) & mask;
242
+ const find = (left, right) => {
243
+ let slot = slotOf(left, right);
244
+ while (
245
+ slotRanks[slot] !== -1 &&
246
+ !(slotLefts[slot] === left && slotRights[slot] === right)
247
+ )
248
+ slot = (slot + 1) & mask;
249
+ return slot;
250
+ };
251
+
252
+ // A pair listed twice keeps its later rank, as the library's Map built from the list does.
253
+ for (let rank = 0; rank < results.length; rank += 1) {
254
+ const slot = find(lefts[rank], rights[rank]);
255
+ slotLefts[slot] = lefts[rank];
256
+ slotRights[slot] = rights[rank];
257
+ slotRanks[slot] = rank;
258
+ }
259
+
260
+ return {
261
+ rank: (left, right) => slotRanks[find(left, right)],
262
+ result: (rank) => results[rank],
263
+ };
264
+ }
265
+
266
+ /**
267
+ * One word's symbols after every merge: ids, and the characters outside the vocabulary as strings (they never merge). The
268
+ * lowest-ranked pair merges first, the leftmost of equal pairs first, as the library's queue orders them.
269
+ */
270
+ function bpe(word, characterIds, merges) {
271
+ const symbols = Array.from(word, (character) =>
272
+ characterIds.has(character) ? characterIds.get(character) : character
273
+ );
274
+ const next = symbols.map((_, i) => (i + 1 < symbols.length ? i + 1 : -1));
275
+ const previous = symbols.map((_, i) => i - 1);
276
+ const alive = symbols.map(() => true);
277
+ const queue = new PairQueue();
278
+
279
+ const consider = (left) => {
280
+ const right = next[left];
281
+ if (right === -1) return;
282
+ const [leftId, rightId] = [symbols[left], symbols[right]];
283
+ if (typeof leftId !== "number" || typeof rightId !== "number") return;
284
+ const rank = merges.rank(leftId, rightId);
285
+ if (rank !== -1) queue.push({ rank, left, right, leftId, rightId });
286
+ };
287
+ const stillThere = (pair) =>
288
+ alive[pair.left] &&
289
+ alive[pair.right] &&
290
+ next[pair.left] === pair.right &&
291
+ symbols[pair.left] === pair.leftId &&
292
+ symbols[pair.right] === pair.rightId;
293
+
294
+ for (let i = 0; i < symbols.length; i += 1) consider(i);
295
+ while (queue.size > 0) {
296
+ const pair = queue.pop();
297
+ if (!stillThere(pair)) continue;
298
+ symbols[pair.left] = merges.result(pair.rank);
299
+ alive[pair.right] = false;
300
+ next[pair.left] = next[pair.right];
301
+ if (next[pair.left] !== -1) previous[next[pair.left]] = pair.left;
302
+ if (previous[pair.left] !== -1) consider(previous[pair.left]);
303
+ consider(pair.left);
304
+ }
305
+ return symbols.filter((_, i) => alive[i]);
306
+ }
307
+
308
+ /** A binary min-heap of candidate merges, by rank and then by the left symbol's place in the word. */
309
+ class PairQueue {
310
+ #heap = [];
311
+ get size() {
312
+ return this.#heap.length;
313
+ }
314
+ #before(a, b) {
315
+ return a.rank < b.rank || (a.rank === b.rank && a.left < b.left);
316
+ }
317
+ push(pair) {
318
+ const heap = this.#heap;
319
+ heap.push(pair);
320
+ let i = heap.length - 1;
321
+ while (i > 0) {
322
+ const parent = (i - 1) >> 1;
323
+ if (!this.#before(heap[i], heap[parent])) break;
324
+ [heap[i], heap[parent]] = [heap[parent], heap[i]];
325
+ i = parent;
326
+ }
327
+ }
328
+ pop() {
329
+ const heap = this.#heap;
330
+ const top = heap[0];
331
+ const last = heap.pop();
332
+ if (heap.length > 0) {
333
+ heap[0] = last;
334
+ let i = 0;
335
+ for (;;) {
336
+ const left = 2 * i + 1;
337
+ const right = left + 1;
338
+ let smallest = i;
339
+ if (left < heap.length && this.#before(heap[left], heap[smallest]))
340
+ smallest = left;
341
+ if (right < heap.length && this.#before(heap[right], heap[smallest]))
342
+ smallest = right;
343
+ if (smallest === i) break;
344
+ [heap[i], heap[smallest]] = [heap[smallest], heap[i]];
345
+ i = smallest;
346
+ }
347
+ }
348
+ return top;
349
+ }
350
+ }
351
+
352
+ /**
353
+ * Splits text around added tokens the way the library's DictionarySplitter does (the longest token starting at each
354
+ * place, scanning left to right, by UTF-16 unit), and applies their lstrip and rstrip to the neighbouring text.
355
+ */
356
+ function addedTokenSplitter(addedTokens) {
357
+ const byContent = new Map(addedTokens.map((token) => [token.content, token]));
358
+ const trie = new Map();
359
+ for (const token of addedTokens) {
360
+ let node = trie;
361
+ for (const unit of token.content) {
362
+ if (!node.has(unit)) node.set(unit, new Map());
363
+ node = node.get(unit);
364
+ }
365
+ node.token = token;
366
+ }
367
+ const longestAt = (text, start) => {
368
+ let node = trie;
369
+ let match = null;
370
+ for (let i = start; i < text.length; i += 1) {
371
+ node = node.get(text[i]);
372
+ if (!node) break;
373
+ if (node.token) match = node.token;
374
+ }
375
+ return match;
376
+ };
377
+ return {
378
+ split(text) {
379
+ const pieces = [];
380
+ let start = 0;
381
+ let i = 0;
382
+ while (i < text.length) {
383
+ const token = longestAt(text, i);
384
+ if (!token) {
385
+ i += 1;
386
+ continue;
387
+ }
388
+ if (i > start) pieces.push(text.slice(start, i));
389
+ pieces.push(token.content);
390
+ i += token.content.length;
391
+ start = i;
392
+ }
393
+ if (start < text.length) pieces.push(text.slice(start));
394
+ // As the library does, a piece is a token by its text, and strips its neighbours in order.
395
+ pieces.forEach((piece, index) => {
396
+ const token = byContent.get(piece);
397
+ if (token?.lstrip && index > 0)
398
+ pieces[index - 1] = pieces[index - 1].trimEnd();
399
+ if (token?.rstrip && index < pieces.length - 1)
400
+ pieces[index + 1] = pieces[index + 1].trimStart();
401
+ });
402
+ return pieces;
403
+ },
404
+ };
405
+ }