tryharness 0.7.45 → 0.7.54

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (125) hide show
  1. package/README.md +239 -14
  2. package/assets/dsh-plugin.mjs +72 -0
  3. package/assets/opencode-plugin.js +4 -3
  4. package/cli.mjs +497 -196
  5. package/lib/actorRoster.mjs +28 -5
  6. package/lib/adapters.mjs +165 -222
  7. package/lib/agentRuntimeConfig.mjs +36 -0
  8. package/lib/agentRuntimes.mjs +570 -0
  9. package/lib/agentSignIn.mjs +226 -0
  10. package/lib/atlas.mjs +7 -2
  11. package/lib/audit.mjs +643 -0
  12. package/lib/backupSetup.mjs +141 -0
  13. package/lib/boardAccess.mjs +8 -1
  14. package/lib/boardBackup.mjs +283 -0
  15. package/lib/boardCommands.mjs +5 -10
  16. package/lib/boardFileCommands.mjs +2 -1
  17. package/lib/boardMirror.mjs +523 -0
  18. package/lib/boardProtocol.json +263 -6
  19. package/lib/boardTools.mjs +2 -6
  20. package/lib/boardWaits.mjs +102 -0
  21. package/lib/book.mjs +659 -0
  22. package/lib/bridge.mjs +26 -4
  23. package/lib/browserWake.mjs +60 -0
  24. package/lib/cards.mjs +159 -16
  25. package/lib/cloudBoot.mjs +124 -0
  26. package/lib/commands.mjs +25 -5
  27. package/lib/connectionTools.d.ts +13 -0
  28. package/lib/connectionTools.js +45 -0
  29. package/lib/connectorAuth.mjs +309 -0
  30. package/lib/connectorCatalog.json +41 -0
  31. package/lib/connectorCommands.mjs +121 -0
  32. package/lib/connectorGoogle.mjs +513 -0
  33. package/lib/connectors.mjs +805 -0
  34. package/lib/context.mjs +18 -4
  35. package/lib/contractGate.mjs +377 -0
  36. package/lib/coordinator.mjs +152 -39
  37. package/lib/coreModels.mjs +78 -0
  38. package/lib/daemon.mjs +973 -54
  39. package/lib/desk.mjs +239 -0
  40. package/lib/desktop.mjs +113 -8
  41. package/lib/digest.mjs +32 -0
  42. package/lib/directLink.mjs +283 -0
  43. package/lib/environments.mjs +825 -0
  44. package/lib/folders.mjs +4 -4
  45. package/lib/gitHost.mjs +174 -0
  46. package/lib/goalCommands.mjs +58 -14
  47. package/lib/hookDialects.mjs +298 -0
  48. package/lib/inference.mjs +263 -152
  49. package/lib/landed.mjs +887 -0
  50. package/lib/lessons.mjs +6 -0
  51. package/lib/lineup.mjs +324 -0
  52. package/lib/live.mjs +284 -13
  53. package/lib/localmodels.mjs +10 -0
  54. package/lib/machineIdentity.mjs +3 -0
  55. package/lib/machineRoutes.mjs +40 -2
  56. package/lib/machineRuns.mjs +30 -3
  57. package/lib/machineSeal.mjs +69 -10
  58. package/lib/mcp-stdio.mjs +189 -120
  59. package/lib/mcpRelay.mjs +119 -0
  60. package/lib/members.json +26 -0
  61. package/lib/members.mjs +33 -0
  62. package/lib/modelMemory.mjs +100 -0
  63. package/lib/models.mjs +48 -15
  64. package/lib/museSession.mjs +122 -0
  65. package/lib/nativeHost.mjs +203 -0
  66. package/lib/neovim.mjs +126 -0
  67. package/lib/network.mjs +7 -2
  68. package/lib/nudge.mjs +57 -18
  69. package/lib/onboard.mjs +430 -1
  70. package/lib/oneclawLocal.mjs +150 -0
  71. package/lib/ownMachines.mjs +828 -39
  72. package/lib/pairing.mjs +182 -0
  73. package/lib/paths.mjs +8 -0
  74. package/lib/personModel.mjs +89 -0
  75. package/lib/phones.mjs +119 -3
  76. package/lib/placeCommands.mjs +27 -0
  77. package/lib/places.mjs +337 -9
  78. package/lib/plans.mjs +151 -27
  79. package/lib/profile.mjs +152 -0
  80. package/lib/projects.mjs +624 -137
  81. package/lib/readiness.mjs +400 -0
  82. package/lib/record.mjs +115 -0
  83. package/lib/repoLinks.mjs +354 -0
  84. package/lib/roomChannels.mjs +641 -0
  85. package/lib/runBrief.mjs +32 -2
  86. package/lib/runRequests.mjs +157 -0
  87. package/lib/runWorker.mjs +37 -9
  88. package/lib/runner.mjs +74 -11
  89. package/lib/secretPatterns.json +55 -0
  90. package/lib/secrets.mjs +10 -27
  91. package/lib/seen.mjs +102 -0
  92. package/lib/service.mjs +11 -6
  93. package/lib/sessionControl.mjs +386 -96
  94. package/lib/sessionIndex.mjs +5 -3
  95. package/lib/specialties.json +74 -0
  96. package/lib/specialties.mjs +477 -0
  97. package/lib/specialtyKeys.d.ts +3 -0
  98. package/lib/specialtyKeys.js +58 -0
  99. package/lib/specialtyNotice.mjs +147 -0
  100. package/lib/specialtySnapshot.mjs +138 -0
  101. package/lib/store.mjs +829 -29
  102. package/lib/subagents.mjs +8 -4
  103. package/lib/swarm.mjs +483 -37
  104. package/lib/swarmMemory.mjs +947 -0
  105. package/lib/t3.mjs +56 -17
  106. package/lib/taskCommands.mjs +91 -13
  107. package/lib/taskRuns.mjs +68 -11
  108. package/lib/team.json +253 -144
  109. package/lib/team.mjs +538 -147
  110. package/lib/threadLine.mjs +116 -0
  111. package/lib/threadRuns.mjs +78 -12
  112. package/lib/threadView.mjs +66 -16
  113. package/lib/tokenizer.mjs +405 -0
  114. package/lib/ui.mjs +16 -2
  115. package/lib/usageTotals.mjs +148 -0
  116. package/lib/vault.mjs +1144 -0
  117. package/lib/vaultCloud.mjs +117 -0
  118. package/lib/vaultCommands.mjs +201 -0
  119. package/lib/vaultTools.d.ts +12 -0
  120. package/lib/vaultTools.js +42 -0
  121. package/lib/windowAsks.mjs +42 -0
  122. package/lib/workFocus.mjs +59 -0
  123. package/lib/workLinks.mjs +90 -0
  124. package/lib/worktrees.mjs +51 -13
  125. package/package.json +10 -5
package/lib/inference.mjs CHANGED
@@ -13,6 +13,12 @@ import {
13
13
  llamaServer,
14
14
  } from "./localmodels.mjs";
15
15
  import { ensureDir } from "./files.mjs";
16
+ import { loadCompactTokenizer } from "./tokenizer.mjs";
17
+ import {
18
+ createModelMemory,
19
+ memoryCeilingMb,
20
+ memoryCostMb,
21
+ } from "./modelMemory.mjs";
16
22
 
17
23
  // The relay's local inference host. GGUF models (the search model) run in one llama.cpp server in
18
24
  // router mode, on the GPU when the machine has one that is faster, the CPU otherwise; the small ONNX classifiers run in
@@ -52,6 +58,7 @@ export function createPower({
52
58
  onBattery: false,
53
59
  percent: null,
54
60
  lowPower: false,
61
+ heldAwake: false,
55
62
  hot: false,
56
63
  at: 0,
57
64
  };
@@ -60,6 +67,7 @@ export function createPower({
60
67
  onBattery: false,
61
68
  percent: null,
62
69
  lowPower: false,
70
+ heldAwake: false,
63
71
  hot: false,
64
72
  at: Date.now(),
65
73
  };
@@ -69,9 +77,12 @@ export function createPower({
69
77
  .stdout || "";
70
78
  next.onBattery = /Battery Power/.test(batt);
71
79
  next.percent = Number(batt.match(/(\d+)%/)?.[1] ?? NaN);
72
- next.lowPower = /lowpowermode\s+1/.test(
73
- run("pmset", ["-g"], { encoding: "utf8", timeout: 3_000 }).stdout || ""
74
- );
80
+ const settings =
81
+ run("pmset", ["-g"], { encoding: "utf8", timeout: 3_000 }).stdout || "";
82
+ // macOS 26 prints `powermode 1` (0 automatic, 1 low, 2 high); older macOS `lowpowermode 1`.
83
+ next.lowPower = /^\s*(?:low)?powermode\s+1\b/m.test(settings);
84
+ // Lid-Closed Mode's helper keeps the Mac awake for the person's agents (packages/desktop lid.rs reads the same).
85
+ next.heldAwake = /^\s*SleepDisabled\s+1\b/m.test(settings);
75
86
  const therm =
76
87
  run("pmset", ["-g", "therm"], { encoding: "utf8", timeout: 3_000 })
77
88
  .stdout || "";
@@ -111,6 +122,20 @@ export function createPower({
111
122
  (now.onBattery && now.percent !== null && now.percent < 20)
112
123
  );
113
124
  },
125
+ /**
126
+ * Whether the dream waits. It runs whenever the computer is on (owner, October 22: "if the computer is just on
127
+ * obviously"), but not hot, below 20% on battery, or in Low Power Mode, unless Lid-Closed Mode holds the Mac awake:
128
+ * it turns Low Power Mode on with the lid closed, and the dream should run then ("if insomnia is on dream mode
129
+ * should run on low power").
130
+ */
131
+ dreamWaits() {
132
+ const now = this.state();
133
+ return (
134
+ now.hot ||
135
+ (now.onBattery && now.percent !== null && now.percent < 20) ||
136
+ (now.lowPower && !now.heldAwake)
137
+ );
138
+ },
114
139
  /** First index passes wait for the charger as well. */
115
140
  onCharger() {
116
141
  return !this.state().onBattery;
@@ -123,6 +148,7 @@ export function createInference({
123
148
  log = () => {},
124
149
  power = createPower(),
125
150
  list = MODELS,
151
+ ceilingMb = memoryCeilingMb(),
126
152
  }) {
127
153
  ensureDir(root);
128
154
  const deviceFile = path.join(root, "device.json");
@@ -131,6 +157,7 @@ export function createInference({
131
157
  const queues = { llama: [], onnx: [] };
132
158
  const busy = { llama: false, onnx: false };
133
159
  const sessions = new Map();
160
+ const memory = createModelMemory({ ceilingMb, log });
134
161
 
135
162
  const device = () => {
136
163
  try {
@@ -277,10 +304,15 @@ export function createInference({
277
304
  starting ||= (async () => {
278
305
  if (!modelsPresent(root, ["search"], list))
279
306
  throw new Error("the search model is not downloaded yet");
280
- server = await launch(await chooseDevice());
281
- server.child.on("exit", () => {
307
+ const started = await launch(await chooseDevice());
308
+ // Only this server's exit clears it: a restarted server must not be forgotten when the old one goes.
309
+ started.child.on("exit", () => {
310
+ if (server !== started) return;
282
311
  server = null;
312
+ llamaServing.clear();
313
+ memory.gone("llama");
283
314
  });
315
+ server = started;
284
316
  return server;
285
317
  })().finally(() => {
286
318
  starting = null;
@@ -288,6 +320,34 @@ export function createInference({
288
320
  return starting;
289
321
  };
290
322
 
323
+ // The llama models live in one server process, so the memory ceiling counts the process: its charge is the cost of
324
+ // the models it has served, and unloading it stops the process.
325
+ const llamaServing = new Set();
326
+ const stopServer = async () => {
327
+ const child = server?.child;
328
+ if (!child || child.exitCode !== null) return;
329
+ const exited = new Promise((resolve) => child.once("exit", resolve));
330
+ child.kill();
331
+ await exited;
332
+ };
333
+ /** The server, with room made for the model about to run on it. Call while the llama lane holds "llama" in use. */
334
+ const llamaFor = async (id) => {
335
+ const serving = new Set([...llamaServing, id]);
336
+ const mb = [...serving].reduce(
337
+ (sum, model) => sum + memoryCostMb(modelById(model, list)),
338
+ 0
339
+ );
340
+ await memory.admit("llama", mb, stopServer);
341
+ try {
342
+ const ready = await ensureServer();
343
+ llamaServing.add(id);
344
+ return ready;
345
+ } catch (error) {
346
+ if (!server) memory.gone("llama");
347
+ throw error;
348
+ }
349
+ };
350
+
291
351
  async function post(base, pathname, body) {
292
352
  const response = await fetch(`${base}${pathname}`, {
293
353
  method: "POST",
@@ -356,29 +416,62 @@ export function createInference({
356
416
  pump(backend);
357
417
  });
358
418
 
359
- // Models on one vocabulary share its tokenizer while any of them is open.
419
+ // Models on one vocabulary share its tokenizer while any of them is open; it goes with the last of them, and its
420
+ // memory is part of each model's cost.
360
421
  const tokenizers = new Map();
361
- const tokenizerOf = (Tokenizer, source) => {
422
+ const tokenizerUsers = new Map();
423
+ const tokenizerOf = (Tokenizer, source, user) => {
362
424
  const dir = modelDir(root, source);
363
- let tokenizer = tokenizers.get(dir)?.deref();
364
- if (!tokenizer) {
365
- const tokenizerPath = source.files.find((file) =>
425
+ if (!tokenizerUsers.has(dir)) tokenizerUsers.set(dir, new Set());
426
+ tokenizerUsers.get(dir).add(user);
427
+ if (!tokenizers.has(dir)) {
428
+ const tokenizerFile = source.files.find((file) =>
366
429
  file.path.endsWith("tokenizer.json")
367
- ).path;
430
+ );
368
431
  const configPath = source.files.find((file) =>
369
432
  file.path.endsWith("tokenizer_config.json")
370
433
  )?.path;
371
- tokenizer = new Tokenizer(
372
- JSON.parse(fs.readFileSync(path.join(dir, tokenizerPath), "utf8")),
373
- configPath
374
- ? JSON.parse(fs.readFileSync(path.join(dir, configPath), "utf8"))
375
- : {}
434
+ const tokenizerPath = path.join(dir, tokenizerFile.path);
435
+ // Saved under the file's pinned hash, so a new tokenizer.json never meets old tables.
436
+ const tablesPath = path.join(
437
+ root,
438
+ "tokenizers",
439
+ `${tokenizerFile.sha256}.bin`
440
+ );
441
+ const library = () =>
442
+ new Tokenizer(
443
+ JSON.parse(fs.readFileSync(tokenizerPath, "utf8")),
444
+ configPath
445
+ ? JSON.parse(fs.readFileSync(path.join(dir, configPath), "utf8"))
446
+ : {}
447
+ );
448
+ tokenizers.set(
449
+ dir,
450
+ loadCompactTokenizer(tokenizerPath, tablesPath) ?? library()
376
451
  );
377
- tokenizers.set(dir, new WeakRef(tokenizer));
378
452
  }
379
- return tokenizer;
453
+ return tokenizers.get(dir);
454
+ };
455
+ const releaseTokenizer = (dir, user) => {
456
+ const users = tokenizerUsers.get(dir);
457
+ users?.delete(user);
458
+ if (users?.size) return;
459
+ tokenizerUsers.delete(dir);
460
+ tokenizers.delete(dir);
380
461
  };
381
462
 
463
+ // An ONNX model is counted against the memory ceiling from just before it opens until it closes, idle or unloaded
464
+ // to make room. The ONNX lane runs one job at a time, so the models not in that job are always free to unload.
465
+ const closeSession = async (id) => {
466
+ const held = sessions.get(id);
467
+ sessions.delete(id);
468
+ memory.gone(id);
469
+ const ready = await held?.catch(() => null);
470
+ if (!ready) return;
471
+ clearTimeout(ready.timer);
472
+ releaseTokenizer(ready.tokenizerDir, id);
473
+ await ready.onnx.release?.();
474
+ };
382
475
  const session = async (id) => {
383
476
  const entry = modelById(id, list);
384
477
  if (!entry || entry.runtime !== "onnx")
@@ -391,6 +484,7 @@ export function createInference({
391
484
  : entry;
392
485
  if (!modelsPresent(root, [...new Set([id, source.id])], list))
393
486
  throw new Error(`${entry.name} is not downloaded yet`);
487
+ await memory.admit(id, memoryCostMb(entry), () => closeSession(id));
394
488
  const [{ Tokenizer }, ort] = await Promise.all([
395
489
  import("@huggingface/tokenizers"),
396
490
  import("onnxruntime-node"),
@@ -403,21 +497,22 @@ export function createInference({
403
497
  });
404
498
  return {
405
499
  ort,
406
- tokenizer: tokenizerOf(Tokenizer, source),
500
+ tokenizer: tokenizerOf(Tokenizer, source, id),
501
+ tokenizerDir: modelDir(root, source),
407
502
  prepare: entry.tokenizer?.prepare ?? ((text) => text),
408
503
  onnx,
409
504
  timer: null,
410
505
  };
411
506
  })();
412
507
  sessions.set(id, held);
413
- held.catch(() => sessions.delete(id));
508
+ held.catch(() => {
509
+ sessions.delete(id);
510
+ memory.gone(id);
511
+ });
414
512
  }
415
513
  const ready = await held;
416
514
  clearTimeout(ready.timer);
417
- ready.timer = setTimeout(() => {
418
- sessions.delete(id);
419
- ready.onnx.release?.();
420
- }, ONNX_IDLE_MS);
515
+ ready.timer = setTimeout(() => void closeSession(id), ONNX_IDLE_MS);
421
516
  ready.timer.unref?.();
422
517
  return ready;
423
518
  };
@@ -425,6 +520,15 @@ export function createInference({
425
520
  // Models made of several ONNX graphs (OCR's detector and recogniser, speech's encoder and decoder) open each graph
426
521
  // on first use and drop all of them together once idle.
427
522
  const graphs = new Map();
523
+ const closeGraphs = async (id) => {
524
+ const held = graphs.get(id);
525
+ graphs.delete(id);
526
+ memory.gone(id);
527
+ if (!held) return;
528
+ clearTimeout(held.timer);
529
+ for (const graph of held.open.values())
530
+ await graph.then((onnx) => onnx.release?.()).catch(() => {});
531
+ };
428
532
  const graphsOf = async (id) => {
429
533
  const entry = modelById(id, list);
430
534
  if (!entry || entry.runtime !== "onnx")
@@ -432,22 +536,18 @@ export function createInference({
432
536
  if (!modelsPresent(root, [id], list))
433
537
  throw new Error(`${entry.name} is not downloaded yet`);
434
538
  let held = graphs.get(id);
435
- if (!held)
436
- graphs.set(
437
- id,
438
- (held = {
439
- ort: await import("onnxruntime-node"),
440
- open: new Map(),
441
- timer: null,
442
- threads: entry.threads ?? 1,
443
- })
444
- );
539
+ if (!held) {
540
+ await memory.admit(id, memoryCostMb(entry), () => closeGraphs(id));
541
+ held = {
542
+ ort: await import("onnxruntime-node"),
543
+ open: new Map(),
544
+ timer: null,
545
+ threads: entry.threads ?? 1,
546
+ };
547
+ graphs.set(id, held);
548
+ }
445
549
  clearTimeout(held.timer);
446
- held.timer = setTimeout(() => {
447
- graphs.delete(id);
448
- for (const graph of held.open.values())
449
- graph.then((onnx) => onnx.release?.()).catch(() => {});
450
- }, ONNX_IDLE_MS);
550
+ held.timer = setTimeout(() => void closeGraphs(id), ONNX_IDLE_MS);
451
551
  held.timer.unref?.();
452
552
  const graph = (file) => {
453
553
  if (!held.open.has(file)) {
@@ -468,6 +568,16 @@ export function createInference({
468
568
  return { ort: held.ort, graph, dir: modelDir(root, entry) };
469
569
  };
470
570
 
571
+ /** Runs work with a model marked in use, so the memory ceiling never unloads it mid-job. */
572
+ const inUse = async (id, work) => {
573
+ const done = memory.use(id);
574
+ try {
575
+ return await work();
576
+ } finally {
577
+ done();
578
+ }
579
+ };
580
+
471
581
  /** Runs an ONNX model on each encoded input; returns the first output's leading row (its logits). */
472
582
  const logits = async (id, pairs, maxLength = 512) => {
473
583
  const { ort, tokenizer, prepare, onnx } = await session(id);
@@ -515,118 +625,128 @@ export function createInference({
515
625
  power,
516
626
  /** Embeds texts with the search model. kind: "query" | "code-query" | "document" (title is a path or "none"). */
517
627
  embed: (items, { lane = "handoff" } = {}) =>
518
- schedule("llama", lane, async () => {
519
- const { base } = await ensureServer();
520
- const input = items.map((item) =>
521
- typeof item === "string"
522
- ? item
523
- : item.kind === "document"
524
- ? `title: ${item.title || "none"} | text: ${item.text}`
525
- : `task: ${item.kind === "code-query" ? "code retrieval" : "search result"} | query: ${item.text}`
526
- );
527
- // A text too long for the model's window is retried alone, halved until it fits.
528
- const one = async (text) => {
529
- for (
530
- let cut = text;
531
- ;
532
- cut = cut.slice(0, Math.floor(cut.length / 2))
533
- ) {
534
- try {
535
- return (
536
- await post(base, "/v1/embeddings", {
537
- model: "search",
538
- input: [cut],
539
- })
540
- ).data[0];
541
- } catch (error) {
542
- if (cut.length < 200) throw error;
628
+ schedule("llama", lane, () =>
629
+ inUse("llama", async () => {
630
+ const { base } = await llamaFor("search");
631
+ const input = items.map((item) =>
632
+ typeof item === "string"
633
+ ? item
634
+ : item.kind === "document"
635
+ ? `title: ${item.title || "none"} | text: ${item.text}`
636
+ : `task: ${item.kind === "code-query" ? "code retrieval" : "search result"} | query: ${item.text}`
637
+ );
638
+ // A text too long for the model's window is retried alone, halved until it fits.
639
+ const one = async (text) => {
640
+ for (
641
+ let cut = text;
642
+ ;
643
+ cut = cut.slice(0, Math.floor(cut.length / 2))
644
+ ) {
645
+ try {
646
+ return (
647
+ await post(base, "/v1/embeddings", {
648
+ model: "search",
649
+ input: [cut],
650
+ })
651
+ ).data[0];
652
+ } catch (error) {
653
+ if (cut.length < 200) throw error;
654
+ }
655
+ }
656
+ };
657
+ const out = [];
658
+ for (let i = 0; i < input.length; i += 16) {
659
+ const slice = input.slice(i, i + 16);
660
+ const data = await post(base, "/v1/embeddings", {
661
+ model: "search",
662
+ input: slice,
663
+ })
664
+ .then((reply) => reply.data.sort((a, b) => a.index - b.index))
665
+ .catch(async () => {
666
+ const rows = [];
667
+ for (const text of slice) rows.push(await one(text));
668
+ return rows;
669
+ });
670
+ for (const row of data) {
671
+ const vector = Float32Array.from(row.embedding);
672
+ let norm = 0;
673
+ for (const value of vector) norm += value * value;
674
+ norm = Math.sqrt(norm) || 1;
675
+ for (let j = 0; j < vector.length; j += 1) vector[j] /= norm;
676
+ out.push(vector);
543
677
  }
544
678
  }
545
- };
546
- const out = [];
547
- for (let i = 0; i < input.length; i += 16) {
548
- const slice = input.slice(i, i + 16);
549
- const data = await post(base, "/v1/embeddings", {
550
- model: "search",
551
- input: slice,
552
- })
553
- .then((reply) => reply.data.sort((a, b) => a.index - b.index))
554
- .catch(async () => {
555
- const rows = [];
556
- for (const text of slice) rows.push(await one(text));
557
- return rows;
558
- });
559
- for (const row of data) {
560
- const vector = Float32Array.from(row.embedding);
561
- let norm = 0;
562
- for (const value of vector) norm += value * value;
563
- norm = Math.sqrt(norm) || 1;
564
- for (let j = 0; j < vector.length; j += 1) vector[j] /= norm;
565
- out.push(vector);
566
- }
567
- }
568
- return out;
569
- }),
679
+ return out;
680
+ })
681
+ ),
570
682
  /**
571
683
  * Embeds recordings and images with the search model's projector, in the same space as typed questions. items:
572
684
  * { title, data (a Buffer of WAV, MP3, FLAC, PNG or JPEG) }. A server started before the projector came down is
573
685
  * restarted once with it.
574
686
  */
575
687
  embedMedia: (items, { lane = "background" } = {}) =>
576
- schedule("llama", lane, async () => {
577
- let ready = await ensureServer();
578
- if (!ready.media) {
579
- ready.child.kill();
580
- server = null;
581
- ready = await ensureServer();
582
- }
583
- if (!ready.media)
584
- throw new Error(
585
- "the image and audio part of the search model is not downloaded yet"
586
- );
587
- // The marker that stands for a recording or an image in the text is the server's own, read once it has loaded.
588
- if (!ready.marker) {
589
- await post(ready.base, "/v1/embeddings", {
590
- model: "media",
591
- input: ["warm up"],
592
- });
593
- ready.marker = (
594
- await (await fetch(`${ready.base}/props?model=media`)).json()
595
- ).media_marker;
596
- }
597
- const out = [];
598
- for (const item of items) {
599
- const reply = await post(ready.base, "/v1/embeddings", {
600
- model: "media",
601
- input: [
602
- {
603
- prompt_string: `title: ${item.title || "none"} | text: ${ready.marker}`,
604
- multimodal_data: [Buffer.from(item.data).toString("base64")],
605
- },
606
- ],
607
- });
608
- const vector = Float32Array.from(reply.data[0].embedding);
609
- let norm = 0;
610
- for (const value of vector) norm += value * value;
611
- norm = Math.sqrt(norm) || 1;
612
- out.push(vector.map((value) => value / norm));
613
- }
614
- return out;
615
- }),
688
+ schedule("llama", lane, () =>
689
+ inUse("llama", async () => {
690
+ let ready = await llamaFor("media");
691
+ if (!ready.media) {
692
+ await stopServer();
693
+ ready = await llamaFor("media");
694
+ }
695
+ if (!ready.media)
696
+ throw new Error(
697
+ "the image and audio part of the search model is not downloaded yet"
698
+ );
699
+ // The marker that stands for a recording or an image in the text is the server's own, read once it has loaded.
700
+ if (!ready.marker) {
701
+ await post(ready.base, "/v1/embeddings", {
702
+ model: "media",
703
+ input: ["warm up"],
704
+ });
705
+ ready.marker = (
706
+ await (await fetch(`${ready.base}/props?model=media`)).json()
707
+ ).media_marker;
708
+ }
709
+ const out = [];
710
+ for (const item of items) {
711
+ const reply = await post(ready.base, "/v1/embeddings", {
712
+ model: "media",
713
+ input: [
714
+ {
715
+ prompt_string: `title: ${item.title || "none"} | text: ${ready.marker}`,
716
+ multimodal_data: [Buffer.from(item.data).toString("base64")],
717
+ },
718
+ ],
719
+ });
720
+ const vector = Float32Array.from(reply.data[0].embedding);
721
+ let norm = 0;
722
+ for (const value of vector) norm += value * value;
723
+ norm = Math.sqrt(norm) || 1;
724
+ out.push(vector.map((value) => value / norm));
725
+ }
726
+ return out;
727
+ })
728
+ ),
616
729
  /** Raw logits rows from an ONNX sequence classifier, for models calibrated on logit margins. */
617
730
  logits: (id, texts, { lane = "handoff", maxLength } = {}) =>
618
731
  schedule("onnx", lane, () =>
619
- logits(
620
- id,
621
- texts.map((text) => [text]),
622
- maxLength
732
+ inUse(id, () =>
733
+ logits(
734
+ id,
735
+ texts.map((text) => [text]),
736
+ maxLength
737
+ )
623
738
  )
624
739
  ),
625
740
  /** Runs work against a multi-graph ONNX model in the scheduler: run({ ort, graph(file), dir }) opens graphs by file path. */
626
741
  onnx: (id, run, { lane = "handoff" } = {}) =>
627
- schedule("onnx", lane, async () => run(await graphsOf(id))),
742
+ schedule("onnx", lane, () =>
743
+ inUse(id, async () => run(await graphsOf(id)))
744
+ ),
628
745
  running: () => Boolean(server && server.child.exitCode === null),
629
- /** What is in memory now: the router's models and their state, and the ONNX models open (for `tryharness local`). */
746
+ /**
747
+ * What is in memory now: the router's models and their state, the ONNX models open (for `tryharness local`), and
748
+ * the memory ceiling with what is counted against it.
749
+ */
630
750
  loaded: async () => {
631
751
  let llama = [];
632
752
  if (server && server.child.exitCode === null) {
@@ -639,25 +759,16 @@ export function createInference({
639
759
  }));
640
760
  } catch {}
641
761
  }
642
- return { llama, onnx: [...sessions.keys(), ...graphs.keys()] };
762
+ return {
763
+ llama,
764
+ onnx: [...sessions.keys(), ...graphs.keys()],
765
+ memory: memory.status(),
766
+ };
643
767
  },
644
768
  stop() {
645
- if (server?.child.exitCode === null) server.child.kill();
646
- server = null;
647
- for (const held of sessions.values())
648
- held
649
- .then((ready) => {
650
- clearTimeout(ready.timer);
651
- ready.onnx.release?.();
652
- })
653
- .catch(() => {});
654
- sessions.clear();
655
- for (const held of graphs.values()) {
656
- clearTimeout(held.timer);
657
- for (const graph of held.open.values())
658
- graph.then((onnx) => onnx.release?.()).catch(() => {});
659
- }
660
- graphs.clear();
769
+ void stopServer();
770
+ for (const id of [...sessions.keys()]) void closeSession(id);
771
+ for (const id of [...graphs.keys()]) void closeGraphs(id);
661
772
  },
662
773
  };
663
774
  }