rigscope 0.1.8 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. package/.next/BUILD_ID +1 -1
  2. package/.next/app-path-routes-manifest.json +1 -1
  3. package/.next/build-manifest.json +2 -2
  4. package/.next/server/app/_global-error/page_client-reference-manifest.js +1 -1
  5. package/.next/server/app/_global-error.html +1 -1
  6. package/.next/server/app/_global-error.rsc +1 -1
  7. package/.next/server/app/_global-error.segments/_full.segment.rsc +1 -1
  8. package/.next/server/app/_global-error.segments/_global-error/__PAGE__.segment.rsc +1 -1
  9. package/.next/server/app/_global-error.segments/_tree.segment.rsc +1 -1
  10. package/.next/server/app/_not-found/page_client-reference-manifest.js +1 -1
  11. package/.next/server/app/_not-found.html +1 -1
  12. package/.next/server/app/_not-found.rsc +3 -3
  13. package/.next/server/app/_not-found.segments/_full.segment.rsc +3 -3
  14. package/.next/server/app/_not-found.segments/_not-found/__PAGE__.segment.rsc +3 -3
  15. package/.next/server/app/_not-found.segments/_tree.segment.rsc +2 -2
  16. package/.next/server/app/administration/page_client-reference-manifest.js +1 -1
  17. package/.next/server/app/administration.html +1 -1
  18. package/.next/server/app/administration.rsc +3 -3
  19. package/.next/server/app/administration.segments/_full.segment.rsc +3 -3
  20. package/.next/server/app/administration.segments/_tree.segment.rsc +2 -2
  21. package/.next/server/app/administration.segments/administration/__PAGE__.segment.rsc +3 -3
  22. package/.next/server/app/alerts/page_client-reference-manifest.js +1 -1
  23. package/.next/server/app/alerts.html +1 -1
  24. package/.next/server/app/alerts.rsc +3 -3
  25. package/.next/server/app/alerts.segments/_full.segment.rsc +3 -3
  26. package/.next/server/app/alerts.segments/_tree.segment.rsc +2 -2
  27. package/.next/server/app/alerts.segments/alerts/__PAGE__.segment.rsc +3 -3
  28. package/.next/server/app/connections/page.js +1 -1
  29. package/.next/server/app/connections/page_client-reference-manifest.js +1 -1
  30. package/.next/server/app/connections.html +1 -1
  31. package/.next/server/app/connections.rsc +4 -4
  32. package/.next/server/app/connections.segments/_full.segment.rsc +4 -4
  33. package/.next/server/app/connections.segments/_tree.segment.rsc +2 -2
  34. package/.next/server/app/connections.segments/connections/__PAGE__.segment.rsc +4 -4
  35. package/.next/server/app/history/page.js +2 -2
  36. package/.next/server/app/history/page_client-reference-manifest.js +1 -1
  37. package/.next/server/app/history.html +1 -1
  38. package/.next/server/app/history.rsc +6 -6
  39. package/.next/server/app/history.segments/_full.segment.rsc +6 -6
  40. package/.next/server/app/history.segments/_tree.segment.rsc +2 -2
  41. package/.next/server/app/history.segments/history/__PAGE__.segment.rsc +6 -6
  42. package/.next/server/app/index.html +1 -1
  43. package/.next/server/app/index.rsc +3 -3
  44. package/.next/server/app/index.segments/__PAGE__.segment.rsc +3 -3
  45. package/.next/server/app/index.segments/_full.segment.rsc +3 -3
  46. package/.next/server/app/index.segments/_tree.segment.rsc +2 -2
  47. package/.next/server/app/login/page_client-reference-manifest.js +1 -1
  48. package/.next/server/app/login.html +1 -1
  49. package/.next/server/app/login.rsc +3 -3
  50. package/.next/server/app/login.segments/_full.segment.rsc +3 -3
  51. package/.next/server/app/login.segments/_tree.segment.rsc +2 -2
  52. package/.next/server/app/login.segments/login/__PAGE__.segment.rsc +3 -3
  53. package/.next/server/app/machines/[machineId]/page_client-reference-manifest.js +1 -1
  54. package/.next/server/app/page_client-reference-manifest.js +1 -1
  55. package/.next/server/app/processes/page_client-reference-manifest.js +1 -1
  56. package/.next/server/app/processes.html +1 -1
  57. package/.next/server/app/processes.rsc +3 -3
  58. package/.next/server/app/processes.segments/_full.segment.rsc +3 -3
  59. package/.next/server/app/processes.segments/_tree.segment.rsc +2 -2
  60. package/.next/server/app/processes.segments/processes/__PAGE__.segment.rsc +3 -3
  61. package/.next/server/app-paths-manifest.json +1 -1
  62. package/.next/server/chunks/161.js +1 -1
  63. package/.next/server/chunks/168.js +1 -1
  64. package/.next/server/middleware-build-manifest.js +1 -1
  65. package/.next/server/pages/404.html +1 -1
  66. package/.next/server/pages/500.html +1 -1
  67. package/.next/static/chunks/408-351fe4a1e9a2d556.js +1 -0
  68. package/.next/static/chunks/939-92930fde48061e20.js +1 -0
  69. package/.next/static/chunks/app/connections/page-7d18b901f2338f20.js +1 -0
  70. package/.next/static/chunks/app/history/page-adc25194b30c689d.js +3 -0
  71. package/.next/static/css/{f5d3162842dcac31.css → 2abef6d49c1c2970.css} +1 -1
  72. package/ARCHITECTURE.md +43 -15
  73. package/CHANGELOG.md +62 -0
  74. package/README.md +10 -3
  75. package/dist/cli.cjs +868 -511
  76. package/dist/collector.cjs +603 -120
  77. package/dist/hub.cjs +3429 -1451
  78. package/dist/ops.cjs +1178 -888
  79. package/dist/system.cjs +1322 -363
  80. package/docs/configuration.md +34 -1
  81. package/docs/installation.md +12 -6
  82. package/docs/monitoring.md +60 -17
  83. package/docs/operations.md +78 -22
  84. package/docs/system-installation.md +11 -4
  85. package/docs/telemetry-storage.md +258 -0
  86. package/package.json +3 -1
  87. package/packaging/collector.env.example +6 -0
  88. package/packaging/system/rigscope-collector.service.example +3 -0
  89. package/.next/static/chunks/408-9fdba9e4e956ef08.js +0 -1
  90. package/.next/static/chunks/939-514e554087419c71.js +0 -1
  91. package/.next/static/chunks/app/connections/page-75abd16fc495fcbd.js +0 -1
  92. package/.next/static/chunks/app/history/page-a25ba8fab7bd7cb9.js +0 -3
  93. /package/.next/static/{ip43TIgmaByEBn2GFtvfO → JbaLjudQSxrMUD_VHPrnD}/_buildManifest.js +0 -0
  94. /package/.next/static/{ip43TIgmaByEBn2GFtvfO → JbaLjudQSxrMUD_VHPrnD}/_ssgManifest.js +0 -0
@@ -108,13 +108,16 @@ function filterInventory(inv, sharing) {
108
108
  swapTotalBytes: policy.swap ? inv.memory.swapTotalBytes : null
109
109
  },
110
110
  gpus: policy.gpus ? inv.gpus.map((gpu) => ({ ...gpu })) : [],
111
- filesystems: policy.disks ? inv.filesystems.map((fs2) => ({ ...fs2 })) : [],
111
+ filesystems: policy.disks ? inv.filesystems.map((fs3) => ({ ...fs3 })) : [],
112
112
  disks: policy.disks ? inv.disks.map((disk) => ({ ...disk })) : [],
113
113
  network,
114
114
  capabilities
115
115
  };
116
116
  }
117
117
 
118
+ // collector/destinations/queue.ts
119
+ var DEFAULT_COLLECTOR_QUEUE_MODE = "memory";
120
+
118
121
  // collector/config.ts
119
122
  var ConfigError = class extends Error {
120
123
  issues;
@@ -320,6 +323,18 @@ function parseDestinations(env, issues, destinations) {
320
323
  }
321
324
  }
322
325
  }
326
+ var QUEUE_MODES = ["memory", "durable"];
327
+ function parseQueueMode(env, issues) {
328
+ const raw = env["COLLECTOR_QUEUE_MODE"]?.trim();
329
+ if (raw === void 0 || raw === "") {
330
+ return DEFAULT_COLLECTOR_QUEUE_MODE;
331
+ }
332
+ if (!QUEUE_MODES.includes(raw)) {
333
+ issues.push(`COLLECTOR_QUEUE_MODE must be one of ${QUEUE_MODES.join(" or ")}`);
334
+ return DEFAULT_COLLECTOR_QUEUE_MODE;
335
+ }
336
+ return raw;
337
+ }
323
338
  function parseIntervalMs(env, issues) {
324
339
  const raw = env["COLLECTOR_INTERVAL_MS"]?.trim();
325
340
  if (raw === void 0 || raw === "") {
@@ -350,6 +365,7 @@ function parseCollectorConfig(env) {
350
365
  }
351
366
  const destinations = [];
352
367
  const intervalMs = parseIntervalMs(env, issues);
368
+ const queueMode = parseQueueMode(env, issues);
353
369
  parseDestinations(env, issues, destinations);
354
370
  if (issues.length > 0) {
355
371
  throw new ConfigError(issues);
@@ -359,7 +375,8 @@ function parseCollectorConfig(env) {
359
375
  hostname,
360
376
  hostnameIsMockDefault: rawHostname === "",
361
377
  intervalMs,
362
- destinations
378
+ destinations,
379
+ queueMode
363
380
  };
364
381
  }
365
382
 
@@ -377,8 +394,8 @@ var ProcessLeaseError = class extends Error {
377
394
  this.code = code;
378
395
  }
379
396
  };
380
- function acquireProcessLease(path2) {
381
- const absolute = (0, import_node_path.resolve)(path2);
397
+ function acquireProcessLease(path3) {
398
+ const absolute = (0, import_node_path.resolve)(path3);
382
399
  (0, import_node_fs.mkdirSync)((0, import_node_path.dirname)(absolute), { recursive: true, mode: 448 });
383
400
  const parent = (0, import_node_fs.lstatSync)((0, import_node_path.dirname)(absolute));
384
401
  if (!parent.isDirectory() || (parent.mode & 63) !== 0) {
@@ -432,10 +449,10 @@ function acquireProcessLease(path2) {
432
449
  "runtime lease could not be acquired after repeated stale-lock recovery"
433
450
  );
434
451
  }
435
- function inspectLock(path2) {
452
+ function inspectLock(path3) {
436
453
  let stats;
437
454
  try {
438
- stats = (0, import_node_fs.lstatSync)(path2);
455
+ stats = (0, import_node_fs.lstatSync)(path3);
439
456
  } catch (error) {
440
457
  if (error.code === "ENOENT") return null;
441
458
  throw new ProcessLeaseError(
@@ -448,7 +465,7 @@ function inspectLock(path2) {
448
465
  }
449
466
  let raw;
450
467
  try {
451
- raw = (0, import_node_fs.readFileSync)(path2, "utf8");
468
+ raw = (0, import_node_fs.readFileSync)(path3, "utf8");
452
469
  } catch (error) {
453
470
  if (error.code === "ENOENT") return null;
454
471
  throw new ProcessLeaseError(
@@ -485,10 +502,10 @@ function inspectLock(path2) {
485
502
  );
486
503
  }
487
504
  }
488
- function createRelease(path2, inode) {
505
+ function createRelease(path3, inode) {
489
506
  let released = false;
490
507
  const cleanup = () => {
491
- removeIfSameInode(path2, inode);
508
+ removeIfSameInode(path3, inode);
492
509
  };
493
510
  const onExit = () => {
494
511
  if (released) return;
@@ -503,10 +520,10 @@ function createRelease(path2, inode) {
503
520
  cleanup();
504
521
  };
505
522
  }
506
- function removeIfSameInode(path2, inode) {
523
+ function removeIfSameInode(path3, inode) {
507
524
  try {
508
- const stat = (0, import_node_fs.lstatSync)(path2);
509
- if (stat.ino === inode.ino && stat.dev === inode.dev) (0, import_node_fs.unlinkSync)(path2);
525
+ const stat = (0, import_node_fs.lstatSync)(path3);
526
+ if (stat.ino === inode.ino && stat.dev === inode.dev) (0, import_node_fs.unlinkSync)(path3);
510
527
  } catch {
511
528
  }
512
529
  }
@@ -515,10 +532,10 @@ function removeIfSameInode(path2, inode) {
515
532
  function defaultStatePath() {
516
533
  return (0, import_node_path2.join)(process.env.XDG_STATE_HOME || (0, import_node_path2.join)((0, import_node_os.homedir)(), ".local", "state"), "rigscope", "collector.json");
517
534
  }
518
- function readState(path2) {
519
- const stat = (0, import_node_fs2.lstatSync)(path2);
535
+ function readState(path3) {
536
+ const stat = (0, import_node_fs2.lstatSync)(path3);
520
537
  if (!stat.isFile() || stat.size > 64 * 1024 || (stat.mode & 63) !== 0) throw new Error("collector state must be a private regular file");
521
- const parsed = JSON.parse((0, import_node_fs2.readFileSync)(path2, "utf8"));
538
+ const parsed = JSON.parse((0, import_node_fs2.readFileSync)(path3, "utf8"));
522
539
  if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) throw new Error("invalid collector state");
523
540
  const raw = parsed;
524
541
  if (Object.hasOwn(raw, "version") || Object.hasOwn(raw, "formatVersion")) {
@@ -543,13 +560,13 @@ function readState(path2) {
543
560
  ...pendingEnrollments === void 0 ? {} : { pendingEnrollments }
544
561
  };
545
562
  }
546
- function updateInstallation(path2, change = (s) => s) {
547
- (0, import_node_fs2.mkdirSync)((0, import_node_path2.dirname)(path2), { recursive: true, mode: 448 });
548
- const directoryStat = (0, import_node_fs2.lstatSync)((0, import_node_path2.dirname)(path2));
563
+ function updateInstallation(path3, change = (s) => s) {
564
+ (0, import_node_fs2.mkdirSync)((0, import_node_path2.dirname)(path3), { recursive: true, mode: 448 });
565
+ const directoryStat = (0, import_node_fs2.lstatSync)((0, import_node_path2.dirname)(path3));
549
566
  if (!directoryStat.isDirectory() || (directoryStat.mode & 63) !== 0) {
550
567
  throw new ConfigError(["collector state requires a private directory (0700)"]);
551
568
  }
552
- const lock = `${path2}.lock`;
569
+ const lock = `${path3}.lock`;
553
570
  let releaseLock;
554
571
  try {
555
572
  releaseLock = acquireProcessLease(lock);
@@ -566,8 +583,8 @@ function updateInstallation(path2, change = (s) => s) {
566
583
  try {
567
584
  let state;
568
585
  try {
569
- (0, import_node_fs2.lstatSync)(path2);
570
- state = readState(path2);
586
+ (0, import_node_fs2.lstatSync)(path3);
587
+ state = readState(path3);
571
588
  } catch (error) {
572
589
  if (error.code !== "ENOENT") throw error;
573
590
  state = { collectorId: (0, import_node_crypto2.randomUUID)(), destinations: [] };
@@ -578,7 +595,7 @@ function updateInstallation(path2, change = (s) => s) {
578
595
  destinations: changed.destinations,
579
596
  ...changed.pendingEnrollments === void 0 ? {} : { pendingEnrollments: changed.pendingEnrollments }
580
597
  };
581
- temporary = `${path2}.${(0, import_node_crypto2.randomUUID)()}.tmp`;
598
+ temporary = `${path3}.${(0, import_node_crypto2.randomUUID)()}.tmp`;
582
599
  const out = (0, import_node_fs2.openSync)(temporary, "wx", 384);
583
600
  try {
584
601
  (0, import_node_fs2.writeFileSync)(out, JSON.stringify(next) + "\n");
@@ -586,9 +603,9 @@ function updateInstallation(path2, change = (s) => s) {
586
603
  } finally {
587
604
  (0, import_node_fs2.closeSync)(out);
588
605
  }
589
- (0, import_node_fs2.renameSync)(temporary, path2);
606
+ (0, import_node_fs2.renameSync)(temporary, path3);
590
607
  temporary = void 0;
591
- const directory = (0, import_node_fs2.openSync)((0, import_node_path2.dirname)(path2), "r");
608
+ const directory = (0, import_node_fs2.openSync)((0, import_node_path2.dirname)(path3), "r");
592
609
  try {
593
610
  (0, import_node_fs2.fsyncSync)(directory);
594
611
  } finally {
@@ -606,14 +623,14 @@ function updateInstallation(path2, change = (s) => s) {
606
623
 
607
624
  // collector/enroll.ts
608
625
  var import_node_crypto3 = require("node:crypto");
609
- async function enrollCollector(path2, connectUrl, enrollmentToken, sharing, replace = false) {
626
+ async function enrollCollector(path3, connectUrl, enrollmentToken, sharing, replace = false) {
610
627
  const issues = [];
611
628
  const url = validateHubUrl(connectUrl, "enrollment destination", issues);
612
629
  if (!url || issues.length) throw new Error("invalid enrollment destination");
613
630
  if (!/^[\x21-\x7e]{1,256}$/.test(enrollmentToken)) throw new Error("invalid enrollment token");
614
- const state = updateInstallation(path2);
631
+ const state = updateInstallation(path3);
615
632
  if (!replace && state.destinations.some((d) => d.url === url.href) || !state.destinations.some((d) => d.url === url.href) && state.destinations.length >= 8) throw new Error("destination already enrolled or destination budget exhausted");
616
- const prepared = updateInstallation(path2, (current) => {
633
+ const prepared = updateInstallation(path3, (current) => {
617
634
  if (current.pendingEnrollments?.some((p) => p.url === url.href)) return current;
618
635
  return { ...current, pendingEnrollments: [
619
636
  ...current.pendingEnrollments ?? [],
@@ -659,7 +676,7 @@ async function enrollCollector(path2, connectUrl, enrollmentToken, sharing, repl
659
676
  if (result.collectorId !== state.collectorId || typeof result.machineId !== "string" || !result.machineId || typeof result.token !== "string") throw new Error("invalid enrollment response");
660
677
  const destination = parseCollectorConfig({ COLLECTOR_HUBS: JSON.stringify([{ url: url.href, token: result.token, sharing }]) }).destinations[0];
661
678
  try {
662
- updateInstallation(path2, (current) => {
679
+ updateInstallation(path3, (current) => {
663
680
  if (current.collectorId !== state.collectorId || !replace && current.destinations.some((d) => d.url === url.href) || !current.destinations.some((d) => d.url === url.href) && current.destinations.length >= 8) throw new Error("collector state changed during enrollment");
664
681
  return {
665
682
  ...current,
@@ -678,9 +695,9 @@ var nodeFs = __toESM(require("node:fs"));
678
695
  var BOOT_ID_UNAVAILABLE = "unavailable";
679
696
  var BOOT_ID_PATH = "/proc/sys/kernel/random/boot_id";
680
697
  var BOOT_ID_PATTERN = /^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$/;
681
- function readBootIdentity(fs2 = nodeFs) {
698
+ function readBootIdentity(fs3 = nodeFs) {
682
699
  try {
683
- const raw = String(fs2.readFileSync(BOOT_ID_PATH, "utf8")).trim();
700
+ const raw = String(fs3.readFileSync(BOOT_ID_PATH, "utf8")).trim();
684
701
  if (BOOT_ID_PATTERN.test(raw)) {
685
702
  return { bootId: raw.toLowerCase(), available: true };
686
703
  }
@@ -758,13 +775,17 @@ var OutboxBudget = class {
758
775
  }
759
776
  return largest;
760
777
  }
761
- /** Physical counterpart of `largest`, using measured main+WAL bytes. */
778
+ /**
779
+ * Physical counterpart of `largest`, using measured main+WAL bytes. A memory
780
+ * queue owns no file, so it is never a physical-budget victim.
781
+ */
762
782
  largestByDisk(exclude) {
763
783
  let largest = null;
764
784
  let largestBytes = -1;
765
785
  for (const outbox of this.outboxes) {
766
786
  if (exclude?.has(outbox)) continue;
767
787
  const bytes = outbox.diskBytes();
788
+ if (bytes === 0) continue;
768
789
  if (bytes > largestBytes) {
769
790
  largest = outbox;
770
791
  largestBytes = bytes;
@@ -779,6 +800,10 @@ var OutboxBudget = class {
779
800
  }
780
801
  };
781
802
 
803
+ // collector/destinations/connection.ts
804
+ var fs2 = __toESM(require("node:fs"));
805
+ var path2 = __toESM(require("node:path"));
806
+
782
807
  // collector/destinations/outbox.ts
783
808
  var import_node_crypto5 = require("node:crypto");
784
809
  var fs = __toESM(require("node:fs"));
@@ -853,6 +878,7 @@ function asRow(value) {
853
878
  };
854
879
  }
855
880
  var DestinationOutbox = class _DestinationOutbox {
881
+ queueMode = "durable";
856
882
  db;
857
883
  limits;
858
884
  budget;
@@ -1101,7 +1127,8 @@ var DestinationOutbox = class _DestinationOutbox {
1101
1127
  inFlightRecords: 0,
1102
1128
  droppedRecords: this.droppedSinceOpen,
1103
1129
  replaying: false,
1104
- lastAckAt: this.lastAckAt
1130
+ lastAckAt: this.lastAckAt,
1131
+ queueMode: this.queueMode
1105
1132
  };
1106
1133
  }
1107
1134
  const pending = this.db.prepare("SELECT captured_at, bytes FROM outbox_records WHERE state = 'pending'").all();
@@ -1122,7 +1149,8 @@ var DestinationOutbox = class _DestinationOutbox {
1122
1149
  inFlightRecords: inFlight,
1123
1150
  droppedRecords: this.droppedSinceOpen,
1124
1151
  replaying: pending.length > 0,
1125
- lastAckAt: this.lastAckAt
1152
+ lastAckAt: this.lastAckAt,
1153
+ queueMode: this.queueMode
1126
1154
  };
1127
1155
  }
1128
1156
  /** Logical bytes of stored record JSON (pending plus in-flight). */
@@ -1330,8 +1358,13 @@ var DestinationOutbox = class _DestinationOutbox {
1330
1358
  let guard = 0;
1331
1359
  while (budget2.diskOverBy() > 0 && guard < 256) {
1332
1360
  guard += 1;
1333
- const target = budget2.largestByDisk(tried);
1334
- if (target === null) break;
1361
+ const candidate = budget2.largestByDisk(tried);
1362
+ if (candidate === null) break;
1363
+ if (!(candidate instanceof _DestinationOutbox)) {
1364
+ tried.add(candidate);
1365
+ continue;
1366
+ }
1367
+ const target = candidate;
1335
1368
  if (!target.canReclaimPhysical()) {
1336
1369
  tried.add(target);
1337
1370
  continue;
@@ -1546,10 +1579,420 @@ var DestinationOutbox = class _DestinationOutbox {
1546
1579
  }
1547
1580
  };
1548
1581
 
1582
+ // collector/destinations/memory-queue.ts
1583
+ var refKey = ({ runId, sequence }) => `${runId}\0${sequence}`;
1584
+ var NO_DISK = {
1585
+ bytes: 0,
1586
+ mainBytes: 0,
1587
+ walBytes: 0,
1588
+ logicalBytes: 0,
1589
+ limitBytes: 0,
1590
+ overBy: 0,
1591
+ pressure: false,
1592
+ walBudgetBytes: 0,
1593
+ checkpointPages: 0,
1594
+ pageCount: 0,
1595
+ maxPageCount: 0,
1596
+ freePages: 0,
1597
+ autoVacuum: 0
1598
+ };
1599
+ var MemoryDestinationQueue = class {
1600
+ queueMode = "memory";
1601
+ limits;
1602
+ budget;
1603
+ allocateSequence;
1604
+ identity;
1605
+ now;
1606
+ entries = [];
1607
+ byRef = /* @__PURE__ */ new Map();
1608
+ nextId = 1;
1609
+ totalBytes = 0;
1610
+ /** id of the newest metrics record not yet offered to `takeLive`. */
1611
+ liveId = null;
1612
+ droppedSinceOpen = 0;
1613
+ lastAckAt = null;
1614
+ closed = false;
1615
+ constructor(opts) {
1616
+ this.limits = { ...DEFAULT_OUTBOX_LIMITS, ...opts.limits };
1617
+ this.budget = opts.budget ?? null;
1618
+ this.allocateSequence = opts.allocateSequence;
1619
+ this.identity = opts.identity;
1620
+ this.now = opts.now ?? Date.now;
1621
+ this.budget?.register(this);
1622
+ }
1623
+ // -------------------------------------------------------------------------
1624
+ // Writes
1625
+ // -------------------------------------------------------------------------
1626
+ /**
1627
+ * Admit one filtered metrics record. Age and byte caps are applied first so
1628
+ * the newest sample always wins its slot, exactly as the durable queue sheds
1629
+ * the oldest pending telemetry under pressure. The shared budget and gap cap
1630
+ * are then settled so admission itself keeps retained data bounded, even if
1631
+ * the maintenance tick never runs. Never throws.
1632
+ */
1633
+ append(record) {
1634
+ if (this.closed) return;
1635
+ try {
1636
+ const json = JSON.stringify(record);
1637
+ const bytes = Buffer.byteLength(json, "utf8");
1638
+ if (bytes > this.limits.maxBytes) {
1639
+ this.droppedSinceOpen += 1;
1640
+ this.writeGap("outbox-bytes", record.capturedAt, record.capturedAt, 1);
1641
+ this.enforceAdmission();
1642
+ return;
1643
+ }
1644
+ this.enforceAge();
1645
+ this.enforceBytes(this.limits.maxBytes - bytes);
1646
+ this.insertJson(record.kind, record, json, bytes);
1647
+ this.enforceAdmission();
1648
+ } catch {
1649
+ this.droppedSinceOpen += 1;
1650
+ }
1651
+ }
1652
+ /** Reapply the current privacy policy to every retained metrics record. */
1653
+ filterPending(transform) {
1654
+ if (this.closed) return;
1655
+ for (const entry of [...this.entries]) {
1656
+ if (entry.kind !== "metrics") continue;
1657
+ const parsed = this.parse(entry);
1658
+ if (parsed === null) continue;
1659
+ const json = JSON.stringify(transform(parsed));
1660
+ const bytes = Buffer.byteLength(json, "utf8");
1661
+ this.totalBytes += bytes - entry.bytes;
1662
+ entry.json = json;
1663
+ entry.bytes = bytes;
1664
+ }
1665
+ this.totalBytes = Math.max(0, this.totalBytes);
1666
+ }
1667
+ // -------------------------------------------------------------------------
1668
+ // Reads
1669
+ // -------------------------------------------------------------------------
1670
+ /**
1671
+ * Newest metrics record appended since the last `takeLive`. It stays pending
1672
+ * so a lost live send is still replayed later, matching the durable queue.
1673
+ */
1674
+ takeLive() {
1675
+ if (this.closed || this.liveId === null) return null;
1676
+ const id2 = this.liveId;
1677
+ this.liveId = null;
1678
+ const entry = this.entries.find((candidate) => candidate.id === id2);
1679
+ if (entry === void 0 || entry.state !== "pending" || entry.kind !== "metrics") return null;
1680
+ return this.parse(entry);
1681
+ }
1682
+ /**
1683
+ * Oldest pending records, oldest first, excluding the record currently held
1684
+ * for the live slot. Bounded by both count and bytes, but always at least one
1685
+ * record so a single large sample cannot stall the queue.
1686
+ */
1687
+ takeReplay(maxRecords, maxBytes) {
1688
+ if (this.closed || !Number.isFinite(maxRecords) || maxRecords < 1) return [];
1689
+ const records = [];
1690
+ let bytes = 0;
1691
+ for (const entry of this.entries) {
1692
+ if (records.length >= Math.floor(maxRecords)) break;
1693
+ if (entry.state !== "pending" || entry.id === this.liveId) continue;
1694
+ if (records.length > 0 && bytes + entry.bytes > maxBytes) break;
1695
+ const parsed = this.parse(entry);
1696
+ if (parsed === null) continue;
1697
+ records.push(parsed);
1698
+ bytes += entry.bytes;
1699
+ }
1700
+ return records;
1701
+ }
1702
+ // -------------------------------------------------------------------------
1703
+ // Delivery state
1704
+ // -------------------------------------------------------------------------
1705
+ markInFlight(refs) {
1706
+ if (this.closed) return;
1707
+ for (const ref of refs) {
1708
+ const entry = this.byRef.get(refKey(ref));
1709
+ if (entry !== void 0 && entry.state === "pending") {
1710
+ entry.state = "inflight";
1711
+ entry.offered = true;
1712
+ }
1713
+ }
1714
+ }
1715
+ release(refs) {
1716
+ if (this.closed) return;
1717
+ for (const ref of refs) {
1718
+ const entry = this.byRef.get(refKey(ref));
1719
+ if (entry !== void 0 && entry.state === "inflight") entry.state = "pending";
1720
+ }
1721
+ }
1722
+ releaseAll() {
1723
+ if (this.closed) return;
1724
+ for (const entry of this.entries) if (entry.state === "inflight") entry.state = "pending";
1725
+ }
1726
+ /** Accepted and rejected records both leave the queue. Unknown refs are ignored. */
1727
+ acknowledge(accepted, rejected) {
1728
+ if (this.closed) return;
1729
+ if (accepted.length === 0 && rejected.length === 0) return;
1730
+ for (const ref of [...accepted, ...rejected]) {
1731
+ const key = refKey(ref);
1732
+ const entry = this.byRef.get(key);
1733
+ if (entry === void 0) continue;
1734
+ this.byRef.delete(key);
1735
+ const index = this.entries.indexOf(entry);
1736
+ if (index >= 0) this.entries.splice(index, 1);
1737
+ if (this.liveId === entry.id) this.liveId = null;
1738
+ this.totalBytes = Math.max(0, this.totalBytes - entry.bytes);
1739
+ }
1740
+ this.lastAckAt = this.isoNow();
1741
+ }
1742
+ // -------------------------------------------------------------------------
1743
+ // Retention
1744
+ // -------------------------------------------------------------------------
1745
+ enforceLimits() {
1746
+ if (this.closed) return;
1747
+ this.enforceAge();
1748
+ this.enforceBytes(this.limits.maxBytes);
1749
+ this.enforceAdmission();
1750
+ }
1751
+ /** Shared-budget victim: shed this queue's oldest pending metrics records. */
1752
+ evictForBudget(reason) {
1753
+ if (this.closed) return 0;
1754
+ const victims = this.entries.filter((entry) => entry.state === "pending" && entry.kind === "metrics").slice(0, 16);
1755
+ if (victims.length === 0) return 0;
1756
+ const evicted = this.evict(victims, reason);
1757
+ this.enforceGapCap();
1758
+ return evicted;
1759
+ }
1760
+ status() {
1761
+ let pendingRecords = 0;
1762
+ let pendingBytes = 0;
1763
+ let inFlightRecords = 0;
1764
+ let oldest = null;
1765
+ for (const entry of this.entries) {
1766
+ if (entry.state === "inflight") {
1767
+ inFlightRecords += 1;
1768
+ continue;
1769
+ }
1770
+ pendingRecords += 1;
1771
+ pendingBytes += entry.bytes;
1772
+ if (oldest === null || entry.capturedAtMs < Date.parse(oldest)) oldest = entry.capturedAt;
1773
+ }
1774
+ return {
1775
+ pendingRecords,
1776
+ pendingBytes,
1777
+ oldestPendingCapturedAt: oldest,
1778
+ inFlightRecords,
1779
+ droppedRecords: this.droppedSinceOpen,
1780
+ replaying: pendingRecords > 0,
1781
+ lastAckAt: this.lastAckAt,
1782
+ queueMode: this.queueMode
1783
+ };
1784
+ }
1785
+ bytes() {
1786
+ return this.totalBytes;
1787
+ }
1788
+ /** A RAM queue owns no file: this is what makes the mode observable. */
1789
+ diskBytes() {
1790
+ return 0;
1791
+ }
1792
+ diskStatus() {
1793
+ return { ...NO_DISK, logicalBytes: this.totalBytes };
1794
+ }
1795
+ checkpoint() {
1796
+ }
1797
+ attachBudget(budget2) {
1798
+ this.budget = budget2;
1799
+ }
1800
+ close() {
1801
+ if (this.closed) return;
1802
+ this.closed = true;
1803
+ this.budget?.unregister(this);
1804
+ this.entries.length = 0;
1805
+ this.byRef.clear();
1806
+ this.totalBytes = 0;
1807
+ this.liveId = null;
1808
+ }
1809
+ // -------------------------------------------------------------------------
1810
+ // Internals
1811
+ // -------------------------------------------------------------------------
1812
+ isoNow() {
1813
+ return new Date(this.now()).toISOString();
1814
+ }
1815
+ insertJson(kind, record, json, bytes) {
1816
+ const entry = {
1817
+ id: this.nextId++,
1818
+ runId: record.runId,
1819
+ sequence: record.sequence,
1820
+ capturedAt: record.capturedAt,
1821
+ capturedAtMs: Date.parse(record.capturedAt),
1822
+ kind,
1823
+ gapReason: record.kind === "gap" ? record.payload.reason : null,
1824
+ state: "pending",
1825
+ offered: false,
1826
+ bytes,
1827
+ json
1828
+ };
1829
+ this.entries.push(entry);
1830
+ this.byRef.set(refKey(entry), entry);
1831
+ this.totalBytes += bytes;
1832
+ if (record.kind === "metrics") this.liveId = entry.id;
1833
+ return entry;
1834
+ }
1835
+ remove(entry) {
1836
+ this.byRef.delete(refKey(entry));
1837
+ const index = this.entries.indexOf(entry);
1838
+ if (index >= 0) this.entries.splice(index, 1);
1839
+ if (this.liveId === entry.id) this.liveId = null;
1840
+ this.totalBytes = Math.max(0, this.totalBytes - entry.bytes);
1841
+ }
1842
+ /** Fold evicted records into one coalesced gap so the hub sees the hole. */
1843
+ evict(victims, reason) {
1844
+ if (victims.length === 0) return 0;
1845
+ let from = victims[0].capturedAt;
1846
+ let to = victims[0].capturedAt;
1847
+ let fromMs = victims[0].capturedAtMs;
1848
+ let toMs = victims[0].capturedAtMs;
1849
+ for (const entry of victims) {
1850
+ this.remove(entry);
1851
+ if (entry.capturedAtMs < fromMs) {
1852
+ fromMs = entry.capturedAtMs;
1853
+ from = entry.capturedAt;
1854
+ }
1855
+ if (entry.capturedAtMs > toMs) {
1856
+ toMs = entry.capturedAtMs;
1857
+ to = entry.capturedAt;
1858
+ }
1859
+ }
1860
+ this.droppedSinceOpen += victims.length;
1861
+ this.writeGap(reason, from, to, victims.length);
1862
+ return victims.length;
1863
+ }
1864
+ /** Fold one dropped interval into a gap payload, keeping the widest range. */
1865
+ extendGapPayload(target, source) {
1866
+ if (Date.parse(source.fromCapturedAt) < Date.parse(target.fromCapturedAt)) {
1867
+ target.fromCapturedAt = source.fromCapturedAt;
1868
+ }
1869
+ if (Date.parse(source.toCapturedAt) > Date.parse(target.toCapturedAt)) {
1870
+ target.toCapturedAt = source.toCapturedAt;
1871
+ }
1872
+ target.droppedRecords += source.droppedRecords;
1873
+ }
1874
+ /**
1875
+ * Coalesce into the newest pending gap of the same reason that has not been
1876
+ * offered yet, so per-reason evidence stays one record while gaps queue.
1877
+ */
1878
+ writeGap(reason, fromCapturedAt, toCapturedAt, droppedRecords) {
1879
+ for (let index = this.entries.length - 1; index >= 0; index -= 1) {
1880
+ const candidate = this.entries[index];
1881
+ if (candidate.kind !== "gap" || candidate.state !== "pending" || candidate.offered) continue;
1882
+ if (candidate.runId !== this.identity.runId || candidate.gapReason !== reason) continue;
1883
+ const record = this.parse(candidate);
1884
+ if (record === null) break;
1885
+ this.extendGapPayload(record.payload, { fromCapturedAt, toCapturedAt, droppedRecords });
1886
+ candidate.capturedAt = record.capturedAt;
1887
+ candidate.capturedAtMs = Date.parse(record.capturedAt);
1888
+ const json2 = JSON.stringify(record);
1889
+ const bytes = Buffer.byteLength(json2, "utf8");
1890
+ this.totalBytes += bytes - candidate.bytes;
1891
+ candidate.json = json2;
1892
+ candidate.bytes = bytes;
1893
+ return;
1894
+ }
1895
+ const gap = {
1896
+ collectorId: this.identity.collectorId,
1897
+ runId: this.identity.runId,
1898
+ bootId: this.identity.bootId,
1899
+ sequence: this.allocateSequence(),
1900
+ capturedAt: this.isoNow(),
1901
+ kind: "gap",
1902
+ payload: { fromCapturedAt, toCapturedAt, droppedRecords, reason }
1903
+ };
1904
+ const json = JSON.stringify(gap);
1905
+ this.insertJson("gap", gap, json, Buffer.byteLength(json, "utf8"));
1906
+ }
1907
+ pendingMetrics() {
1908
+ return this.entries.filter((entry) => entry.state === "pending" && entry.kind === "metrics");
1909
+ }
1910
+ enforceAge() {
1911
+ const cutoff = this.now() - this.limits.maxAgeMs;
1912
+ const expired = this.pendingMetrics().filter((entry) => entry.capturedAtMs < cutoff);
1913
+ if (expired.length > 0) this.evict(expired, "outbox-age");
1914
+ }
1915
+ /** Evict oldest pending metrics until the queue fits `targetBytes`. */
1916
+ enforceBytes(targetBytes) {
1917
+ if (this.totalBytes <= targetBytes) return;
1918
+ const pending = this.pendingMetrics();
1919
+ const victims = [];
1920
+ for (const entry of pending) {
1921
+ if (this.totalBytes - sumBytes(victims) <= targetBytes) break;
1922
+ victims.push(entry);
1923
+ }
1924
+ if (victims.length === 0 && pending.length > 0) victims.push(pending[0]);
1925
+ if (victims.length > 0) this.evict(victims, "outbox-bytes");
1926
+ }
1927
+ /** Settle the caps that can change after any admission or eviction. */
1928
+ enforceAdmission() {
1929
+ this.enforceBudget();
1930
+ this.enforceGapCap();
1931
+ }
1932
+ enforceBudget() {
1933
+ const budget2 = this.budget;
1934
+ if (budget2 === null) return;
1935
+ const tried = /* @__PURE__ */ new Set();
1936
+ let guard = 0;
1937
+ while (budget2.overBy() > 0 && guard < 1e4) {
1938
+ guard += 1;
1939
+ const target = budget2.largest(tried);
1940
+ if (target === null) break;
1941
+ if (target.evictForBudget("disk-budget") === 0) tried.add(target);
1942
+ }
1943
+ }
1944
+ /** Hard ceiling on retained gap records: merge the oldest pending gaps. */
1945
+ enforceGapCap() {
1946
+ let guard = 0;
1947
+ while (this.gapCount() > this.limits.maxGapRecords && guard < 1e4) {
1948
+ guard += 1;
1949
+ const gaps = this.entries.filter((entry) => entry.kind === "gap" && entry.state === "pending");
1950
+ if (gaps.length < 2) return;
1951
+ const older = gaps[0];
1952
+ const newer = gaps[1];
1953
+ const olderRecord = this.parse(older);
1954
+ const merged = this.parse(newer);
1955
+ if (olderRecord === null || merged === null) return;
1956
+ this.extendGapPayload(merged.payload, olderRecord.payload);
1957
+ merged.collectorId = this.identity.collectorId;
1958
+ merged.runId = this.identity.runId;
1959
+ merged.bootId = this.identity.bootId;
1960
+ merged.sequence = this.allocateSequence();
1961
+ merged.capturedAt = this.isoNow();
1962
+ this.remove(newer);
1963
+ this.remove(older);
1964
+ const json = JSON.stringify(merged);
1965
+ this.insertJson("gap", merged, json, Buffer.byteLength(json, "utf8"));
1966
+ }
1967
+ }
1968
+ /**
1969
+ * Read one entry as a record. Ours is always self-written, so a parse failure
1970
+ * means an internal bug: count it as dropped and forget the row rather than
1971
+ * stalling every later record, matching the durable queue's corrupt-row path.
1972
+ */
1973
+ parse(entry) {
1974
+ try {
1975
+ return JSON.parse(entry.json);
1976
+ } catch {
1977
+ this.remove(entry);
1978
+ this.droppedSinceOpen += 1;
1979
+ return null;
1980
+ }
1981
+ }
1982
+ gapCount() {
1983
+ return this.entries.reduce((count, entry) => entry.kind === "gap" ? count + 1 : count, 0);
1984
+ }
1985
+ };
1986
+ function sumBytes(entries2) {
1987
+ return entries2.reduce((total, entry) => total + entry.bytes, 0);
1988
+ }
1989
+
1549
1990
  // src/contracts/telemetry.ts
1550
1991
  var MAX_MESSAGE_BYTES = 262144;
1551
1992
  var MAX_DETAIL_BYTES = 1048576;
1552
1993
  var MAX_BATCH_RECORDS = 32;
1994
+ var MAX_IN_FLIGHT_BATCHES = 64;
1995
+ var MAX_PERSISTENT_FLUSH_MS = 5e3;
1553
1996
  var MAX_CORES = 1024;
1554
1997
  var MAX_GPUS = 16;
1555
1998
  var MAX_FILESYSTEMS = 64;
@@ -1572,8 +2015,16 @@ function parseHubMessage(value) {
1572
2015
  switch (o.type) {
1573
2016
  case "welcome": {
1574
2017
  keys(o, ["type", "hubRunId", "machineId", "delivery", "maxInFlightBatches"], "welcome");
1575
- if (o.delivery !== "durable") throw new Error(`invalid welcome: unexpected delivery ${desc(o.delivery)}`);
1576
- return { type: "welcome", hubRunId: id(o.hubRunId, "hubRunId"), machineId: id(o.machineId, "machineId"), delivery: "durable", maxInFlightBatches: int(o.maxInFlightBatches, "maxInFlightBatches", 1, 64) };
2018
+ if (o.delivery !== "durable" && o.delivery !== "memory") {
2019
+ throw new Error(`invalid welcome: unexpected delivery ${desc(o.delivery)}`);
2020
+ }
2021
+ return {
2022
+ type: "welcome",
2023
+ hubRunId: id(o.hubRunId, "hubRunId"),
2024
+ machineId: id(o.machineId, "machineId"),
2025
+ delivery: o.delivery,
2026
+ maxInFlightBatches: int(o.maxInFlightBatches, "maxInFlightBatches", 1, MAX_IN_FLIGHT_BATCHES)
2027
+ };
1577
2028
  }
1578
2029
  case "ack": {
1579
2030
  keys(o, ["type", "accepted", "rejected"], "ack", ["retryAfterMs"]);
@@ -1673,7 +2124,7 @@ var MAX_SOCKET_INBOUND_BYTES = MAX_MESSAGE_BYTES;
1673
2124
  var MAX_UNACKED_SEND_BYTES = 256 * 1024;
1674
2125
  var DEFAULT_ACK_TIMEOUT_MS = 3e4;
1675
2126
  var DEFAULT_REPLAY_BATCHES_PER_SECOND = 4;
1676
- var DEFAULT_MAX_LOCAL_IN_FLIGHT_BATCHES = 8;
2127
+ var DEFAULT_MAX_LOCAL_IN_FLIGHT_BATCHES = 64;
1677
2128
  var DEFAULT_MAX_BATCH_BYTES = 2e5;
1678
2129
  var OUTBOX_ENFORCE_INTERVAL_MS = 1e4;
1679
2130
  var MAX_OUTBOUND_BUFFERED_BYTES = MAX_UNACKED_SEND_BYTES;
@@ -1692,7 +2143,7 @@ function inboundText(data) {
1692
2143
  }
1693
2144
  return "";
1694
2145
  }
1695
- function refKey(ref) {
2146
+ function refKey2(ref) {
1696
2147
  return `${ref.runId}\0${ref.sequence}`;
1697
2148
  }
1698
2149
  function toRef(record) {
@@ -2154,9 +2605,12 @@ var DestinationSession = class {
2154
2605
  // -------------------------------------------------------------------------
2155
2606
  // Delivery
2156
2607
  // -------------------------------------------------------------------------
2157
- /** Window slots reserved for live delivery (one when the window is > 1). */
2608
+ /** Reserve a flush window of live samples so replay cannot starve fresh delivery. */
2158
2609
  liveReservation() {
2159
- return this.inFlightWindow > 1 ? 1 : 0;
2610
+ if (this.inFlightWindow <= 1) return 0;
2611
+ if (this.inFlightWindow < MAX_IN_FLIGHT_BATCHES) return 1;
2612
+ const liveSlots = Math.ceil(MAX_PERSISTENT_FLUSH_MS / this.expectedIntervalMs) + 2;
2613
+ return Math.min(this.inFlightWindow - 1, liveSlots);
2160
2614
  }
2161
2615
  replayCapacity() {
2162
2616
  return Math.max(0, this.inFlightWindow - this.liveReservation());
@@ -2232,7 +2686,7 @@ var DestinationSession = class {
2232
2686
  const id2 = this.nextBatchId++;
2233
2687
  const entry = {
2234
2688
  refs,
2235
- keys: new Set(refs.map(refKey)),
2689
+ keys: new Set(refs.map(refKey2)),
2236
2690
  mode,
2237
2691
  timer: setTimeout(() => this.handleAckTimeout(id2), this.ackTimeoutMs)
2238
2692
  };
@@ -2258,6 +2712,7 @@ var DestinationSession = class {
2258
2712
  this.inFlight.delete(batchId);
2259
2713
  this.outbox.release(batch.refs);
2260
2714
  this.log(`ack timed out for ${batch.refs.length} record(s); will resend`);
2715
+ this.sendLiveBatch();
2261
2716
  this.pumpReplay();
2262
2717
  }
2263
2718
  handleAck(message) {
@@ -2269,7 +2724,7 @@ var DestinationSession = class {
2269
2724
  `hub rejected ${message.rejected.length} record(s): ${message.rejected[0]?.reason ?? "no reason given"}`
2270
2725
  );
2271
2726
  }
2272
- const ackedKeys = new Set([...accepted, ...rejected].map(refKey));
2727
+ const ackedKeys = new Set([...accepted, ...rejected].map(refKey2));
2273
2728
  const touched = [];
2274
2729
  for (const [id2, batch] of [...this.inFlight.entries()]) {
2275
2730
  let intersects = false;
@@ -2289,10 +2744,11 @@ var DestinationSession = class {
2289
2744
  for (const batch of toRelease) {
2290
2745
  this.inFlight.delete(this.batchIdOf(batch));
2291
2746
  clearTimeout(batch.timer);
2292
- this.outbox.release(batch.refs.filter((ref) => batch.keys.has(refKey(ref))));
2747
+ this.outbox.release(batch.refs.filter((ref) => batch.keys.has(refKey2(ref))));
2293
2748
  }
2294
2749
  this.log(`hub asked to retry after ${delay}ms; replay paused`);
2295
2750
  }
2751
+ this.sendLiveBatch();
2296
2752
  this.pumpReplay();
2297
2753
  }
2298
2754
  batchIdOf(target) {
@@ -2338,6 +2794,13 @@ var DestinationSession = class {
2338
2794
  };
2339
2795
 
2340
2796
  // collector/destinations/connection.ts
2797
+ function leftoverDurableQueue(dir) {
2798
+ try {
2799
+ return fs2.existsSync(path2.join(dir, "outbox.sqlite"));
2800
+ } catch {
2801
+ return false;
2802
+ }
2803
+ }
2341
2804
  var HubConnection = class _HubConnection {
2342
2805
  index;
2343
2806
  policy;
@@ -2379,22 +2842,29 @@ var HubConnection = class _HubConnection {
2379
2842
  });
2380
2843
  }
2381
2844
  /**
2382
- * Opens this connection's durable outbox under its canonical endpoint URL
2383
- * and re-applies the privacy policy to recovered pending records so none of
2384
- * them can leave with forbidden fields. Sanitation never cadence-filters:
2385
- * retained samples were already cadence-selected when they were captured.
2845
+ * Opens this connection's delivery queue in the configured mode and re-applies
2846
+ * the privacy policy to any recovered pending record so none of them can
2847
+ * leave with forbidden fields. Sanitation never cadence-filters: retained
2848
+ * samples were already cadence-selected when they were captured.
2849
+ *
2850
+ * In `memory` mode no queue file is created at all. A durable queue left over
2851
+ * from an earlier `durable` install is not opened or replayed; it is reported
2852
+ * once so the operator knows unacknowledged samples are sitting on disk and
2853
+ * how to discard them.
2386
2854
  */
2387
2855
  static open(options) {
2388
- const outbox = DestinationOutbox.open({
2389
- dir: outboxDirFor(options.stateDir, options.endpoint.url),
2390
- budget: options.budget,
2391
- allocateSequence: options.allocateSequence,
2392
- identity: {
2393
- collectorId: options.identity.collectorId,
2394
- runId: options.identity.runId,
2395
- bootId: options.identity.bootId
2396
- }
2397
- });
2856
+ const identity = {
2857
+ collectorId: options.identity.collectorId,
2858
+ runId: options.identity.runId,
2859
+ bootId: options.identity.bootId
2860
+ };
2861
+ const dir = outboxDirFor(options.stateDir, options.endpoint.url);
2862
+ const outbox = options.queueMode === "durable" ? DestinationOutbox.open({ dir, budget: options.budget, allocateSequence: options.allocateSequence, identity }) : new MemoryDestinationQueue({ budget: options.budget, allocateSequence: options.allocateSequence, identity });
2863
+ if (options.queueMode === "memory" && leftoverDurableQueue(dir)) {
2864
+ options.onEvent?.(
2865
+ `hub${options.index}: a durable outbox from an earlier COLLECTOR_QUEUE_MODE=durable install is unused; run rigscope upgrade, or delete it, to discard those unacknowledged samples`
2866
+ );
2867
+ }
2398
2868
  const connection = new _HubConnection(options, outbox);
2399
2869
  try {
2400
2870
  outbox.filterPending((record) => applySharing(record, connection.policy));
@@ -2508,9 +2978,9 @@ var nodeFs2 = __toESM(require("node:fs"));
2508
2978
  var nodeOs = __toESM(require("node:os"));
2509
2979
 
2510
2980
  // collector/sampling/procfs.ts
2511
- function readTextOrNull(read, path2) {
2981
+ function readTextOrNull(read, path3) {
2512
2982
  try {
2513
- return read(path2);
2983
+ return read(path3);
2514
2984
  } catch {
2515
2985
  return null;
2516
2986
  }
@@ -2736,8 +3206,8 @@ var CPU_HWMON_NAMES = /* @__PURE__ */ new Set([
2736
3206
  "cpu_thermal",
2737
3207
  "soc_thermal"
2738
3208
  ]);
2739
- function readCounter(read, path2) {
2740
- const text2 = readTextOrNull(read, path2);
3209
+ function readCounter(read, path3) {
3210
+ const text2 = readTextOrNull(read, path3);
2741
3211
  if (text2 === null || text2.trim() === "") return null;
2742
3212
  const value = Number(text2.trim());
2743
3213
  if (!Number.isSafeInteger(value) || value < 0) return null;
@@ -2758,8 +3228,8 @@ function hwmonOrder(name) {
2758
3228
  const match = /^hwmon(\d+)$/.exec(name);
2759
3229
  return match === null ? Number.MAX_SAFE_INTEGER : Number(match[1]);
2760
3230
  }
2761
- function readCandidateTemperature(read, path2) {
2762
- const text2 = readTextOrNull(read, path2);
3231
+ function readCandidateTemperature(read, path3) {
3232
+ const text2 = readTextOrNull(read, path3);
2763
3233
  if (text2 === null || text2.trim() === "") return null;
2764
3234
  const millidegrees = Number(text2.trim());
2765
3235
  if (!Number.isFinite(millidegrees)) return null;
@@ -2890,21 +3360,21 @@ function parsePressure(text2) {
2890
3360
  }
2891
3361
  return sawLine ? { someAvg10, fullAvg10 } : null;
2892
3362
  }
2893
- function readPressure(read, path2) {
2894
- const text2 = readTextOrNull(read, path2);
3363
+ function readPressure(read, path3) {
3364
+ const text2 = readTextOrNull(read, path3);
2895
3365
  return text2 === null ? null : parsePressure(text2);
2896
3366
  }
2897
3367
 
2898
3368
  // collector/sampling/sysfs-gpu.ts
2899
- function number(read, path2, max = Number.MAX_SAFE_INTEGER, min = 0) {
2900
- const raw = readTextOrNull(read, path2)?.trim();
3369
+ function number(read, path3, max = Number.MAX_SAFE_INTEGER, min = 0) {
3370
+ const raw = readTextOrNull(read, path3)?.trim();
2901
3371
  if (!raw || !/^[-]?\d+$/.test(raw)) return null;
2902
3372
  const value = Number(raw);
2903
3373
  return Number.isSafeInteger(value) && value >= min && value <= max ? value : null;
2904
3374
  }
2905
- function entries(readers, path2) {
3375
+ function entries(readers, path3) {
2906
3376
  try {
2907
- return readers.readdir(path2).filter((name) => /^hwmon\d+$/.test(name)).sort().slice(0, 8);
3377
+ return readers.readdir(path3).filter((name) => /^hwmon\d+$/.test(name)).sort().slice(0, 8);
2908
3378
  } catch {
2909
3379
  return [];
2910
3380
  }
@@ -2927,9 +3397,9 @@ function readSysfsGpu(readers, address) {
2927
3397
  if (values.memoryTotalBytes === 0) values.memoryTotalBytes = null;
2928
3398
  if (values.memoryTotalBytes === null || values.memoryUsedBytes !== null && values.memoryUsedBytes > values.memoryTotalBytes) values.memoryUsedBytes = null;
2929
3399
  for (const hwmon of entries(readers, `${base}/hwmon`)) {
2930
- const path2 = `${base}/hwmon/${hwmon}`;
2931
- const temp = number(read, `${path2}/temp1_input`, 2e5, -5e4);
2932
- const power = number(read, `${path2}/power1_average`, 1e10) ?? number(read, `${path2}/power1_input`, 1e10);
3400
+ const path3 = `${base}/hwmon/${hwmon}`;
3401
+ const temp = number(read, `${path3}/temp1_input`, 2e5, -5e4);
3402
+ const power = number(read, `${path3}/power1_average`, 1e10) ?? number(read, `${path3}/power1_input`, 1e10);
2933
3403
  if (temp !== null && values.temperatureC === null) values.temperatureC = temp / 1e3;
2934
3404
  if (power !== null && values.powerWatts === null) values.powerWatts = power / 1e6;
2935
3405
  }
@@ -2940,9 +3410,9 @@ function readSysfsGpu(readers, address) {
2940
3410
  }
2941
3411
 
2942
3412
  // collector/sampling/network.ts
2943
- function readOptional(read, path2) {
3413
+ function readOptional(read, path3) {
2944
3414
  try {
2945
- return read(path2);
3415
+ return read(path3);
2946
3416
  } catch {
2947
3417
  return null;
2948
3418
  }
@@ -3461,10 +3931,10 @@ var USER_HZ = 100;
3461
3931
  var CPU_TOLERANCE_FRACTION = 0.05;
3462
3932
  var MAX_RATE_BYTES_PER_SEC = 1e13;
3463
3933
  var defaultReaders = {
3464
- readFile: (path2) => String(nodeFs2.readFileSync(path2, "utf8")),
3465
- readdir: (path2) => nodeFs2.readdirSync(path2),
3466
- statfs: (path2) => {
3467
- const stats = nodeFs2.statfsSync(path2);
3934
+ readFile: (path3) => String(nodeFs2.readFileSync(path3, "utf8")),
3935
+ readdir: (path3) => nodeFs2.readdirSync(path3),
3936
+ statfs: (path3) => {
3937
+ const stats = nodeFs2.statfsSync(path3);
3468
3938
  return {
3469
3939
  bsize: Number(stats.bsize),
3470
3940
  blocks: Number(stats.blocks),
@@ -3512,7 +3982,7 @@ function meanLatencyMs(operations, timeMs) {
3512
3982
  function inventorySignature(inventory) {
3513
3983
  return JSON.stringify({
3514
3984
  gpus: inventory.gpus.map((gpu) => [gpu.id, gpu.vendor, gpu.telemetry, gpu.memoryTotalBytes, gpu.pciBusId]),
3515
- filesystems: inventory.filesystems.map((fs2) => [fs2.id, fs2.totalBytes]),
3985
+ filesystems: inventory.filesystems.map((fs3) => [fs3.id, fs3.totalBytes]),
3516
3986
  disks: inventory.disks.map((disk) => [disk.id, disk.sizeBytes]),
3517
3987
  network: inventory.network.map((iface) => [
3518
3988
  iface.id,
@@ -4679,9 +5149,9 @@ var ProcessScanner = class {
4679
5149
  }
4680
5150
  };
4681
5151
  }
4682
- async readText(path2) {
5152
+ async readText(path3) {
4683
5153
  try {
4684
- return { ok: true, text: await import_node_fs3.promises.readFile(path2, "utf8") };
5154
+ return { ok: true, text: await import_node_fs3.promises.readFile(path3, "utf8") };
4685
5155
  } catch (error) {
4686
5156
  return { ok: false, code: errorCode(error) };
4687
5157
  }
@@ -4766,8 +5236,8 @@ function parseIo(text2) {
4766
5236
  function parseCgroupUnit(text2) {
4767
5237
  for (const line of text2.split("\n")) {
4768
5238
  if (!line.startsWith("0::")) continue;
4769
- const path2 = line.slice(3);
4770
- const components = path2.split("/").filter((part) => part.length > 0);
5239
+ const path3 = line.slice(3);
5240
+ const components = path3.split("/").filter((part) => part.length > 0);
4771
5241
  const last = components[components.length - 1];
4772
5242
  if (!last) return null;
4773
5243
  if (!last.endsWith(".service") && !last.endsWith(".scope")) return null;
@@ -5217,7 +5687,7 @@ function budget(value, fallback) {
5217
5687
  // package.json
5218
5688
  var package_default = {
5219
5689
  name: "rigscope",
5220
- version: "0.1.8",
5690
+ version: "0.2.1",
5221
5691
  private: false,
5222
5692
  engines: {
5223
5693
  node: ">=26.0.0"
@@ -5239,6 +5709,7 @@ var package_default = {
5239
5709
  test: "node scripts/test.cjs",
5240
5710
  "test:runtime": "npm run build:runtime && node scripts/test-runtime.cjs",
5241
5711
  "test:dev": "npm run build:runtime && node scripts/test.cjs --suite development",
5712
+ "benchmark:telemetry": "tsx scripts/telemetry-benchmark.ts",
5242
5713
  "validate:https": "node scripts/validate-https.cjs",
5243
5714
  "build:ui": "next build --webpack",
5244
5715
  rigscope: "node bin/rigscope.cjs",
@@ -5287,6 +5758,7 @@ var package_default = {
5287
5758
  "docs/system-installation.md",
5288
5759
  "docs/configuration.md",
5289
5760
  "docs/monitoring.md",
5761
+ "docs/telemetry-storage.md",
5290
5762
  "docs/operations.md",
5291
5763
  "docs/bottlenecks.md",
5292
5764
  "docs/exports.md",
@@ -5354,8 +5826,8 @@ var OperationsError = class extends Error {
5354
5826
  function isPrivateMode(mode) {
5355
5827
  return (mode & 63) === 0;
5356
5828
  }
5357
- function ensurePrivateDir(path2) {
5358
- const absolute = (0, import_node_path3.resolve)(path2);
5829
+ function ensurePrivateDir(path3) {
5830
+ const absolute = (0, import_node_path3.resolve)(path3);
5359
5831
  (0, import_node_fs4.mkdirSync)(absolute, { recursive: true, mode: PRIVATE_DIR_MODE });
5360
5832
  const stat = (0, import_node_fs4.lstatSync)(absolute);
5361
5833
  if (!stat.isDirectory() || stat.isSymbolicLink()) {
@@ -5368,23 +5840,48 @@ function ensurePrivateDir(path2) {
5368
5840
  );
5369
5841
  }
5370
5842
  }
5843
+ function assertPrivateRegularFile(path3, label) {
5844
+ const absolute = (0, import_node_path3.resolve)(path3);
5845
+ let stat;
5846
+ try {
5847
+ stat = (0, import_node_fs4.lstatSync)(absolute);
5848
+ } catch (error) {
5849
+ throw new OperationsError("missing-file", `${label} does not exist: ${absolute}`, { cause: error });
5850
+ }
5851
+ if (!stat.isFile() || stat.isSymbolicLink()) {
5852
+ throw new OperationsError("unsafe-path", `${label} must be a regular file, not a symlink: ${absolute}`);
5853
+ }
5854
+ if (!isPrivateMode(stat.mode)) {
5855
+ throw new OperationsError(
5856
+ "insecure-permissions",
5857
+ `${label} must be private (0600/0400); found ${formatMode(stat.mode)}: ${absolute}`
5858
+ );
5859
+ }
5860
+ return stat;
5861
+ }
5371
5862
  function formatMode(mode) {
5372
5863
  return `0${(mode & 511).toString(8).padStart(3, "0")}`;
5373
5864
  }
5374
- function writeFileAtomic(path2, content, mode) {
5375
- const absolute = (0, import_node_path3.resolve)(path2);
5865
+ function writePrivateFileBuffered(path3, content) {
5866
+ const absolute = (0, import_node_path3.resolve)(path3);
5376
5867
  const parent = (0, import_node_path3.dirname)(absolute);
5868
+ ensurePrivateDir(parent);
5869
+ try {
5870
+ assertPrivateRegularFile(absolute, "private file");
5871
+ } catch (error) {
5872
+ if (!(error instanceof OperationsError) || error.code !== "missing-file") throw error;
5873
+ }
5377
5874
  const temporary = (0, import_node_path3.join)(parent, `.${(0, import_node_path3.basename)(absolute)}.tmp-${(0, import_node_crypto6.randomUUID)()}`);
5378
5875
  let fd = null;
5876
+ let temporaryCreated = false;
5379
5877
  try {
5380
- fd = (0, import_node_fs4.openSync)(temporary, "wx", mode);
5878
+ fd = (0, import_node_fs4.openSync)(temporary, "wx", PRIVATE_FILE_MODE);
5879
+ temporaryCreated = true;
5381
5880
  (0, import_node_fs4.writeFileSync)(fd, content);
5382
- (0, import_node_fs4.fsyncSync)(fd);
5383
5881
  (0, import_node_fs4.closeSync)(fd);
5384
5882
  fd = null;
5385
- (0, import_node_fs4.chmodSync)(temporary, mode);
5883
+ (0, import_node_fs4.chmodSync)(temporary, PRIVATE_FILE_MODE);
5386
5884
  (0, import_node_fs4.renameSync)(temporary, absolute);
5387
- fsyncDir(parent);
5388
5885
  } catch (error) {
5389
5886
  if (fd !== null) {
5390
5887
  try {
@@ -5392,39 +5889,24 @@ function writeFileAtomic(path2, content, mode) {
5392
5889
  } catch {
5393
5890
  }
5394
5891
  }
5395
- removeIfExists(temporary);
5396
- throw error;
5397
- }
5398
- }
5399
- function writePrivateFile(path2, content) {
5400
- ensurePrivateDir((0, import_node_path3.dirname)((0, import_node_path3.resolve)(path2)));
5401
- writeFileAtomic(path2, content, PRIVATE_FILE_MODE);
5402
- }
5403
- function fsyncDir(path2) {
5404
- let fd = null;
5405
- try {
5406
- fd = (0, import_node_fs4.openSync)(path2, "r");
5407
- (0, import_node_fs4.fsyncSync)(fd);
5408
- } catch {
5409
- } finally {
5410
- if (fd !== null) {
5411
- try {
5412
- (0, import_node_fs4.closeSync)(fd);
5413
- } catch {
5414
- }
5892
+ if (temporaryCreated) removeIfExists(temporary);
5893
+ if (error.code === "ENOENT") {
5894
+ throw new OperationsError("missing-file", `directory does not exist: ${parent}`, { cause: error });
5415
5895
  }
5896
+ throw error;
5416
5897
  }
5417
5898
  }
5418
- function removeIfExists(path2) {
5899
+ function removeIfExists(path3) {
5419
5900
  try {
5420
- (0, import_node_fs4.unlinkSync)(path2);
5901
+ (0, import_node_fs4.unlinkSync)(path3);
5421
5902
  } catch (error) {
5422
5903
  if (error.code !== "ENOENT") throw error;
5423
5904
  }
5424
5905
  }
5425
5906
 
5426
5907
  // collector/runtime.ts
5427
- async function runDurableCollector(config, identity, statePath, mock) {
5908
+ var HEALTH_WRITE_INTERVAL_MS = 5e3;
5909
+ async function runCollectorLoop(config, identity, statePath, mock) {
5428
5910
  const temporary = mock && !process.env.COLLECTOR_STATE_FILE ? (0, import_node_fs5.mkdtempSync)((0, import_node_path4.join)((0, import_node_os2.tmpdir)(), "rigscope-mock-")) : null;
5429
5911
  const stateDir = temporary ?? (0, import_node_path4.dirname)(statePath);
5430
5912
  const releaseLease = acquireProcessLease((0, import_node_path4.join)(stateDir, "collector-runtime.lock"));
@@ -5441,6 +5923,7 @@ async function runDurableCollector(config, identity, statePath, mock) {
5441
5923
  stateDir,
5442
5924
  baseIntervalMs: config.intervalMs,
5443
5925
  identity,
5926
+ queueMode: config.queueMode,
5444
5927
  budget: budget2,
5445
5928
  allocateSequence: () => sequence.next(),
5446
5929
  sampler,
@@ -5475,8 +5958,8 @@ async function runDurableCollector(config, identity, statePath, mock) {
5475
5958
  const at = performance.now();
5476
5959
  for (const connection of connections) connection.acceptSample(record, at);
5477
5960
  const healthFile = process.env.COLLECTOR_HEALTH_FILE;
5478
- if (healthFile && Date.now() - lastHealthWrite >= 1e3) {
5479
- writePrivateFile(healthFile, JSON.stringify({
5961
+ if (healthFile && Date.now() - lastHealthWrite >= HEALTH_WRITE_INTERVAL_MS) {
5962
+ writePrivateFileBuffered(healthFile, JSON.stringify({
5480
5963
  version: APPLICATION_VERSION,
5481
5964
  pid: process.pid,
5482
5965
  sampledAt: Date.now()
@@ -5503,7 +5986,7 @@ async function runDurableCollector(config, identity, statePath, mock) {
5503
5986
  for (const connection of connections) connection.stop();
5504
5987
  releaseLease();
5505
5988
  if (temporary) (0, import_node_fs5.rmSync)(temporary, { recursive: true, force: true });
5506
- console.log("collector stopping; durable pending delivery saved");
5989
+ console.log(config.queueMode === "durable" ? "collector stopping; durable pending delivery saved" : "collector stopping; unacknowledged samples in RAM are discarded");
5507
5990
  setTimeout(() => process.exit(0), 250).unref();
5508
5991
  setTimeout(() => process.exit(0), 3e3).unref();
5509
5992
  };
@@ -5582,7 +6065,7 @@ async function run() {
5582
6065
  runId,
5583
6066
  bootId: boot.bootId
5584
6067
  };
5585
- await runDurableCollector(config, identity, process.env.COLLECTOR_STATE_FILE || defaultStatePath(), isMock);
6068
+ await runCollectorLoop(config, identity, process.env.COLLECTOR_STATE_FILE || defaultStatePath(), isMock);
5586
6069
  return 0;
5587
6070
  }
5588
6071
  function isMainModule() {