rigscope 0.1.7 → 0.2.0

package/dist/cli.cjs

@@ -661,6 +661,13 @@ function verifyPassword(password, hash2) {
 // src/hub/config.ts
 var import_node_net = require("node:net");
 var import_node_path3 = require("node:path");
+
+// src/contracts/telemetry.ts
+var MAX_PERSISTENT_FLUSH_MS = 5e3;
+var MIN_EPOCH = Date.parse("2000-01-01T00:00:00Z");
+var MAX_EPOCH = Date.parse("2200-01-01T00:00:00Z");
+
+// src/hub/config.ts
 var MAX_HUB_COLLECTORS_BYTES = 1024 * 1024;
 var MAX_COLLECTOR_ID_LENGTH = 128;
 var MAX_MACHINE_ID_LENGTH = 128;
@@ -688,6 +695,7 @@ function parseHubConfig(env) {
     maxCollectorConnections = parsed;
   }
   const connectionBudget = maxCollectorConnections === void 0 ? {} : { maxCollectorConnections };
+  const history = parseHistoryConfig(env);
   if (publicOrigin === void 0 !== (socketPath === void 0)) {
     throw new HubConfigError(
       "HUB_PUBLIC_ORIGIN and HUB_SOCKET_PATH must be set together: remote mode needs both the exact HTTPS origin and the Unix socket path"
@@ -707,6 +715,7 @@ function parseHubConfig(env) {
       collectors,
       development,
       remote: parseRemoteConfig(publicOrigin, socketPath),
+      history,
       ...connectionBudget
     };
   }
@@ -716,9 +725,65 @@ function parseHubConfig(env) {
     collectors,
     development,
     remote: null,
+    history,
     ...connectionBudget
   };
 }
+var DEFAULT_MEMORY_BUDGET_BYTES = 128 * 1024 * 1024;
+var DEFAULT_PERSISTENT_FLUSH_MS = MAX_PERSISTENT_FLUSH_MS;
+var DEFAULT_RECORDING_MAX_DURATION_MS = 60 * 6e4;
+var DEFAULT_RECORDING_MAX_BYTES = 64 * 1024 * 1024;
+var DEFAULT_RECORDING_BACKLOG_MS = 10 * 6e4;
+var DEFAULT_MAX_ACTIVE_RECORDINGS = 4;
+var HOUR_MS = 36e5;
+var DAY_MS = 864e5;
+function parseHistoryConfig(env) {
+  const modeValue = env.HUB_TELEMETRY_MODE ?? "memory";
+  if (modeValue !== "memory" && modeValue !== "persistent") {
+    throw new HubConfigError("HUB_TELEMETRY_MODE must be memory or persistent");
+  }
+  const memoryBudgetBytes = envInteger(
+    env.HUB_HISTORY_MEMORY_BYTES,
+    DEFAULT_MEMORY_BUDGET_BYTES,
+    1 * 1024 * 1024,
+    16 * 1024 * 1024 * 1024,
+    "HUB_HISTORY_MEMORY_BYTES"
+  );
+  const flushIntervalMs = envInteger(
+    env.HUB_TELEMETRY_FLUSH_MS,
+    DEFAULT_PERSISTENT_FLUSH_MS,
+    1e3,
+    MAX_PERSISTENT_FLUSH_MS,
+    "HUB_TELEMETRY_FLUSH_MS"
+  );
+  const retention = {
+    rawMs: envInteger(env.HUB_RETENTION_RAW_MS, 2 * DAY_MS, 0, 400 * DAY_MS, "HUB_RETENTION_RAW_MS"),
+    tenSecondMs: envInteger(env.HUB_RETENTION_10S_MS, 2 * HOUR_MS, 0, 30 * DAY_MS, "HUB_RETENTION_10S_MS"),
+    oneMinuteMs: envInteger(env.HUB_RETENTION_1M_MS, 30 * DAY_MS, 0, 400 * DAY_MS, "HUB_RETENTION_1M_MS"),
+    oneHourMs: envInteger(env.HUB_RETENTION_1H_MS, 400 * DAY_MS, 0, 800 * DAY_MS, "HUB_RETENTION_1H_MS"),
+    receiptsMs: envInteger(env.HUB_RECEIPT_RETENTION_MS, 7 * DAY_MS, HOUR_MS, 400 * DAY_MS, "HUB_RECEIPT_RETENTION_MS"),
+    gapsMs: envInteger(env.HUB_GAP_RETENTION_MS, 400 * DAY_MS, 0, 800 * DAY_MS, "HUB_GAP_RETENTION_MS")
+  };
+  const recordings = {
+    directory: nonEmpty(env.HUB_RECORDINGS_DIR) ?? null,
+    maxDurationMs: envInteger(env.HUB_RECORDING_MAX_DURATION_MS, DEFAULT_RECORDING_MAX_DURATION_MS, 1e3, 24 * 60 * 6e4, "HUB_RECORDING_MAX_DURATION_MS"),
+    maxBytes: envInteger(env.HUB_RECORDING_MAX_BYTES, DEFAULT_RECORDING_MAX_BYTES, 64 * 1024, 1024 * 1024 * 1024, "HUB_RECORDING_MAX_BYTES"),
+    backlogMs: envInteger(env.HUB_RECORDING_BACKLOG_MS, DEFAULT_RECORDING_BACKLOG_MS, 0, 24 * 60 * 6e4, "HUB_RECORDING_BACKLOG_MS"),
+    maxActive: envInteger(env.HUB_RECORDING_MAX_ACTIVE, DEFAULT_MAX_ACTIVE_RECORDINGS, 1, 32, "HUB_RECORDING_MAX_ACTIVE")
+  };
+  return { mode: modeValue, memoryBudgetBytes, flushIntervalMs, retention, recordings };
+}
+function envInteger(raw, fallback, min, max, name) {
+  if (raw === void 0 || raw === "") return fallback;
+  if (!/^\d+$/.test(raw)) {
+    throw new HubConfigError(`${name} must be a nonnegative integer`);
+  }
+  const value = Number(raw);
+  if (!Number.isSafeInteger(value) || value < min || value > max) {
+    throw new HubConfigError(`${name} must be between ${min} and ${max}`);
+  }
+  return value;
+}
 function nonEmpty(value) {
   return value === void 0 || value === "" ? void 0 : value;
 }
@@ -1446,8 +1511,45 @@ var MIGRATIONS = [
   },
   { version: 3, name: "admin", up: adminSchema },
   { version: 4, name: "alerts", up: installAlertSchema },
-  { version: 5, name: "collector_registry", up: installRegistrySchema }
+  { version: 5, name: "collector_registry", up: installRegistrySchema },
+  { version: 6, name: "telemetry_modes_and_recordings", up: telemetryModesAndRecordings }
 ];
+function telemetryModesAndRecordings(db) {
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS points_10s (
+      series_id INTEGER NOT NULL,
+      bucket_ms INTEGER NOT NULL,
+      min REAL,
+      max REAL,
+      sum REAL NOT NULL DEFAULT 0,
+      count INTEGER NOT NULL DEFAULT 0,
+      null_count INTEGER NOT NULL DEFAULT 0,
+      PRIMARY KEY (series_id, bucket_ms)
+    ) WITHOUT ROWID;
+
+    CREATE TABLE IF NOT EXISTS recordings (
+      id TEXT PRIMARY KEY,
+      created_by TEXT NOT NULL,
+      status TEXT NOT NULL,
+      machines_json TEXT NOT NULL,
+      metrics_json TEXT NOT NULL,
+      resolution_ms INTEGER NOT NULL,
+      created_at_ms INTEGER NOT NULL,
+      started_at_ms INTEGER NOT NULL,
+      stopped_at_ms INTEGER,
+      deadline_ms INTEGER NOT NULL,
+      max_bytes INTEGER NOT NULL,
+      bytes_written INTEGER NOT NULL DEFAULT 0,
+      file_name TEXT NOT NULL,
+      include_backlog INTEGER NOT NULL DEFAULT 0,
+      reason TEXT,
+      error TEXT
+    );
+
+    CREATE INDEX IF NOT EXISTS recordings_created
+      ON recordings (created_at_ms DESC);
+  `);
+}
 
 // src/hub/storage/db.ts
 var BUSY_TIMEOUT_MS = 5e3;
@@ -1524,7 +1626,7 @@ var SqliteHubDatabase = class {
     const walBudgetBytes = Math.max(64 * 1024, Math.min(HUB_WAL_RESERVE_BYTES, Math.floor(limitBytes / 16)));
     const mainBudgetBytes = Math.max(pageSize, limitBytes - walBudgetBytes);
     const requestedPages = Math.max(1, Math.floor(mainBudgetBytes / pageSize));
-    const checkpointPages = Math.max(16, Math.min(1e3, Math.floor(walBudgetBytes / pageSize)));
+    const checkpointPages = Math.max(16, Math.min(16384, Math.floor(walBudgetBytes / pageSize)));
     this.db.exec(`PRAGMA journal_size_limit = ${walBudgetBytes}`);
     this.db.exec(`PRAGMA wal_autocheckpoint = ${checkpointPages}`);
     let maxPageCount = requestedPages;
@@ -3555,7 +3657,9 @@ var COUNTED_TABLES = /* @__PURE__ */ new Set([
   "machine_meta",
   "machine_latest",
   "inventory",
+  "points_10s",
   "gaps",
+  "recordings",
   "alert_rules",
   "alert_incidents",
   "alert_jobs",

package/dist/collector.cjs

@@ -1550,6 +1550,8 @@ var DestinationOutbox = class _DestinationOutbox {
 var MAX_MESSAGE_BYTES = 262144;
 var MAX_DETAIL_BYTES = 1048576;
 var MAX_BATCH_RECORDS = 32;
+var MAX_IN_FLIGHT_BATCHES = 64;
+var MAX_PERSISTENT_FLUSH_MS = 5e3;
 var MAX_CORES = 1024;
 var MAX_GPUS = 16;
 var MAX_FILESYSTEMS = 64;
@@ -1572,8 +1574,16 @@ function parseHubMessage(value) {
   switch (o.type) {
     case "welcome": {
       keys(o, ["type", "hubRunId", "machineId", "delivery", "maxInFlightBatches"], "welcome");
-      if (o.delivery !== "durable") throw new Error(`invalid welcome: unexpected delivery ${desc(o.delivery)}`);
-      return { type: "welcome", hubRunId: id(o.hubRunId, "hubRunId"), machineId: id(o.machineId, "machineId"), delivery: "durable", maxInFlightBatches: int(o.maxInFlightBatches, "maxInFlightBatches", 1, 64) };
+      if (o.delivery !== "durable" && o.delivery !== "memory") {
+        throw new Error(`invalid welcome: unexpected delivery ${desc(o.delivery)}`);
+      }
+      return {
+        type: "welcome",
+        hubRunId: id(o.hubRunId, "hubRunId"),
+        machineId: id(o.machineId, "machineId"),
+        delivery: o.delivery,
+        maxInFlightBatches: int(o.maxInFlightBatches, "maxInFlightBatches", 1, MAX_IN_FLIGHT_BATCHES)
+      };
     }
     case "ack": {
       keys(o, ["type", "accepted", "rejected"], "ack", ["retryAfterMs"]);
@@ -1673,7 +1683,7 @@ var MAX_SOCKET_INBOUND_BYTES = MAX_MESSAGE_BYTES;
 var MAX_UNACKED_SEND_BYTES = 256 * 1024;
 var DEFAULT_ACK_TIMEOUT_MS = 3e4;
 var DEFAULT_REPLAY_BATCHES_PER_SECOND = 4;
-var DEFAULT_MAX_LOCAL_IN_FLIGHT_BATCHES = 8;
+var DEFAULT_MAX_LOCAL_IN_FLIGHT_BATCHES = 64;
 var DEFAULT_MAX_BATCH_BYTES = 2e5;
 var OUTBOX_ENFORCE_INTERVAL_MS = 1e4;
 var MAX_OUTBOUND_BUFFERED_BYTES = MAX_UNACKED_SEND_BYTES;
@@ -2154,9 +2164,12 @@ var DestinationSession = class {
   // -------------------------------------------------------------------------
   // Delivery
   // -------------------------------------------------------------------------
-  /** Window slots reserved for live delivery (one when the window is > 1). */
+  /** Reserve a flush window of live samples so replay cannot starve fresh delivery. */
   liveReservation() {
-    return this.inFlightWindow > 1 ? 1 : 0;
+    if (this.inFlightWindow <= 1) return 0;
+    if (this.inFlightWindow < MAX_IN_FLIGHT_BATCHES) return 1;
+    const liveSlots = Math.ceil(MAX_PERSISTENT_FLUSH_MS / this.expectedIntervalMs) + 2;
+    return Math.min(this.inFlightWindow - 1, liveSlots);
   }
   replayCapacity() {
     return Math.max(0, this.inFlightWindow - this.liveReservation());
@@ -2258,6 +2271,7 @@ var DestinationSession = class {
     this.inFlight.delete(batchId);
     this.outbox.release(batch.refs);
     this.log(`ack timed out for ${batch.refs.length} record(s); will resend`);
+    this.sendLiveBatch();
     this.pumpReplay();
   }
   handleAck(message) {
@@ -2293,6 +2307,7 @@ var DestinationSession = class {
       }
       this.log(`hub asked to retry after ${delay}ms; replay paused`);
     }
+    this.sendLiveBatch();
     this.pumpReplay();
   }
   batchIdOf(target) {
@@ -5217,7 +5232,7 @@ function budget(value, fallback) {
 // package.json
 var package_default = {
   name: "rigscope",
-  version: "0.1.7",
+  version: "0.2.0",
   private: false,
   engines: {
     node: ">=26.0.0"
@@ -5239,6 +5254,7 @@ var package_default = {
     test: "node scripts/test.cjs",
     "test:runtime": "npm run build:runtime && node scripts/test-runtime.cjs",
     "test:dev": "npm run build:runtime && node scripts/test.cjs --suite development",
+    "benchmark:telemetry": "tsx scripts/telemetry-benchmark.ts",
     "validate:https": "node scripts/validate-https.cjs",
     "build:ui": "next build --webpack",
     rigscope: "node bin/rigscope.cjs",
@@ -5287,6 +5303,7 @@ var package_default = {
     "docs/system-installation.md",
     "docs/configuration.md",
     "docs/monitoring.md",
+    "docs/telemetry-storage.md",
     "docs/operations.md",
     "docs/bottlenecks.md",
     "docs/exports.md",