peekinside 1.2.1 → 1.2.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "peekinside",
-  "version": "1.2.1",
+  "version": "1.2.3",
   "description": "Debug Node.js from the inside: errors with the requests, queries, logs and jobs behind them. A free local dashboard, and encrypted monitoring for production",
   "main": "index.js",
   "types": "./index.d.ts",

package/src/cli.js

@@ -70,6 +70,9 @@ export class Cli {
    * @throws {Error} When the hub did not start in time
    */
   async start(openBrowser) {
+    // A hub left running by an older PeekInside is replaced by this version's (its data stays).
+    const replaced = await this.hub.replaceIfOlder();
+    if (replaced) console.log(`Restarted PeekInside Local: it was version ${replaced}, now ${VERSION}.`);
     let running = await this.hub.status();
     if (!running) {
       this.hub.spawn();

package/src/hub/client.js

@@ -9,7 +9,7 @@ import fs from 'fs';
 import http from 'http';
 import path from 'path';
 import { fileURLToPath } from 'url';
-import { HUB_APP_ID, VERSION, settings } from '../config.js';
+import { HUB_API_VERSION, HUB_APP_ID, VERSION, settings } from '../config.js';
 
 const HUB_ENTRY = fileURLToPath(new URL('./entry.js', import.meta.url));
 
@@ -153,6 +153,50 @@ export class HubClient {
     }
   }
 
+  /**
+   * Replaces a running hub from an older PeekInside with this version's. Hubs outlive the apps
+   * that start them, so after an upgrade the old one would keep serving, its dashboard page
+   * included. Only a hub with the same API and data folder is replaced, so its data stays, and
+   * only the process hub.json names, so nothing else listening on the port is ever stopped.
+   *
+   * @returns {Promise<string|null>} The version that was replaced, or null when none was
+   */
+  async replaceIfOlder() {
+    const running = await this.status().catch(() => null);
+    const dataDir = settings.dataDir();
+    if (!HubClient.shouldReplace(running, { version: VERSION, dataDir, infoPid: readInfoPid(dataDir) })) return null;
+    try {
+      process.kill(running.pid, 'SIGTERM');
+    } catch (error) {
+      return null;
+    }
+    for (let attempt = 0; attempt < 50; attempt++) {
+      await new Promise((resolve) => setTimeout(resolve, 100));
+      if (!(await this.status().catch(() => null))) {
+        this.spawn();
+        await this.waitUntilReady(8000).catch(() => null);
+        return running.version;
+      }
+    }
+    return null;
+  }
+
+  /**
+   * Whether a running hub is an older version than this one that may be replaced: same API, same
+   * data folder, and the process its hub.json names.
+   *
+   * @param {Object|null} running - Its GET /api/status
+   * @param {{ version: string, dataDir: string, infoPid: number|null }} self - This version, its
+   *   data folder, and the pid in that folder's hub.json
+   * @returns {boolean}
+   */
+  static shouldReplace(running, { version, dataDir, infoPid }) {
+    if (!running || running.apiVersion !== HUB_API_VERSION) return false;
+    if (!Number.isInteger(running.pid) || running.pid !== infoPid || running.pid === process.pid) return false;
+    if (typeof running.dataDir !== 'string' || path.resolve(running.dataDir) !== path.resolve(dataDir)) return false;
+    return compareVersions(running.version, version) < 0;
+  }
+
   /**
    * Waits until the hub answers.
    *
@@ -182,3 +226,34 @@ export class HubClient {
     if (this.agent) this.agent.destroy();
   }
 }
+
+// The pid in the data folder's hub.json (written by the hub that runs), or null.
+function readInfoPid(dataDir) {
+  try {
+    const info = JSON.parse(fs.readFileSync(path.join(dataDir, 'hub.json'), 'utf8'));
+    return Number.isInteger(info.pid) ? info.pid : null;
+  } catch (error) {
+    return null;
+  }
+}
+
+/**
+ * Compares two versions by major, minor and patch ("1.10.0" is after "1.9.3"). Anything after the
+ * patch number (a prerelease) is ignored, and versions that can't be read compare as equal, so
+ * nothing is replaced on a guess.
+ *
+ * @param {string} a
+ * @param {string} b
+ * @returns {number} Negative when a is older, positive when newer, 0 otherwise
+ */
+export function compareVersions(a, b) {
+  const parse = (value) => /^(\d+)\.(\d+)\.(\d+)/.exec(String(value || ''));
+  const left = parse(a);
+  const right = parse(b);
+  if (!left || !right) return 0;
+  for (let index = 1; index <= 3; index++) {
+    const difference = Number(left[index]) - Number(right[index]);
+    if (difference) return difference;
+  }
+  return 0;
+}

package/src/spool.js

@@ -20,6 +20,10 @@ import path from 'path';
 export const CLOUD_FOLDER = 'cloud';
 // A temporary file this old was left by a writer that died mid-write.
 const STALE_TEMP_MS = 60_000;
+// Most files, and bytes, one folder keeps: past either, the oldest files go. An app that crashes
+// in a loop while its destination is down can't fill the disk.
+const MAX_FILES = 100;
+const MAX_BYTES = 50 * 1024 * 1024;
 
 /**
  * One spool folder.
@@ -76,9 +80,31 @@ export class Spool {
     const temp = `${file}.${process.pid}.tmp`;
     fs.writeFileSync(temp, JSON.stringify(content), { mode: 0o600 });
     fs.renameSync(temp, file);
+    this._prune(name);
     return name;
   }
 
+  // Deletes the oldest waiting files beyond MAX_FILES or MAX_BYTES, never the one just written.
+  _prune(keep) {
+    try {
+      const files = [];
+      for (const name of fs.readdirSync(this.dir)) {
+        if (!name.endsWith('.json')) continue;
+        const stat = fs.statSync(path.join(this.dir, name));
+        files.push({ name, at: stat.mtimeMs, size: stat.size });
+      }
+      if (files.length <= MAX_FILES && files.reduce((total, file) => total + file.size, 0) <= MAX_BYTES) return;
+      files.sort((a, b) => (a.name === keep ? -1 : b.name === keep ? 1 : b.at - a.at));
+      let total = 0;
+      files.forEach((file, index) => {
+        total += file.size;
+        if (file.name !== keep && (index >= MAX_FILES || total > MAX_BYTES)) fs.rmSync(path.join(this.dir, file.name), { force: true });
+      });
+    } catch (error) {
+      // Pruning is housekeeping: a folder that can't be read is left as it is.
+    }
+  }
+
   /**
    * The files waiting, oldest first. Files older than `maxAgeMs` are deleted instead, as are
    * temporary files left by writers that died.

package/src/transport/batch.js

@@ -19,6 +19,15 @@ import { RequestRecord } from '../record.js';
 
 // A destination that stays down must not flood the app's logs.
 const WARNING_INTERVAL_MS = 60_000;
+// While the destination is down, the queue is bounded by size as well as by number: a backlog of
+// heavy requests (large bodies, long timelines) never holds more memory than this.
+const MAX_QUEUE_BYTES = 32 * 1024 * 1024;
+// Sizes are measured only once this many records wait (the most that can pass 32 MB unmeasured is
+// this many of the largest records), so normal sending rarely measures anything.
+const MEASURE_FROM = 50;
+// A record whose size can't be measured counts as this much.
+const UNKNOWN_SIZE = 16 * 1024;
+const sizes = new WeakMap();
 // Process samples kept while the destination is down: the newest win.
 const MAX_SAMPLES = 60;
 
@@ -314,11 +323,34 @@ export class BatchTransport {
     }
   }
 
-  // Drops the oldest records beyond maxQueue.
+  // Drops the oldest records beyond maxQueue, and beyond MAX_QUEUE_BYTES once a backlog builds.
   _trimQueue() {
     if (this.queue.length > this.maxQueue) {
       this.dropped += this.queue.length - this.maxQueue;
       this.queue.splice(0, this.queue.length - this.maxQueue);
     }
+    if (this.queue.length <= MEASURE_FROM) return;
+    let total = 0;
+    for (const record of this.queue) total += sizeOf(record);
+    let drop = 0;
+    while (total > MAX_QUEUE_BYTES && drop < this.queue.length - 1) total -= sizeOf(this.queue[drop++]);
+    if (drop) {
+      this.dropped += drop;
+      this.queue.splice(0, drop);
+    }
+  }
+}
+
+// About how much memory a queued record holds: its JSON size, measured once.
+function sizeOf(record) {
+  let size = sizes.get(record);
+  if (size === undefined) {
+    try {
+      size = Buffer.byteLength(JSON.stringify(record));
+    } catch (error) {
+      size = UNKNOWN_SIZE;
+    }
+    sizes.set(record, size);
   }
+  return size;
 }

package/src/transport/cloud.js

@@ -26,6 +26,9 @@ const FLUSH_INTERVAL_MS = 1000;
 const BATCH_SIZE = 100;
 const MAX_QUEUE = 2000;
 const REQUEST_TIMEOUT_MS = 10_000;
+// When the event loop empties (a script ending), the last batch waits at most this long; what
+// doesn't go is spooled for the next run instead.
+const EXIT_TIMEOUT_MS = 2500;
 // Sealed JSON per POST: records that don't fit wait for the next POST. PeekInside Cloud refuses
 // bodies over 8 MB, and serverless hosts such as Vercel stop requests at 4.5 MB before they reach it.
 const MAX_POST_BYTES = 4 * 1024 * 1024;
@@ -98,7 +101,13 @@ export class CloudTransport extends BatchTransport {
     // Never keeps the process alive on its own, and never runs as part of a request.
     this.timer = RequestContext.outside(() => setInterval(() => this.flush(), FLUSH_INTERVAL_MS));
     if (typeof this.timer.unref === 'function') this.timer.unref();
-    // One more batch when the event loop empties, for scripts that end without calling flush().
+    // One more batch when the event loop empties, for scripts that end without calling flush():
+    // briefly, so a slow or unreachable PeekInside Cloud never holds the script up for long.
+    this.exiting = false;
+    this._onBeforeExit = () => {
+      this.exiting = true;
+      return this.flush();
+    };
     process.once('beforeExit', this._onBeforeExit);
 
     this._resendTimer = null;
@@ -235,10 +244,35 @@ export class CloudTransport extends BatchTransport {
     throw new Error(reason);
   }
 
+  // Ending while waiting to retry after a failure: what's queued is spooled rather than lost.
+  _onBackingOff() {
+    if (this.exiting) this._spoolQueue();
+  }
+
   _onRetry(error) {
+    if (this.exiting) {
+      // The script is ending: what's queued waits on disk for the next run.
+      this._spoolQueue();
+      return;
+    }
     this._warn(`Could not reach PeekInside Cloud (${error && error.message}). Retrying in ${Math.round(this.backoff / 1000)} s.`);
   }
 
+  // Seals the newest queued records (within the crash budget) and spools them, synchronously.
+  _spoolQueue() {
+    try {
+      const taken = this._takeNewest(SyncSealer.create(this.apiKey.publicKey, 'event'), MAX_QUEUE);
+      if (!taken.length) return;
+      const body = { events: [], exceptions: [] };
+      for (const { record, payload } of taken) (body[listFor(record)] ??= []).push(payload);
+      Spool.cloud(settings.dataDir(), this.apiKey.projectId).write({ createdAt: Date.now(), body });
+      // Sent by the next run; or by this one, should it keep running after all.
+      this._scheduleResend(SPOOL_RESEND_DELAY_MS);
+    } catch (error) {
+      // A data folder that can't be written loses the records, never the app.
+    }
+  }
+
   // An invalid key or an inactive plan loses every request from now on: an error, not a warning.
   _logDisabled(line) {
     console.error(line);
@@ -258,7 +292,7 @@ export class CloudTransport extends BatchTransport {
       method: 'POST',
       headers: this._headers(),
       body,
-      signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS)
+      signal: AbortSignal.timeout(this.exiting ? EXIT_TIMEOUT_MS : REQUEST_TIMEOUT_MS)
     });
   }
 

package/src/transport/local.js

@@ -79,6 +79,7 @@ export class LocalTransport extends BatchTransport {
     this.blocked = new BlockedRequests();
     this.blockedWarnings = new Set();
     this.outdatedHubNoted = false;
+    this.hubVersionChecked = false;
   }
 
   /**
@@ -198,6 +199,12 @@ export class LocalTransport extends BatchTransport {
   // Sends the batch with the blocked-request counts gathered since the last batch. Records are
   // dropped only when the hub refuses them; when it isn't reachable, both are kept for later.
   async _deliver(batch) {
+    // Once per process: a hub left running by an older PeekInside is replaced by this version's.
+    if (!this.hubVersionChecked) {
+      this.hubVersionChecked = true;
+      const replaced = await this.hub.replaceIfOlder().catch(() => null);
+      if (replaced) console.log(`${LOG_PREFIX} Restarted the local dashboard: it was PeekInside ${replaced}, and this app uses ${VERSION}.`);
+    }
     const blocked = this.blocked.take();
     const samples = this.takeSamples();
     const { requests, exceptions, jobs, logs } = BatchTransport.split(batch);