peekinside 1.2.2 → 1.2.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "peekinside",
-  "version": "1.2.2",
+  "version": "1.2.3",
   "description": "Debug Node.js from the inside: errors with the requests, queries, logs and jobs behind them. A free local dashboard, and encrypted monitoring for production",
   "main": "index.js",
   "types": "./index.d.ts",

package/src/spool.js

@@ -20,6 +20,10 @@ import path from 'path';
 export const CLOUD_FOLDER = 'cloud';
 // A temporary file this old was left by a writer that died mid-write.
 const STALE_TEMP_MS = 60_000;
+// Most files, and bytes, one folder keeps: past either, the oldest files go. An app that crashes
+// in a loop while its destination is down can't fill the disk.
+const MAX_FILES = 100;
+const MAX_BYTES = 50 * 1024 * 1024;
 
 /**
  * One spool folder.
@@ -76,9 +80,31 @@ export class Spool {
     const temp = `${file}.${process.pid}.tmp`;
     fs.writeFileSync(temp, JSON.stringify(content), { mode: 0o600 });
     fs.renameSync(temp, file);
+    this._prune(name);
     return name;
   }
 
+  // Deletes the oldest waiting files beyond MAX_FILES or MAX_BYTES, never the one just written.
+  _prune(keep) {
+    try {
+      const files = [];
+      for (const name of fs.readdirSync(this.dir)) {
+        if (!name.endsWith('.json')) continue;
+        const stat = fs.statSync(path.join(this.dir, name));
+        files.push({ name, at: stat.mtimeMs, size: stat.size });
+      }
+      if (files.length <= MAX_FILES && files.reduce((total, file) => total + file.size, 0) <= MAX_BYTES) return;
+      files.sort((a, b) => (a.name === keep ? -1 : b.name === keep ? 1 : b.at - a.at));
+      let total = 0;
+      files.forEach((file, index) => {
+        total += file.size;
+        if (file.name !== keep && (index >= MAX_FILES || total > MAX_BYTES)) fs.rmSync(path.join(this.dir, file.name), { force: true });
+      });
+    } catch (error) {
+      // Pruning is housekeeping: a folder that can't be read is left as it is.
+    }
+  }
+
   /**
    * The files waiting, oldest first. Files older than `maxAgeMs` are deleted instead, as are
    * temporary files left by writers that died.

package/src/transport/batch.js

@@ -19,6 +19,15 @@ import { RequestRecord } from '../record.js';
 
 // A destination that stays down must not flood the app's logs.
 const WARNING_INTERVAL_MS = 60_000;
+// While the destination is down, the queue is bounded by size as well as by number: a backlog of
+// heavy requests (large bodies, long timelines) never holds more memory than this.
+const MAX_QUEUE_BYTES = 32 * 1024 * 1024;
+// Sizes are measured only once this many records wait (the most that can pass 32 MB unmeasured is
+// this many of the largest records), so normal sending rarely measures anything.
+const MEASURE_FROM = 50;
+// A record whose size can't be measured counts as this much.
+const UNKNOWN_SIZE = 16 * 1024;
+const sizes = new WeakMap();
 // Process samples kept while the destination is down: the newest win.
 const MAX_SAMPLES = 60;
 
@@ -314,11 +323,34 @@ export class BatchTransport {
     }
   }
 
-  // Drops the oldest records beyond maxQueue.
+  // Drops the oldest records beyond maxQueue, and beyond MAX_QUEUE_BYTES once a backlog builds.
   _trimQueue() {
     if (this.queue.length > this.maxQueue) {
       this.dropped += this.queue.length - this.maxQueue;
       this.queue.splice(0, this.queue.length - this.maxQueue);
     }
+    if (this.queue.length <= MEASURE_FROM) return;
+    let total = 0;
+    for (const record of this.queue) total += sizeOf(record);
+    let drop = 0;
+    while (total > MAX_QUEUE_BYTES && drop < this.queue.length - 1) total -= sizeOf(this.queue[drop++]);
+    if (drop) {
+      this.dropped += drop;
+      this.queue.splice(0, drop);
+    }
+  }
+}
+
+// About how much memory a queued record holds: its JSON size, measured once.
+function sizeOf(record) {
+  let size = sizes.get(record);
+  if (size === undefined) {
+    try {
+      size = Buffer.byteLength(JSON.stringify(record));
+    } catch (error) {
+      size = UNKNOWN_SIZE;
+    }
+    sizes.set(record, size);
   }
+  return size;
 }

package/src/transport/cloud.js

@@ -26,6 +26,9 @@ const FLUSH_INTERVAL_MS = 1000;
 const BATCH_SIZE = 100;
 const MAX_QUEUE = 2000;
 const REQUEST_TIMEOUT_MS = 10_000;
+// When the event loop empties (a script ending), the last batch waits at most this long; what
+// doesn't go is spooled for the next run instead.
+const EXIT_TIMEOUT_MS = 2500;
 // Sealed JSON per POST: records that don't fit wait for the next POST. PeekInside Cloud refuses
 // bodies over 8 MB, and serverless hosts such as Vercel stop requests at 4.5 MB before they reach it.
 const MAX_POST_BYTES = 4 * 1024 * 1024;
@@ -98,7 +101,13 @@ export class CloudTransport extends BatchTransport {
     // Never keeps the process alive on its own, and never runs as part of a request.
     this.timer = RequestContext.outside(() => setInterval(() => this.flush(), FLUSH_INTERVAL_MS));
     if (typeof this.timer.unref === 'function') this.timer.unref();
-    // One more batch when the event loop empties, for scripts that end without calling flush().
+    // One more batch when the event loop empties, for scripts that end without calling flush():
+    // briefly, so a slow or unreachable PeekInside Cloud never holds the script up for long.
+    this.exiting = false;
+    this._onBeforeExit = () => {
+      this.exiting = true;
+      return this.flush();
+    };
     process.once('beforeExit', this._onBeforeExit);
 
     this._resendTimer = null;
@@ -235,10 +244,35 @@ export class CloudTransport extends BatchTransport {
     throw new Error(reason);
   }
 
+  // Ending while waiting to retry after a failure: what's queued is spooled rather than lost.
+  _onBackingOff() {
+    if (this.exiting) this._spoolQueue();
+  }
+
   _onRetry(error) {
+    if (this.exiting) {
+      // The script is ending: what's queued waits on disk for the next run.
+      this._spoolQueue();
+      return;
+    }
     this._warn(`Could not reach PeekInside Cloud (${error && error.message}). Retrying in ${Math.round(this.backoff / 1000)} s.`);
   }
 
+  // Seals the newest queued records (within the crash budget) and spools them, synchronously.
+  _spoolQueue() {
+    try {
+      const taken = this._takeNewest(SyncSealer.create(this.apiKey.publicKey, 'event'), MAX_QUEUE);
+      if (!taken.length) return;
+      const body = { events: [], exceptions: [] };
+      for (const { record, payload } of taken) (body[listFor(record)] ??= []).push(payload);
+      Spool.cloud(settings.dataDir(), this.apiKey.projectId).write({ createdAt: Date.now(), body });
+      // Sent by the next run; or by this one, should it keep running after all.
+      this._scheduleResend(SPOOL_RESEND_DELAY_MS);
+    } catch (error) {
+      // A data folder that can't be written loses the records, never the app.
+    }
+  }
+
   // An invalid key or an inactive plan loses every request from now on: an error, not a warning.
   _logDisabled(line) {
     console.error(line);
@@ -258,7 +292,7 @@ export class CloudTransport extends BatchTransport {
       method: 'POST',
       headers: this._headers(),
       body,
-      signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS)
+      signal: AbortSignal.timeout(this.exiting ? EXIT_TIMEOUT_MS : REQUEST_TIMEOUT_MS)
     });
   }