peekinside 1.2.2 → 1.2.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "peekinside",
-  "version": "1.2.2",
+  "version": "1.2.4",
   "description": "Debug Node.js from the inside: errors with the requests, queries, logs and jobs behind them. A free local dashboard, and encrypted monitoring for production",
   "main": "index.js",
   "types": "./index.d.ts",

package/src/spool.js

@@ -20,6 +20,10 @@ import path from 'path';
 export const CLOUD_FOLDER = 'cloud';
 // A temporary file this old was left by a writer that died mid-write.
 const STALE_TEMP_MS = 60_000;
+// Most files, and bytes, one folder keeps: past either, the oldest files go. An app that crashes
+// in a loop while its destination is down can't fill the disk.
+const MAX_FILES = 100;
+const MAX_BYTES = 50 * 1024 * 1024;
 
 /**
  * One spool folder.
@@ -76,9 +80,31 @@ export class Spool {
     const temp = `${file}.${process.pid}.tmp`;
     fs.writeFileSync(temp, JSON.stringify(content), { mode: 0o600 });
     fs.renameSync(temp, file);
+    this._prune(name);
     return name;
   }
 
+  // Deletes the oldest waiting files beyond MAX_FILES or MAX_BYTES, never the one just written.
+  _prune(keep) {
+    try {
+      const files = [];
+      for (const name of fs.readdirSync(this.dir)) {
+        if (!name.endsWith('.json')) continue;
+        const stat = fs.statSync(path.join(this.dir, name));
+        files.push({ name, at: stat.mtimeMs, size: stat.size });
+      }
+      if (files.length <= MAX_FILES && files.reduce((total, file) => total + file.size, 0) <= MAX_BYTES) return;
+      files.sort((a, b) => (a.name === keep ? -1 : b.name === keep ? 1 : b.at - a.at));
+      let total = 0;
+      files.forEach((file, index) => {
+        total += file.size;
+        if (file.name !== keep && (index >= MAX_FILES || total > MAX_BYTES)) fs.rmSync(path.join(this.dir, file.name), { force: true });
+      });
+    } catch (error) {
+      // Pruning is housekeeping: a folder that can't be read is left as it is.
+    }
+  }
+
   /**
    * The files waiting, oldest first. Files older than `maxAgeMs` are deleted instead, as are
    * temporary files left by writers that died.

package/src/transport/batch.js

@@ -19,6 +19,15 @@ import { RequestRecord } from '../record.js';
 
 // A destination that stays down must not flood the app's logs.
 const WARNING_INTERVAL_MS = 60_000;
+// While the destination is down, the queue is bounded by size as well as by number: a backlog of
+// heavy requests (large bodies, long timelines) never holds more memory than this.
+const MAX_QUEUE_BYTES = 32 * 1024 * 1024;
+// Sizes are measured only once this many records wait (the most that can pass 32 MB unmeasured is
+// this many of the largest records), so normal sending rarely measures anything.
+const MEASURE_FROM = 50;
+// A record whose size can't be measured counts as this much.
+const UNKNOWN_SIZE = 16 * 1024;
+const sizes = new WeakMap();
 // Process samples kept while the destination is down: the newest win.
 const MAX_SAMPLES = 60;
 
@@ -314,11 +323,34 @@ export class BatchTransport {
     }
   }
 
-  // Drops the oldest records beyond maxQueue.
+  // Drops the oldest records beyond maxQueue, and beyond MAX_QUEUE_BYTES once a backlog builds.
   _trimQueue() {
     if (this.queue.length > this.maxQueue) {
       this.dropped += this.queue.length - this.maxQueue;
       this.queue.splice(0, this.queue.length - this.maxQueue);
     }
+    if (this.queue.length <= MEASURE_FROM) return;
+    let total = 0;
+    for (const record of this.queue) total += sizeOf(record);
+    let drop = 0;
+    while (total > MAX_QUEUE_BYTES && drop < this.queue.length - 1) total -= sizeOf(this.queue[drop++]);
+    if (drop) {
+      this.dropped += drop;
+      this.queue.splice(0, drop);
+    }
+  }
+}
+
+// About how much memory a queued record holds: its JSON size, measured once.
+function sizeOf(record) {
+  let size = sizes.get(record);
+  if (size === undefined) {
+    try {
+      size = Buffer.byteLength(JSON.stringify(record));
+    } catch (error) {
+      size = UNKNOWN_SIZE;
+    }
+    sizes.set(record, size);
   }
+  return size;
 }

package/src/transport/cloud.js

@@ -26,6 +26,9 @@ const FLUSH_INTERVAL_MS = 1000;
 const BATCH_SIZE = 100;
 const MAX_QUEUE = 2000;
 const REQUEST_TIMEOUT_MS = 10_000;
+// When the event loop empties (a script ending), the last batch waits at most this long; what
+// doesn't go is spooled for the next run instead.
+const EXIT_TIMEOUT_MS = 2500;
 // Sealed JSON per POST: records that don't fit wait for the next POST. PeekInside Cloud refuses
 // bodies over 8 MB, and serverless hosts such as Vercel stop requests at 4.5 MB before they reach it.
 const MAX_POST_BYTES = 4 * 1024 * 1024;
@@ -53,7 +56,16 @@ process.stdin.on('data', (chunk) => { input += chunk; });
 process.stdin.on('end', () => {
   let job;
   try { job = JSON.parse(input); } catch (error) { process.exit(1); }
-  fetch(job.url, { method: 'POST', headers: job.headers, body: job.body, signal: AbortSignal.timeout(${CRASH_SEND_TIMEOUT_MS - 500}) })
+  const signal = AbortSignal.timeout(${CRASH_SEND_TIMEOUT_MS - 500});
+  const post = (url) => fetch(url, { method: 'POST', headers: job.headers, body: job.body, redirect: 'manual', signal });
+  post(job.url)
+    .then((response) => {
+      const location = response.headers.get('location');
+      if ((response.status !== 307 && response.status !== 308) || !location) return response;
+      const target = new URL(location, job.url);
+      const ok = job.follow.hosts.includes(target.hostname) && job.follow.protocols.includes(target.protocol) && target.pathname === job.follow.path && (target.port === '' || target.port === job.follow.port);
+      return ok ? post(target.href) : response;
+    })
     .then((response) => process.exit(response.ok ? ${SENT} : response.status === 400 || response.status === 413 ? ${REFUSED} : 2))
     .catch(() => process.exit(1));
 });
@@ -98,7 +110,13 @@ export class CloudTransport extends BatchTransport {
     // Never keeps the process alive on its own, and never runs as part of a request.
     this.timer = RequestContext.outside(() => setInterval(() => this.flush(), FLUSH_INTERVAL_MS));
     if (typeof this.timer.unref === 'function') this.timer.unref();
-    // One more batch when the event loop empties, for scripts that end without calling flush().
+    // One more batch when the event loop empties, for scripts that end without calling flush():
+    // briefly, so a slow or unreachable PeekInside Cloud never holds the script up for long.
+    this.exiting = false;
+    this._onBeforeExit = () => {
+      this.exiting = true;
+      return this.flush();
+    };
     process.once('beforeExit', this._onBeforeExit);
 
     this._resendTimer = null;
@@ -235,10 +253,35 @@ export class CloudTransport extends BatchTransport {
     throw new Error(reason);
   }
 
+  // Ending while waiting to retry after a failure: what's queued is spooled rather than lost.
+  _onBackingOff() {
+    if (this.exiting) this._spoolQueue();
+  }
+
   _onRetry(error) {
+    if (this.exiting) {
+      // The script is ending: what's queued waits on disk for the next run.
+      this._spoolQueue();
+      return;
+    }
     this._warn(`Could not reach PeekInside Cloud (${error && error.message}). Retrying in ${Math.round(this.backoff / 1000)} s.`);
   }
 
+  // Seals the newest queued records (within the crash budget) and spools them, synchronously.
+  _spoolQueue() {
+    try {
+      const taken = this._takeNewest(SyncSealer.create(this.apiKey.publicKey, 'event'), MAX_QUEUE);
+      if (!taken.length) return;
+      const body = { events: [], exceptions: [] };
+      for (const { record, payload } of taken) (body[listFor(record)] ??= []).push(payload);
+      Spool.cloud(settings.dataDir(), this.apiKey.projectId).write({ createdAt: Date.now(), body });
+      // Sent by the next run; or by this one, should it keep running after all.
+      this._scheduleResend(SPOOL_RESEND_DELAY_MS);
+    } catch (error) {
+      // A data folder that can't be written loses the records, never the app.
+    }
+  }
+
   // An invalid key or an inactive plan loses every request from now on: an error, not a warning.
   _logDisabled(line) {
     console.error(line);
@@ -253,13 +296,21 @@ export class CloudTransport extends BatchTransport {
     };
   }
 
-  _post(body) {
-    return fetch(this.endpoint, {
+  // Posts to the endpoint. A redirect to the site's twin (see redirectTarget) is followed here,
+  // with the API key, and remembered; fetch() would follow it but drop the key.
+  async _post(body) {
+    const post = (url) => fetch(url, {
       method: 'POST',
       headers: this._headers(),
       body,
-      signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS)
+      redirect: 'manual',
+      signal: AbortSignal.timeout(this.exiting ? EXIT_TIMEOUT_MS : REQUEST_TIMEOUT_MS)
     });
+    const response = await post(this.endpoint);
+    const moved = redirectTarget(response, this.endpoint);
+    if (!moved) return response;
+    this.endpoint = moved;
+    return post(moved);
   }
 
   // The newest queued records, sealed, within the crash budget; they leave the queue.
@@ -280,7 +331,7 @@ export class CloudTransport extends BatchTransport {
   // Posts from a child process and waits for it: the only way to send over the network
   // synchronously. The secret travels on stdin, never on the command line.
   _sendNow(body) {
-    const input = JSON.stringify({ url: this.endpoint, headers: this._headers(), body });
+    const input = JSON.stringify({ url: this.endpoint, headers: this._headers(), body, follow: followRule(this.endpoint) });
     const env = process.platform === 'win32' && process.env.SystemRoot ? { SystemRoot: process.env.SystemRoot } : {};
     const result = spawnSync(process.execPath, ['-e', SENDER], {
       input,
@@ -361,3 +412,39 @@ function listFor(record) {
   if (record instanceof LogRecord) return 'logs';
   return 'events';
 }
+
+/**
+ * Where a 307 or 308 answer from PeekInside Cloud moves its endpoint, when the API key may follow
+ * it there, or null. Hosting often sends a site's bare domain to its www name, or back (Vercel
+ * does by default), and fetch() drops the Authorization header on such a redirect, so it is
+ * followed by hand, and only to the same path on the same host or its www twin, never downgrading
+ * https. Anywhere else, the answer stands (and the batch is retried later).
+ *
+ * @param {Response} response
+ * @param {string} from - The URL that answered
+ * @returns {string|null}
+ */
+export function redirectTarget(response, from) {
+  if (response.status !== 307 && response.status !== 308) return null;
+  const location = response.headers.get('location');
+  if (!location) return null;
+  let target;
+  try {
+    target = new URL(location, from);
+  } catch (error) {
+    return null;
+  }
+  const rule = followRule(from);
+  const ok = rule.hosts.includes(target.hostname) && rule.protocols.includes(target.protocol) && target.pathname === rule.path && (target.port === '' || target.port === rule.port);
+  return ok ? target.href : null;
+}
+
+// What redirectTarget accepts, as data, so the crash sender's child process applies the same rule.
+function followRule(from) {
+  const url = new URL(from);
+  const host = url.hostname;
+  const named = host.includes('.') && !host.startsWith('[') && !/^[\d.]+$/.test(host) && host !== 'localhost' && !host.endsWith('.localhost');
+  const hosts = [host];
+  if (named) hosts.push(host.startsWith('www.') ? host.slice(4) : `www.${host}`);
+  return { hosts, protocols: url.protocol === 'http:' ? ['http:', 'https:'] : [url.protocol], path: url.pathname, port: url.port };
+}