peekinside 1.2.4 → 1.3.0

package/src/runtime/sampler.js

@@ -1,10 +1,12 @@
 /**
- * Samples the process now and then: memory (to spot leaks, as growth over hours), CPU and the
- * event-loop delay. Samples are numbers only, and go to the dashboard with the other records:
- * every 10 seconds to the local dashboard, every minute to PeekInside Cloud.
+ * Samples the process now and then: memory (to spot leaks, as growth over hours, and how close the
+ * heap is to its limit), CPU, the event loop (its delay, and how busy it was), garbage collection,
+ * open handles and the requests it served. Samples are numbers only, and go to the dashboard with
+ * the other records: every 10 seconds to the local dashboard, every minute to PeekInside Cloud.
  */
 
-import { performance } from 'perf_hooks';
+import { PerformanceObserver, performance } from 'perf_hooks';
+import v8 from 'v8';
 import { RequestContext } from '../exceptions/request-context.js';
 
 /**
@@ -18,8 +20,73 @@ import { RequestContext } from '../exceptions/request-context.js';
  * @property {number|null} loopP50 - Event-loop delay since the last sample, ms
  * @property {number|null} loopP99
  * @property {number|null} loopMax
+ * @property {number|null} arrayBuffers - Bytes of ArrayBuffers and Buffers (part of external)
+ * @property {number|null} heapLimit - Bytes the heap may grow to before the process runs out of memory
+ * @property {number|null} elu - Percent of the time since the last sample the event loop was busy
+ * @property {number|null} gcCount - Garbage collections since the last sample
+ * @property {number|null} gcMs - Time they paused the process, ms
+ * @property {number|null} gcMax - The longest of them, ms
+ * @property {number|null} handles - Open handles and requests (sockets, timers, files) now
+ * @property {number|null} requests - Requests this process finished since the last sample
+ * @property {number|null} inFlight - Requests open when the sample was taken
  */
 
+/**
+ * Garbage collections since the last look: how many, and how long they paused the process.
+ */
+class GcWatch {
+  constructor() {
+    this.observer = null;
+    this.reset();
+  }
+
+  /**
+   * @returns {void}
+   */
+  start() {
+    if (this.observer) return;
+    try {
+      this.observer = new PerformanceObserver((list) => {
+        for (const entry of list.getEntries()) {
+          this.count++;
+          this.ms += entry.duration;
+          if (entry.duration > this.max) this.max = entry.duration;
+        }
+      });
+      // Outside any request: collections belong to the process, not to the request that started it.
+      RequestContext.outside(() => this.observer.observe({ entryTypes: ['gc'] }));
+    } catch (error) {
+      this.observer = null;
+    }
+  }
+
+  /**
+   * @returns {void}
+   */
+  stop() {
+    if (this.observer) this.observer.disconnect();
+    this.observer = null;
+  }
+
+  /**
+   * The collections since the last call, or null when they can't be watched.
+   *
+   * @returns {{ count: number, ms: number, max: number }|null}
+   */
+  take() {
+    if (!this.observer) return null;
+    const taken = { count: this.count, ms: round(this.ms), max: round(this.max) };
+    this.reset();
+    return taken;
+  }
+
+  reset() {
+    this.count = 0;
+    this.ms = 0;
+    this.max = 0;
+  }
+}
+
 /**
  * Takes a sample every `intervalMs` and hands it over.
  */
@@ -29,14 +96,18 @@ export class RuntimeSampler {
    * @param {number} options.intervalMs
    * @param {(sample: RuntimeSample) => void} options.onSample
    * @param {import('./loop-monitor.js').LoopMonitor} options.monitor - Measures the delay
+   * @param {import('./request-counter.js').RequestCounter} [options.requests] - Counts the requests served
    */
-  constructor({ intervalMs, onSample, monitor }) {
+  constructor({ intervalMs, onSample, monitor, requests = null }) {
     this.intervalMs = intervalMs;
     this.onSample = onSample;
     this.monitor = monitor;
+    this.requests = requests;
     this.timer = null;
+    this.gc = new GcWatch();
     this.lastAt = performance.now();
     this.lastCpu = process.cpuUsage();
+    this.lastElu = loopUtilization();
   }
 
   /**
@@ -46,6 +117,8 @@ export class RuntimeSampler {
     if (this.timer) return this;
     this.lastAt = performance.now();
     this.lastCpu = process.cpuUsage();
+    this.lastElu = loopUtilization();
+    this.gc.start();
     this.timer = RequestContext.outside(() => setInterval(() => this.take(), this.intervalMs));
     if (typeof this.timer.unref === 'function') this.timer.unref();
     return this;
@@ -57,6 +130,7 @@ export class RuntimeSampler {
   stop() {
     clearInterval(this.timer);
     this.timer = null;
+    this.gc.stop();
   }
 
   /**
@@ -70,9 +144,14 @@ export class RuntimeSampler {
       const elapsed = Math.max(1, now - this.lastAt);
       const cpu = process.cpuUsage(this.lastCpu);
       const delay = this.monitor ? this.monitor.delaySince(this.lastAt) : null;
+      const elu = loopUtilization();
+      const busy = elu && this.lastElu ? performance.eventLoopUtilization(elu, this.lastElu).utilization : null;
       this.lastAt = now;
       this.lastCpu = process.cpuUsage();
+      this.lastElu = elu;
       const memory = process.memoryUsage();
+      const gc = this.gc.take();
+      const traffic = this.requests ? this.requests.take() : null;
       const sample = {
         at: Date.now(),
         rss: memory.rss,
@@ -82,7 +161,16 @@ export class RuntimeSampler {
         cpu: Math.round(((cpu.user + cpu.system) / 1000 / elapsed) * 1000) / 10,
         loopP50: delay ? delay.p50 : null,
         loopP99: delay ? delay.p99 : null,
-        loopMax: delay ? delay.max : null
+        loopMax: delay ? delay.max : null,
+        arrayBuffers: Number.isFinite(memory.arrayBuffers) ? memory.arrayBuffers : null,
+        heapLimit: heapLimit(),
+        elu: Number.isFinite(busy) ? Math.round(busy * 1000) / 10 : null,
+        gcCount: gc ? gc.count : null,
+        gcMs: gc ? gc.ms : null,
+        gcMax: gc ? gc.max : null,
+        handles: openHandles(),
+        requests: traffic ? traffic.finished : null,
+        inFlight: traffic ? traffic.open : null
       };
       this.onSample(sample);
       return sample;
@@ -91,3 +179,29 @@ export class RuntimeSampler {
     }
   }
 }
+
+// How busy the loop has been since the process started (Node.js 14.10 and later).
+function loopUtilization() {
+  return typeof performance.eventLoopUtilization === 'function' ? performance.eventLoopUtilization() : null;
+}
+
+function heapLimit() {
+  try {
+    return v8.getHeapStatistics().heap_size_limit;
+  } catch (error) {
+    return null;
+  }
+}
+
+// Sockets, timers, files and other resources keeping the loop alive (Node.js 17 and later).
+function openHandles() {
+  try {
+    return typeof process.getActiveResourcesInfo === 'function' ? process.getActiveResourcesInfo().length : null;
+  } catch (error) {
+    return null;
+  }
+}
+
+function round(value) {
+  return Math.round(value * 10) / 10;
+}

package/src/timeline/timeline.js

@@ -37,7 +37,7 @@ export const TIMELINE_DEFAULTS = Object.freeze({
 });
 
 // Spans listed per request, by type.
-const SPAN_CAPS = Object.freeze({ query: 500, http: 100, log: 200, measure: 100, exception: 10, block: 20 });
+const SPAN_CAPS = Object.freeze({ query: 500, http: 100, log: 200, measure: 100, exception: 10, block: 20, dump: 50 });
 const MAX_STATEMENTS = 200;
 // Executions per statement whose parameters are kept.
 const PARAMS_PER_STATEMENT = 20;
@@ -121,7 +121,7 @@ export class Timeline {
     this.origin = origin;
     this.closed = false;
     this.spans = [];
-    this.listed = { query: 0, http: 0, log: 0, measure: 0, exception: 0, block: 0 };
+    this.listed = { query: 0, http: 0, log: 0, measure: 0, exception: 0, block: 0, dump: 0 };
     this.dropped = 0;
     this.statements = [];
     this.byFingerprint = new Map();
@@ -263,7 +263,7 @@ export class Timeline {
    *
    * @param {'debug'|'info'|'warn'|'error'} level
    * @param {string} message - Formatted as it was written
-   * @param {'console'|'pino'|'winston'} [source='console']
+   * @param {'console'|'pino'|'winston'|'nest'} [source='console']
    * @returns {void}
    */
   log(level, message, source = 'console') {
@@ -282,6 +282,27 @@ export class Timeline {
     }
   }
 
+  /**
+   * Records a dump made during the request (local dashboard only): its label and a preview of the
+   * value; the Dumps page has the value itself.
+   *
+   * @param {{ id: string, label: string|null, summary: { preview: string } }} record - A DumpRecord
+   * @returns {void}
+   */
+  dump(record) {
+    if (this.closed) return;
+    try {
+      if (this.listed.dump < SPAN_CAPS.dump) {
+        this.listed.dump++;
+        this.spans.push({ type: 'dump', start: this._offset(performance.now()), id: record.id, label: record.label, preview: cut(record.summary.preview, MAX_LOG) });
+      } else {
+        this.dropped++;
+      }
+    } catch (error) {
+      // The dump is still on the Dumps page.
+    }
+  }
+
   /**
    * Records an exception recorded during the request.
    *

package/src/transport/batch.js

@@ -13,6 +13,7 @@
 import { LOG_PREFIX } from '../config.js';
 import { ExceptionRecord } from '../exceptions/exception-record.js';
 import { RequestContext } from '../exceptions/request-context.js';
+import { DumpRecord } from '../dumps/dump-record.js';
 import { JobRunRecord } from '../jobs/job-run-record.js';
 import { LogRecord } from '../logs/log-record.js';
 import { RequestRecord } from '../record.js';
@@ -170,13 +171,15 @@ export class BatchTransport {
     const exceptions = [];
     const jobs = [];
     const logs = [];
+    const dumps = [];
     for (const record of records) {
       if (record instanceof ExceptionRecord) exceptions.push(record);
       else if (record instanceof JobRunRecord) jobs.push(record);
       else if (record instanceof LogRecord) logs.push(record);
+      else if (record instanceof DumpRecord) dumps.push(record);
       else requests.push(record);
     }
-    return { requests, exceptions, jobs, logs };
+    return { requests, exceptions, jobs, logs, dumps };
   }
 
   /**

package/src/transport/cloud.js

@@ -16,10 +16,12 @@ import { settings } from '../config.js';
 import { ApiKey, Sealer, SyncSealer } from '../crypto.js';
 import { RequestContext } from '../exceptions/request-context.js';
 import { ExceptionRecord } from '../exceptions/exception-record.js';
+import { DumpRecord } from '../dumps/dump-record.js';
 import { JobRunRecord } from '../jobs/job-run-record.js';
 import { LogRecord } from '../logs/log-record.js';
 import { RequestRecord } from '../record.js';
 import { Spool } from '../spool.js';
+import { RuleMatcher } from '../alerts/rule-matcher.js';
 import { BatchTransport } from './batch.js';
 
 const FLUSH_INTERVAL_MS = 1000;
@@ -86,7 +88,7 @@ export class CloudTransport extends BatchTransport {
   constructor({ apiKey, environment, release = null, version }) {
     const key = ApiKey.parse(apiKey);
     if (!key) {
-      throw new Error('PEEKINSIDE_API_KEY is not a valid PeekInside API key');
+      throw new Error(ApiKey.problem(apiKey));
     }
     super({
       environment: RequestRecord.normalizeEnvironment(environment, 'production'),
@@ -121,12 +123,25 @@ export class CloudTransport extends BatchTransport {
 
     this._resendTimer = null;
     this._resending = null;
+    // Alert rules on sealed fields, checked here before records are sealed (started with the first batch).
+    this.rules = new RuleMatcher({ fetchRules: () => this._fetchRules() });
     // Set when records that didn't fit in the last POST wait for the next one.
     this._more = false;
     // Batches spooled by an earlier run that crashed.
     this._scheduleResend(SPOOL_RESEND_DELAY_MS);
   }
 
+  /**
+   * Queues a record for sending. Dumps are for the local dashboard: PeekInside Cloud never gets them.
+   *
+   * @param {Object} record
+   * @returns {void}
+   */
+  enqueueRecord(record) {
+    if (record instanceof DumpRecord) return;
+    super.enqueueRecord(record);
+  }
+
   /**
    * Stops the background timer and the flush at exit. Sending itself stays possible: drain()
    * still sends what is queued.
@@ -137,6 +152,7 @@ export class CloudTransport extends BatchTransport {
     clearInterval(this.timer);
     clearTimeout(this._resendTimer);
     this._resendTimer = null;
+    this.rules.stop();
     super.stop();
   }
 
@@ -189,6 +205,7 @@ export class CloudTransport extends BatchTransport {
   // even a large one, or it would block the queue.
   async _pack(batch, sealer) {
     const lists = { events: [], exceptions: [], jobs: [], logs: [] };
+    const alerts = [];
     let size = 32;
     let count = 0;
     for (const record of batch) {
@@ -198,8 +215,10 @@ export class CloudTransport extends BatchTransport {
       size += length;
       count++;
       lists[listFor(record)].push(sealed);
+      // Which alert rules it met on what only this server can read: ids, never the content.
+      alerts.push(...this.rules.matches(record));
     }
-    return { ...lists, rest: batch.slice(count) };
+    return { ...lists, alerts, rest: batch.slice(count) };
   }
 
   // Records that didn't fit go back to the front of the queue, in order, and go out right away.
@@ -213,11 +232,12 @@ export class CloudTransport extends BatchTransport {
   async _deliver(batch) {
     // A fresh ephemeral key per batch: one key agreement, a new IV per message.
     const sealer = await Sealer.create(this.apiKey.publicKey, 'event');
-    const { events, exceptions, jobs, logs, rest } = await this._pack(batch, sealer);
+    const { events, exceptions, jobs, logs, alerts, rest } = await this._pack(batch, sealer);
     const body = { events };
     if (exceptions.length) body.exceptions = exceptions;
     if (jobs.length) body.jobs = jobs;
     if (logs.length) body.logs = logs;
+    if (alerts.length) body.alerts = alerts;
     const samples = this.takeSamples();
     if (samples.length) {
       body.instance = { ...this.instance, environment: this.environment, release: this.release };
@@ -234,6 +254,8 @@ export class CloudTransport extends BatchTransport {
     const sent = batch.length - rest.length;
 
     if (response.ok) {
+      // PeekInside Cloud answers: from now on, the alert rules it holds are checked here too.
+      this.rules.start();
       this._requeue(rest);
       return true;
     }
@@ -287,6 +309,15 @@ export class CloudTransport extends BatchTransport {
     console.error(line);
   }
 
+  // The project's alert rules on sealed fields, from beside the ingest endpoint.
+  async _fetchRules() {
+    if (this.exiting) throw new Error('exiting');
+    // Never after a redirect: the API key goes only where batches go (see _post).
+    const response = await fetch(`${this.endpoint}/alert-rules`, { headers: this._headers(), redirect: 'manual', signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS) });
+    if (!response.ok) throw new Error(`HTTP ${response.status}`);
+    return response.json();
+  }
+
   _headers() {
     return {
       'content-type': 'application/json',

package/src/transport/local.js

@@ -182,14 +182,15 @@ export class LocalTransport extends BatchTransport {
       const records = fatal ? [...this.sending, ...this.queue.splice(0)] : [];
       for (const exchange of exchanges) records.push(RequestRecord.fromExchange(exchange, this.environment, this.release));
       records.push(...exceptions);
-      const { requests, exceptions: errors, jobs, logs } = BatchTransport.split(records);
+      const { requests, exceptions: errors, jobs, logs, dumps } = BatchTransport.split(records);
       Spool.local(settings.dataDir(), this.project.id).write({
         project: this.project,
         instance: this.instance,
         requests: requests.slice(-MAX_SPOOLED_REQUESTS),
         exceptions: errors.slice(-MAX_SPOOLED_EXCEPTIONS),
         jobs: jobs.slice(-MAX_SPOOLED_EXCEPTIONS),
-        logs: logs.slice(-MAX_SPOOLED_REQUESTS)
+        logs: logs.slice(-MAX_SPOOLED_REQUESTS),
+        dumps: dumps.slice(-MAX_SPOOLED_EXCEPTIONS)
       });
     } catch (error) {
       // A data folder that can't be written loses the records, never the app.
@@ -207,13 +208,14 @@ export class LocalTransport extends BatchTransport {
     }
     const blocked = this.blocked.take();
     const samples = this.takeSamples();
-    const { requests, exceptions, jobs, logs } = BatchTransport.split(batch);
+    const { requests, exceptions, jobs, logs, dumps } = BatchTransport.split(batch);
     try {
       const answer = await this.hub.request('POST', '/api/ingest', {
-        body: { project: this.project, instance: this.instance, requests, exceptions, jobs, logs, samples, blocked: blocked.map(([, entry]) => entry) }
+        body: { project: this.project, instance: this.instance, requests, exceptions, jobs, logs, dumps, samples, blocked: blocked.map(([, entry]) => entry) }
       });
       if (exceptions.length && answer.acceptedExceptions === undefined) this._noteOutdatedHub();
       else if ((jobs.length || logs.length) && answer.acceptedJobs === undefined) this._noteOutdatedHub();
+      else if (dumps.length && answer.acceptedDumps === undefined) this._noteOutdatedHub();
       return true;
     } catch (error) {
       if (error.code === 'NOT_HUB') {
@@ -238,7 +240,7 @@ export class LocalTransport extends BatchTransport {
   }
 
   // A hub started before this version keeps running until it is restarted, and ignores what it
-  // doesn't know (exceptions, job runs, log lines).
+  // doesn't know (exceptions, job runs, log lines, dumps).
   _noteOutdatedHub() {
     if (this.outdatedHubNoted) return;
     this.outdatedHubNoted = true;

package/src/user.js

@@ -9,12 +9,14 @@
 const MAX_ID = 128;
 const MAX_EMAIL = 254;
 const MAX_NAME = 128;
+const MAX_PHONE = 32;
 
 /**
  * @typedef {Object} User
  * @property {string|null} id
  * @property {string|null} email
  * @property {string|null} name
+ * @property {string|null} phone
  */
 
 /**
@@ -22,14 +24,15 @@ const MAX_NAME = 128;
  */
 export class UserInfo {
   /**
-   * Accepts `{ id, email, name }` (also `_id`, `userId` and `username`, as user objects of common
-   * libraries have them), or just an id. Anything else is no user.
+   * Accepts `{ id, email, name, phone }` (also `_id`, `userId`, `username`, `phoneNumber`,
+   * `phone_number` and `mobile`, as user objects of common libraries and apps have them), or just
+   * an id. Anything else is no user.
    *
    * @param {*} user
    * @returns {User|null} null when it names nobody
    *
    * @example
-   * UserInfo.normalize({ id: 42, email: 'ada@example.com' }) // { id: '42', email: 'ada@example.com', name: null }
+   * UserInfo.normalize({ id: 42, phone: '+91 98765 43210' }) // { id: '42', email: null, name: null, phone: '+91 98765 43210' }
    */
   static normalize(user) {
     if (user === null || user === undefined) return null;
@@ -37,25 +40,27 @@ export class UserInfo {
     let id;
     let email;
     let name;
+    let phone;
     try {
       id = firstDefined(user.id, user._id, user.userId);
       email = user.email;
       name = firstDefined(user.name, user.username);
+      phone = firstDefined(user.phone, user.phoneNumber, user.phone_number, user.mobile);
     } catch (error) {
       return null;
     }
-    const clean = { id: identifier(id), email: label(email, MAX_EMAIL), name: label(name, MAX_NAME) };
-    return clean.id || clean.email || clean.name ? clean : null;
+    const clean = { id: identifier(id), email: label(email, MAX_EMAIL), name: label(name, MAX_NAME), phone: phoneNumber(phone) };
+    return clean.id || clean.email || clean.name || clean.phone ? clean : null;
   }
 
   /**
-   * One string that tells users apart: the id, else the email, else the name.
+   * One string that tells users apart: the id, else the email, else the phone, else the name.
    *
    * @param {User|null} user
    * @returns {string|null}
    */
   static key(user) {
-    return user ? user.id || user.email || user.name || null : null;
+    return user ? user.id || user.email || user.phone || user.name || null : null;
   }
 }
 
@@ -76,6 +81,12 @@ function identifier(value) {
   return null;
 }
 
+// As the app wrote it ("+91 98765 43210"); numbers (and bigints) as their digits.
+function phoneNumber(value) {
+  if (typeof value === 'number' || typeof value === 'bigint') return label(String(value), MAX_PHONE);
+  return label(value, MAX_PHONE);
+}
+
 function label(value, max) {
   if (typeof value !== 'string') return null;
   const clean = value.trim().slice(0, max);