peekinside 0.0.0-stage → 1.2.0

package/src/hub/store.js

@@ -0,0 +1,404 @@
+/**
+ * Disk storage for the local hub: one folder per project under <data dir>/projects, each held
+ * by a ProjectLog (see project-log.js for the file format).
+ *
+ * The store is the registry: it loads projects at startup, creates them when an app first
+ * checks in, and routes every operation to the right project.
+ */
+
+import fs from 'fs';
+import path from 'path';
+import { ProjectLog } from './project-log.js';
+
+// Project ids become folder names, so they can never contain separators or dots.
+export const PROJECT_ID = /^[a-z0-9](?:[a-z0-9-]{0,62}[a-z0-9])?$/;
+export const DEFAULT_MAX_REQUESTS = 1000;
+export const DEFAULT_MAX_EXCEPTIONS = 500;
+export const DEFAULT_MAX_JOBS = 2000;
+export const DEFAULT_MAX_LOGS = 10000;
+
+/**
+ * All projects on this machine.
+ */
+export class Store {
+  /**
+   * @param {string} dataDir - PeekInside data folder
+   * @param {{ maxRequests?: number, maxExceptions?: number }} [options] - Requests and exceptions
+   *   kept per project (at least 10 each)
+   */
+  constructor(dataDir, { maxRequests = DEFAULT_MAX_REQUESTS, maxExceptions = DEFAULT_MAX_EXCEPTIONS } = {}) {
+    this.dir = path.join(dataDir, 'projects');
+    this.maxRequests = Math.max(10, Math.floor(maxRequests) || DEFAULT_MAX_REQUESTS);
+    this.maxExceptions = Math.max(10, Math.floor(maxExceptions) || DEFAULT_MAX_EXCEPTIONS);
+    // Four segments per log: retention deletes a quarter of the records at a time.
+    this.segmentSize = Math.max(10, Math.ceil(this.maxRequests / 4));
+    this.exceptionSegmentSize = Math.max(10, Math.ceil(this.maxExceptions / 4));
+    /** @type {Map<string, ProjectLog>} */
+    this.projects = new Map();
+  }
+
+  /**
+   * Loads the projects on disk. A project that can't be read is skipped (and logged), so one
+   * damaged folder never stops the hub.
+   *
+   * @returns {Store} This store
+   */
+  load() {
+    fs.mkdirSync(this.dir, { recursive: true, mode: 0o700 });
+    for (const id of fs.readdirSync(this.dir)) {
+      if (!PROJECT_ID.test(id)) continue;
+      try {
+        const project = ProjectLog.load(path.join(this.dir, id), id, this._limits());
+        this.projects.set(id, project);
+        // After registering: a segment that can't be deleted never hides the project.
+        project.enforceRetention();
+      } catch (error) {
+        console.error(`Skipping project ${id}: ${error.message}`);
+      }
+    }
+    return this;
+  }
+
+  /**
+   * Number of projects.
+   *
+   * @type {number}
+   */
+  get size() {
+    return this.projects.size;
+  }
+
+  /**
+   * @param {string} projectId
+   * @returns {boolean} Whether the project exists
+   */
+  has(projectId) {
+    return this.projects.has(projectId);
+  }
+
+  /**
+   * Creates or refreshes a project and records that one of its processes checked in.
+   *
+   * @param {{ id: string, name: string, root: string }} info
+   * @param {Object} instance - Process details: pid, port, framework, node, sdk, environment, startedAt
+   * @param {number} [now=Date.now()]
+   * @returns {ProjectLog}
+   */
+  touch(info, instance, now = Date.now()) {
+    const project = this.ensure(info, now);
+    project.checkIn(info, instance, now);
+    return project;
+  }
+
+  /**
+   * Creates a project if it doesn't exist yet, without counting a check-in: for records spooled
+   * by a process that has exited since.
+   *
+   * @param {{ id: string, name: string, root: string }} info
+   * @param {number} [now=Date.now()]
+   * @returns {ProjectLog}
+   */
+  ensure(info, now = Date.now()) {
+    let project = this.projects.get(info.id);
+    if (!project) {
+      project = ProjectLog.create(path.join(this.dir, info.id), info, now, this._limits());
+      this.projects.set(info.id, project);
+    } else {
+      project.rename(info);
+    }
+    return project;
+  }
+
+  /**
+   * Adds counts of requests that were not recorded because they reached a server (local mode
+   * is for development; servers need PeekInside Cloud).
+   *
+   * @param {string} projectId
+   * @param {{ kind: 'host' | 'client', value: string, count: number }[]} entries
+   * @param {number} [now=Date.now()]
+   * @returns {void}
+   */
+  noteBlocked(projectId, entries, now = Date.now()) {
+    const project = this.projects.get(projectId);
+    if (!project || !entries.length) return;
+    project.noteBlocked(entries, now);
+  }
+
+  /**
+   * Stores new job runs; ones already stored are skipped.
+   *
+   * @param {string} projectId
+   * @param {Object[]} records
+   * @returns {Object[]}
+   */
+  appendJobs(projectId, records) {
+    const project = this.projects.get(projectId);
+    return project && records.length ? project.appendJobs(records) : [];
+  }
+
+  /**
+   * @param {string} projectId
+   * @param {{ after?: number, before?: number, limit: number }} query
+   * @returns {{ jobs: Object[], hasMore: boolean }|null}
+   */
+  listJobs(projectId, query) {
+    const project = this.projects.get(projectId);
+    return project ? project.listJobs(query) : null;
+  }
+
+  /**
+   * @param {string} projectId
+   * @param {string} jobRunId
+   * @returns {Object|null}
+   */
+  jobDetail(projectId, jobRunId) {
+    const project = this.projects.get(projectId);
+    return project ? project.jobDetail(jobRunId) : null;
+  }
+
+  /**
+   * @param {string} projectId
+   * @returns {void}
+   */
+  clearJobs(projectId) {
+    const project = this.projects.get(projectId);
+    if (project) project.clearJobs();
+  }
+
+  /**
+   * Stores new log lines; ones already stored are skipped.
+   *
+   * @param {string} projectId
+   * @param {Object[]} records
+   * @returns {Object[]}
+   */
+  appendLogs(projectId, records) {
+    const project = this.projects.get(projectId);
+    return project && records.length ? project.appendLogs(records) : [];
+  }
+
+  /**
+   * @param {string} projectId
+   * @param {{ after?: number, before?: number, limit: number }} query
+   * @returns {{ logs: Object[], hasMore: boolean }|null}
+   */
+  listLogs(projectId, query) {
+    const project = this.projects.get(projectId);
+    return project ? project.listLogs(query) : null;
+  }
+
+  /**
+   * @param {string} projectId
+   * @param {string} logId
+   * @returns {Object|null}
+   */
+  logDetail(projectId, logId) {
+    const project = this.projects.get(projectId);
+    return project ? project.logDetail(logId) : null;
+  }
+
+  /**
+   * @param {string} projectId
+   * @returns {void}
+   */
+  clearLogs(projectId) {
+    const project = this.projects.get(projectId);
+    if (project) project.clearLogs();
+  }
+
+  /**
+   * Adds process samples of a project's process (kept in memory, for the Runtime page).
+   *
+   * @param {string} projectId
+   * @param {Object} instance - Validated process details
+   * @param {Object[]} samples - Validated samples
+   * @param {number} [now=Date.now()]
+   * @returns {void}
+   */
+  addSamples(projectId, instance, samples, now = Date.now()) {
+    const project = this.projects.get(projectId);
+    if (project) project.addSamples(instance, samples, now);
+  }
+
+  /**
+   * A project's recent processes with their samples.
+   *
+   * @param {string} projectId
+   * @param {number} [now=Date.now()]
+   * @returns {{ processes: Object[] }|null} null for an unknown project
+   */
+  runtime(projectId, now = Date.now()) {
+    const project = this.projects.get(projectId);
+    return project ? project.runtimeHistory(now) : null;
+  }
+
+  /**
+   * Stores new requests. Requests already stored (same id) are skipped, so retried batches are harmless.
+   *
+   * @param {string} projectId
+   * @param {Object[]} records - Validated records
+   * @returns {Object[]} Summaries of the stored requests
+   */
+  append(projectId, records) {
+    const project = this.projects.get(projectId);
+    return project ? project.append(records) : [];
+  }
+
+  /**
+   * Summaries, newest first; oldest first when `after` is given (for polling new requests).
+   *
+   * @param {string} projectId
+   * @param {{ after?: number, before?: number, limit: number }} query
+   * @returns {{ requests: Object[], hasMore: boolean } | null} null for an unknown project
+   */
+  list(projectId, query) {
+    const project = this.projects.get(projectId);
+    return project ? project.list(query) : null;
+  }
+
+  /**
+   * The full detail of one request.
+   *
+   * @param {string} projectId
+   * @param {string} requestId
+   * @returns {Object|null} null when it is no longer stored
+   */
+  detail(projectId, requestId) {
+    const project = this.projects.get(projectId);
+    return project ? project.detail(requestId) : null;
+  }
+
+  /**
+   * Deletes a project's requests.
+   *
+   * @param {string} projectId
+   * @returns {boolean} False for an unknown project
+   */
+  clear(projectId) {
+    const project = this.projects.get(projectId);
+    if (!project) return false;
+    project.clear();
+    return true;
+  }
+
+  /**
+   * Stores new exceptions; ones already stored (same id) are skipped.
+   *
+   * @param {string} projectId
+   * @param {Object[]} records - Validated exceptions
+   * @returns {Object[]} Summaries of the stored exceptions
+   */
+  appendExceptions(projectId, records) {
+    const project = this.projects.get(projectId);
+    return project && records.length ? project.appendExceptions(records) : [];
+  }
+
+  /**
+   * Exception summaries, newest first; oldest first when `after` is given.
+   *
+   * @param {string} projectId
+   * @param {{ after?: number, before?: number, limit: number }} query
+   * @returns {{ exceptions: Object[], hasMore: boolean } | null} null for an unknown project
+   */
+  listExceptions(projectId, query) {
+    const project = this.projects.get(projectId);
+    return project ? project.listExceptions(query) : null;
+  }
+
+  /**
+   * The full detail of one exception.
+   *
+   * @param {string} projectId
+   * @param {string} exceptionId
+   * @returns {Object|null} null when it is no longer stored
+   */
+  exceptionDetail(projectId, exceptionId) {
+    const project = this.projects.get(projectId);
+    return project ? project.exceptionDetail(exceptionId) : null;
+  }
+
+  /**
+   * Deletes a project's exceptions.
+   *
+   * @param {string} projectId
+   * @returns {boolean} False for an unknown project
+   */
+  clearExceptions(projectId) {
+    const project = this.projects.get(projectId);
+    if (!project) return false;
+    project.clearExceptions();
+    return true;
+  }
+
+  /**
+   * The project's exception state (resolved fingerprints) and its version.
+   *
+   * @param {string} projectId
+   * @returns {{ state: Object, version: number } | null} null for an unknown project
+   */
+  exceptionState(projectId) {
+    const project = this.projects.get(projectId);
+    return project ? project.exceptionState.read() : null;
+  }
+
+  /**
+   * Saves the project's exception state if `version` is the current one.
+   *
+   * @param {string} projectId
+   * @param {Object} state - Validated state (see ExceptionState.validate())
+   * @param {number} version - The version the dashboard read
+   * @returns {{ saved: boolean, state: Object, version: number } | null} null for an unknown project
+   */
+  saveExceptionState(projectId, state, version) {
+    const project = this.projects.get(projectId);
+    return project ? project.exceptionState.write(state, version) : null;
+  }
+
+  /**
+   * Forgets a project and deletes its folder.
+   *
+   * @param {string} projectId
+   * @returns {boolean} False for an unknown project
+   */
+  remove(projectId) {
+    const project = this.projects.get(projectId);
+    if (!project) return false;
+    project.remove();
+    this.projects.delete(projectId);
+    return true;
+  }
+
+  /**
+   * Projects with their request counts and running processes, most recently active first.
+   *
+   * @param {number} [now=Date.now()]
+   * @returns {Object[]}
+   */
+  listProjects(now = Date.now()) {
+    const result = [];
+    for (const project of this.projects.values()) {
+      result.push(project.describe(now));
+    }
+    return result.sort((a, b) => Number(b.online) - Number(a.online) || b.lastSeenAt - a.lastSeenAt);
+  }
+
+  /**
+   * Writes changed project metadata (last-seen times) to disk.
+   *
+   * @returns {void}
+   */
+  persist() {
+    for (const project of this.projects.values()) project.persist();
+  }
+
+  _limits() {
+    return {
+      maxRequests: this.maxRequests,
+      segmentSize: this.segmentSize,
+      maxExceptions: this.maxExceptions,
+      exceptionSegmentSize: this.exceptionSegmentSize,
+      maxJobs: DEFAULT_MAX_JOBS,
+      maxLogs: DEFAULT_MAX_LOGS
+    };
+  }
+}

package/src/instrumentation/console.js

@@ -0,0 +1,83 @@
+/**
+ * Console lines: console.log/info/warn/error/debug, and console.trace. A line written while a
+ * request (or a job run) is handled goes on its timeline; every line goes to the Logs page (see
+ * LogCapture). What the console prints is untouched: the original method runs with the same
+ * arguments.
+ *
+ * console.trace is not wrapped: the stack it prints would include the wrapper. It writes through
+ * console.error, so it is recognized there instead (a single "Trace: ..." string, written by
+ * console.trace itself) and recorded at debug level, as the contract maps it.
+ *
+ * PeekInside's own lines ("[peekinside] ...") are never recorded.
+ */
+
+import { format } from 'util';
+import { LOG_PREFIX } from '../config.js';
+import { LogCapture } from '../logs/log-capture.js';
+import { Timeline } from '../timeline/timeline.js';
+
+const LEVELS = { log: 'info', info: 'info', warn: 'warn', error: 'error', debug: 'debug' };
+const TRACE = /^Trace(?:: |\n|$)/;
+let installed = false;
+
+/**
+ * Wraps the global console's methods, once per process.
+ */
+export class ConsoleInstrumentation {
+  /**
+   * @returns {void}
+   */
+  static install() {
+    if (installed) return;
+    installed = true;
+    for (const [method, level] of Object.entries(LEVELS)) {
+      const original = console[method];
+      if (typeof original !== 'function') continue;
+      const wrapper = {
+        [method](...args) {
+          record(level, args, wrapper);
+          return original.apply(this, arguments);
+        }
+      }[method];
+      console[method] = wrapper;
+    }
+  }
+}
+
+function record(level, args, wrapper) {
+  try {
+    if (typeof args[0] === 'string' && args[0].startsWith(LOG_PREFIX)) return;
+    const timeline = Timeline.current();
+    const onTimeline = Boolean(timeline && timeline.settings.captureLogs);
+    let line = null;
+    if (level === 'error' && args.length === 1 && typeof args[0] === 'string' && TRACE.test(args[0]) && calledByTrace(wrapper)) {
+      const text = args[0];
+      const frames = text.indexOf('\n    at ');
+      const head = frames === -1 ? text : text.slice(0, frames);
+      line = { level: 'debug', message: head.startsWith('Trace: ') ? head.slice(7) : '' };
+    }
+    const lineLevel = line ? line.level : level;
+    const toLogs = LogCapture.wants(lineLevel);
+    if (!onTimeline && !toLogs) return;
+    if (!line) line = { level, message: format(...args) };
+    if (onTimeline) timeline.log(line.level, line.message);
+    if (toLogs) LogCapture.record(line.level, line.message, { source: 'console' });
+  } catch (error) {
+    // A value whose formatting throws is printed (or not) by the console itself.
+  }
+}
+
+// Whether console.error was called by console.trace: its frame is right below the wrapper's.
+function calledByTrace(wrapper) {
+  const saved = Error.stackTraceLimit;
+  const holder = {};
+  try {
+    Error.stackTraceLimit = 2;
+    Error.captureStackTrace(holder, wrapper);
+    return /\bat (?:console\.|Console\.)?trace\b/.test(String(holder.stack));
+  } catch (error) {
+    return false;
+  } finally {
+    Error.stackTraceLimit = saved;
+  }
+}

package/src/instrumentation/http.js

@@ -0,0 +1,146 @@
+/**
+ * Outgoing HTTP calls on the request's timeline: node:http/https requests (and every client built
+ * on them: axios, got, node-fetch...) and fetch (undici, built into Node.js).
+ *
+ * Both are observed through diagnostics_channel, which Node.js publishes to without any patching:
+ *
+ * - node:http: 'http.client.request.start' (in the app's context, when the request is created)
+ *   starts the span; 'http.client.response.finish' (when the response's headers arrive) gives the
+ *   status; the span ends when the response's body has been read. Errors are seen by wrapping the
+ *   request's and the response's emit(), never by adding an 'error' listener, which would change
+ *   what an unhandled error does.
+ * - fetch: 'undici:request:create' starts the span, 'undici:request:headers' gives the status,
+ *   'undici:request:trailers' (the body is complete) or 'undici:request:error' ends it.
+ *
+ * Channels that don't exist in the running Node.js version are simply never published to.
+ */
+
+import dc from 'diagnostics_channel';
+import { Timeline } from '../timeline/timeline.js';
+
+let installed = false;
+// node:http requests and undici requests -> their span and status.
+const calls = new WeakMap();
+
+/**
+ * Subscribes to the channels, once per process.
+ */
+export class HttpInstrumentation {
+  /**
+   * @returns {void}
+   */
+  static install() {
+    if (installed || typeof dc.subscribe !== 'function') return;
+    installed = true;
+    subscribe('http.client.request.start', onRequestStart);
+    subscribe('http.client.response.finish', onResponse);
+    subscribe('undici:request:create', onFetchCreate);
+    subscribe('undici:request:headers', onFetchHeaders);
+    subscribe('undici:request:trailers', onFetchTrailers);
+    subscribe('undici:request:error', onFetchError);
+  }
+}
+
+function subscribe(name, handler) {
+  try {
+    dc.subscribe(name, (message) => {
+      try {
+        handler(message);
+      } catch (error) {
+        // An observer must never break the app's call.
+      }
+    });
+  } catch (error) {
+    // Channel unavailable in this Node.js version.
+  }
+}
+
+// --- node:http ---------------------------------------------------------------------------------
+
+function onRequestStart({ request }) {
+  const timeline = Timeline.current();
+  if (!timeline || !request || calls.has(request)) return;
+  // `host` is the hostname option; strip a port only from "name:port", never from an IPv6 address.
+  const raw = String(request.host || '');
+  const host = raw.split(':').length === 2 ? raw.slice(0, raw.indexOf(':')) : raw.replace(/^\[|\]$/g, '');
+  const span = timeline.http({ client: 'http', method: request.method, url: urlOf(request), host });
+  if (!span) return;
+  const call = { span, status: null };
+  calls.set(request, call);
+  watch(request, (event, value) => {
+    if (event === 'error') span.end({ status: call.status, error: value });
+    else if (event === 'close' && call.status === null) span.end({ error: 'socket closed before a response' });
+  });
+}
+
+function onResponse({ request, response }) {
+  const call = request && calls.get(request);
+  if (!call) return;
+  call.status = response && Number.isInteger(response.statusCode) ? response.statusCode : null;
+  if (!response || typeof response.emit !== 'function') {
+    call.span.end({ status: call.status });
+    return;
+  }
+  watch(response, (event, value) => {
+    if (event === 'end' || event === 'close') call.span.end({ status: call.status });
+    else if (event === 'error') call.span.end({ status: call.status, error: value });
+    else if (event === 'aborted') call.span.end({ status: call.status, error: 'response aborted' });
+  });
+}
+
+// Sees an emitter's events without becoming a listener.
+function watch(emitter, observe) {
+  if (!emitter || typeof emitter.emit !== 'function') return;
+  const emit = emitter.emit;
+  emitter.emit = function (event, value) {
+    try {
+      observe(event, value);
+    } catch (error) {
+      // Ignore.
+    }
+    return emit.apply(this, arguments);
+  };
+}
+
+function urlOf(request) {
+  const path = String(request.path || '/');
+  if (/^[a-z][a-z0-9+.-]*:\/\//i.test(path)) return path; // absolute-form, through a proxy
+  let host;
+  try {
+    host = request.getHeader('host');
+  } catch (error) {
+    host = undefined;
+  }
+  return `${request.protocol || 'http:'}//${host || request.host || 'localhost'}${path}`;
+}
+
+// --- fetch (undici) ---------------------------------------------------------------------------
+
+function onFetchCreate({ request }) {
+  const timeline = Timeline.current();
+  if (!timeline || !request) return;
+  const origin = String(request.origin || '');
+  let host = '';
+  try {
+    host = new URL(origin).hostname;
+  } catch (error) {
+    host = '';
+  }
+  const span = timeline.http({ client: 'fetch', method: request.method, url: `${origin}${request.path || ''}`, host });
+  if (span) calls.set(request, { span, status: null });
+}
+
+function onFetchHeaders({ request, response }) {
+  const call = request && calls.get(request);
+  if (call && response) call.status = Number.isInteger(response.statusCode) ? response.statusCode : null;
+}
+
+function onFetchTrailers({ request }) {
+  const call = request && calls.get(request);
+  if (call) call.span.end({ status: call.status });
+}
+
+function onFetchError({ request, error }) {
+  const call = request && calls.get(request);
+  if (call) call.span.end({ status: call.status, error });
+}