peekinside 0.0.0-stage → 1.2.0

package/src/instrumentation/index.js

@@ -0,0 +1,234 @@
+/**
+ * Turns the timeline's instrumentations on.
+ *
+ * Database drivers are found from the app's project folder: each one installed there (pg, mysql2,
+ * mongodb, the mongodb copy Mongoose carries, @prisma/client) is loaded and its prototypes are
+ * patched. Loading a driver first is harmless: the app's own require() or import gets the same
+ * module instance (ES modules importing a CommonJS driver share it too), already patched. A
+ * driver that isn't installed is skipped; one that fails to load is left alone.
+ *
+ * Apps whose drivers can't be found that way (bundled apps, a monorepo package with its own copy)
+ * pass them to PeekInside#instrument().
+ *
+ * Outgoing HTTP and console capture are process-wide subscriptions; they record only for code
+ * running inside a request (log lines go to the Logs page from anywhere). Loggers (pino, winston)
+ * and job libraries (node-cron, cron, BullMQ) are found and patched like the drivers.
+ */
+
+import { createRequire } from 'module';
+import fs from 'fs';
+import path from 'path';
+import { ConsoleInstrumentation } from './console.js';
+import { HttpInstrumentation } from './http.js';
+import { JobInstrumentation } from './jobs.js';
+import { LoggerInstrumentation } from './loggers.js';
+import { MongoInstrumentation } from './mongodb.js';
+import { Mysql2Instrumentation } from './mysql2.js';
+import { PgInstrumentation } from './pg.js';
+import { PrismaInstrumentation } from './prisma.js';
+
+const requireHere = createRequire(import.meta.url);
+const scanned = new Set();
+const scannedLoggers = new Set();
+const scannedJobs = new Set();
+
+/**
+ * Installs instrumentations.
+ */
+export class Instrumentation {
+  /**
+   * Turns on what a recording instance captures. Safe to call again: each piece installs once.
+   *
+   * @param {Object} options
+   * @param {string} options.root - The project folder drivers are resolved from
+   * @param {boolean} options.queries - Database drivers
+   * @param {boolean} options.outgoing - node:http/https and fetch
+   * @param {boolean} options.logs - The console, pino and winston
+   * @param {boolean} [options.jobs=false] - node-cron, cron and BullMQ
+   * @returns {void}
+   */
+  static install({ root, queries, outgoing, logs, jobs = false }) {
+    try {
+      if (outgoing) HttpInstrumentation.install();
+      if (logs) ConsoleInstrumentation.install();
+      if (queries) Instrumentation.patchInstalled(root);
+      if (logs) Instrumentation.patchLoggers(root);
+      if (jobs) Instrumentation.patchJobs(root);
+    } catch (error) {
+      // Monitoring must never break the host application.
+    }
+  }
+
+  /**
+   * Patches the loggers installed in a project (pino, winston), once per project folder.
+   *
+   * @param {string} root
+   * @returns {string[]} Names of the loggers patched
+   */
+  static patchLoggers(root) {
+    const patched = [];
+    const fromRoot = projectRequire(root, scannedLoggers);
+    if (!fromRoot) return patched;
+    if (patchWith(fromRoot, 'pino', LoggerInstrumentation.pino)) patched.push('pino');
+    if (patchWith(fromRoot, 'winston', LoggerInstrumentation.winston)) patched.push('winston');
+    return patched;
+  }
+
+  /**
+   * Patches the job libraries installed in a project (node-cron, cron, BullMQ), once per project
+   * folder. Their ES module builds are patched as soon as they are imported.
+   *
+   * @param {string} root
+   * @returns {string[]} Names of the libraries patched (CommonJS builds)
+   */
+  static patchJobs(root) {
+    const patched = [];
+    const fromRoot = projectRequire(root, scannedJobs);
+    if (!fromRoot) return patched;
+    for (const [name, patch] of [['node-cron', JobInstrumentation.nodeCron], ['cron', JobInstrumentation.cron], ['bullmq', JobInstrumentation.bullmq]]) {
+      const main = resolve(fromRoot, name);
+      if (!main) continue;
+      if (patchWith(fromRoot, name, patch)) patched.push(name);
+      const directory = packageDirectory(main, name);
+      if (directory) void JobInstrumentation.patchModuleBuild(directory, patch);
+    }
+    return patched;
+  }
+
+  /**
+   * Patches the database drivers installed in a project, once per project folder.
+   *
+   * @param {string} root
+   * @returns {string[]} Names of the drivers patched
+   */
+  static patchInstalled(root) {
+    const patched = [];
+    if (!root || scanned.has(root)) return patched;
+    scanned.add(root);
+    let fromRoot;
+    try {
+      fromRoot = createRequire(path.join(root, 'package.json'));
+    } catch (error) {
+      return patched;
+    }
+
+    if (patchWith(fromRoot, 'pg', PgInstrumentation.patch)) patched.push('pg');
+    if (patchWith(fromRoot, 'mysql2', Mysql2Instrumentation.patch)) patched.push('mysql2');
+    if (patchMongo(fromRoot)) patched.push('mongodb');
+    // Mongoose depends on an exact driver version, often installed under its own node_modules.
+    const mongoose = resolve(fromRoot, 'mongoose');
+    if (mongoose && patchMongo(createRequire(mongoose))) patched.push('mongoose');
+    if (patchWith(fromRoot, '@prisma/client', PrismaInstrumentation.patch)) patched.push('@prisma/client');
+    return patched;
+  }
+
+  /**
+   * Patches modules given by the app: pg, mysql2, mysql2/promise, mongodb, mongoose, a PrismaClient
+   * class or instance, or the @prisma/client module. Anything else is ignored.
+   *
+   * @param {Array} modules
+   * @returns {number} How many were recognized
+   */
+  static instrument(modules) {
+    let recognized = 0;
+    for (const value of modules) {
+      try {
+        if (patchModule(value) || (value && typeof value === 'object' && patchModule(value.default))) recognized++;
+      } catch (error) {
+        // An unknown value is ignored.
+      }
+    }
+    return recognized;
+  }
+}
+
+function patchModule(value) {
+  if (!value || (typeof value !== 'object' && typeof value !== 'function')) return false;
+  if (LoggerInstrumentation.pino(value) || LoggerInstrumentation.winston(value)) return true;
+  if (JobInstrumentation.nodeCron(value) || JobInstrumentation.cron(value) || JobInstrumentation.bullmq(value)) return true;
+  if (PgInstrumentation.patch(value)) return true;
+  if (Mysql2Instrumentation.patch(value)) return true;
+  if (PrismaInstrumentation.patch(value)) return true;
+  if (typeof value.MongoClient === 'function') return patchMongoModule(value);
+  // Mongoose exposes the driver it uses.
+  if (value.mongo && typeof value.mongo.MongoClient === 'function') return patchMongoModule(value.mongo);
+  // mysql2/promise runs on the core module next to it.
+  const file = fileOf(value);
+  if (file && /mysql2[\\/]promise\.js$/.test(file)) {
+    return Mysql2Instrumentation.patch(createRequire(file)('./index.js'));
+  }
+  return false;
+}
+
+// require() from a project folder, the first time the folder is scanned for `scanned`.
+function projectRequire(root, scanned) {
+  if (!root || scanned.has(root)) return null;
+  scanned.add(root);
+  try {
+    return createRequire(path.join(root, 'package.json'));
+  } catch (error) {
+    return null;
+  }
+}
+
+function patchWith(fromRoot, name, patch) {
+  if (!resolve(fromRoot, name)) return false;
+  try {
+    return patch(fromRoot(name));
+  } catch (error) {
+    return false;
+  }
+}
+
+// The driver's Connection is internal: it is loaded from the package's own files, which gives the
+// same module instance the driver uses.
+function patchMongo(fromSomewhere) {
+  const main = resolve(fromSomewhere, 'mongodb');
+  return main ? patchMongoDirectory(packageDirectory(main, 'mongodb')) : false;
+}
+
+function patchMongoModule(mongodb) {
+  const file = fileOf(mongodb);
+  return file ? patchMongoDirectory(packageDirectory(file, 'mongodb')) : false;
+}
+
+function patchMongoDirectory(directory) {
+  if (!directory) return false;
+  try {
+    const load = createRequire(path.join(directory, 'package.json'));
+    const { Connection } = load(path.join(directory, 'lib', 'cmap', 'connection.js'));
+    let ConnectionPool;
+    try {
+      ({ ConnectionPool } = load(path.join(directory, 'lib', 'cmap', 'connection_pool.js')));
+    } catch (error) {
+      ConnectionPool = undefined;
+    }
+    return MongoInstrumentation.patch({ Connection, ConnectionPool });
+  } catch (error) {
+    return false;
+  }
+}
+
+function resolve(fromSomewhere, name) {
+  try {
+    return fromSomewhere.resolve(name);
+  } catch (error) {
+    return null;
+  }
+}
+
+// The folder of the package `name` that contains `file`.
+function packageDirectory(file, name) {
+  for (let directory = path.dirname(file); ; directory = path.dirname(directory)) {
+    if (path.basename(directory) === name && fs.existsSync(path.join(directory, 'package.json'))) return directory;
+    if (path.dirname(directory) === directory) return null;
+  }
+}
+
+// The file a module object was loaded from, found in the CommonJS cache.
+function fileOf(exports) {
+  for (const module of Object.values(requireHere.cache)) {
+    if (module && module.exports === exports) return module.filename;
+  }
+  return null;
+}

package/src/instrumentation/jobs.js

@@ -0,0 +1,186 @@
+/**
+ * Job runs from the common job libraries, without code changes: node-cron and cron (scheduled
+ * tasks, with their cron expression, so PeekInside can tell when one misses its run) and BullMQ
+ * (queue workers, with the queue, the attempt and the job's data, and the schedule of repeatable
+ * jobs). Each run goes through JobRunner, exactly as PeekInside.job() would run it.
+ *
+ * Libraries that ship separate CommonJS and ES module builds (node-cron, BullMQ) are patched in
+ * both: the CommonJS one right away, the ES module one as soon as it is imported here, which is
+ * before any job can run.
+ */
+
+import fs from 'fs';
+import path from 'path';
+import { pathToFileURL } from 'url';
+import { JobRunner } from '../jobs/job-runner.js';
+
+const kWrapped = Symbol('peekInside.job');
+// node-cron tasks created through named imports are found by looking again now and then.
+const RESCAN_MS = 30_000;
+
+/**
+ * Patches job libraries.
+ */
+export class JobInstrumentation {
+  /**
+   * node-cron: tasks scheduled through the module (and those already scheduled).
+   *
+   * @param {*} nodeCron - The module, or its default export
+   * @returns {boolean}
+   */
+  static nodeCron(nodeCron) {
+    const api = nodeCron && typeof nodeCron.schedule === 'function' ? nodeCron : nodeCron && nodeCron.default;
+    if (!api || typeof api.schedule !== 'function' || typeof api.getTasks !== 'function') return false;
+    if (api[kWrapped]) return true;
+    for (const method of ['schedule', 'createTask']) {
+      const original = api[method];
+      if (typeof original !== 'function') continue;
+      try {
+        api[method] = function () {
+          const task = original.apply(this, arguments);
+          wrapCronTask(task);
+          return task;
+        };
+      } catch (error) {
+        // A frozen module namespace: the rescan below still finds its tasks.
+      }
+    }
+    wrapAll(api);
+    const timer = setInterval(() => wrapAll(api), RESCAN_MS);
+    if (typeof timer.unref === 'function') timer.unref();
+    try {
+      Object.defineProperty(api, kWrapped, { value: true });
+    } catch (error) {
+      // Marked or not, patching twice is avoided per task.
+    }
+    return true;
+  }
+
+  /**
+   * cron: every CronJob's callbacks, wrapped when the job first fires.
+   *
+   * @param {*} cron - The module
+   * @returns {boolean}
+   */
+  static cron(cron) {
+    const CronJob = cron && (cron.CronJob || (cron.default && cron.default.CronJob));
+    const proto = typeof CronJob === 'function' ? CronJob.prototype : null;
+    if (!proto || typeof proto.fireOnTick !== 'function') return false;
+    if (proto[kWrapped]) return true;
+    const original = proto.fireOnTick;
+    proto.fireOnTick = function () {
+      if (Array.isArray(this._callbacks)) this._callbacks = this._callbacks.map((callback) => wrapCronJobCallback(this, callback));
+      return original.apply(this, arguments);
+    };
+    Object.defineProperty(proto, kWrapped, { value: true });
+    return true;
+  }
+
+  /**
+   * BullMQ: every job a Worker processes.
+   *
+   * @param {*} bullmq - The module
+   * @returns {boolean}
+   */
+  static bullmq(bullmq) {
+    const Worker = bullmq && (bullmq.Worker || (bullmq.default && bullmq.default.Worker));
+    const proto = typeof Worker === 'function' ? Worker.prototype : null;
+    if (!proto || typeof proto.callProcessJob !== 'function') return false;
+    if (proto[kWrapped]) return true;
+    const original = proto.callProcessJob;
+    proto.callProcessJob = function (job) {
+      const args = arguments;
+      const repeat = job && job.opts && job.opts.repeat;
+      const schedule = repeat ? repeat.pattern || repeat.cron || (Number.isFinite(repeat.every) ? `@every ${repeat.every}ms` : undefined) : undefined;
+      return JobRunner.run(
+        job && job.name && job.name !== '__default__' ? job.name : this.name,
+        () => original.apply(this, args),
+        {
+          queue: this.name,
+          attempt: job && Number.isInteger(job.attemptsMade) ? job.attemptsMade + 1 : 1,
+          data: job ? job.data : undefined,
+          schedule,
+          timezone: repeat && typeof repeat.tz === 'string' ? repeat.tz : undefined,
+          trigger: schedule ? 'cron' : 'queue'
+        }
+      );
+    };
+    Object.defineProperty(proto, kWrapped, { value: true });
+    return true;
+  }
+
+  /**
+   * Patches the ES module build of a package installed in the project, when it has one.
+   *
+   * @param {string} packageDir - The package's folder
+   * @param {(module: *) => boolean} patch
+   * @returns {Promise<boolean>}
+   */
+  static async patchModuleBuild(packageDir, patch) {
+    const entry = esmEntry(packageDir);
+    if (!entry) return false;
+    try {
+      return patch(await import(pathToFileURL(entry).href));
+    } catch (error) {
+      return false;
+    }
+  }
+}
+
+function wrapAll(api) {
+  try {
+    const tasks = api.getTasks();
+    for (const task of tasks instanceof Map ? tasks.values() : Object.values(tasks || {})) wrapCronTask(task);
+  } catch (error) {
+    // The rescan tries again.
+  }
+}
+
+// A node-cron task runs its function through runner.onMatch.
+function wrapCronTask(task) {
+  const runner = task && task.runner;
+  if (!runner || typeof runner.onMatch !== 'function' || runner.onMatch[kWrapped]) return;
+  const original = runner.onMatch;
+  const expression = typeof task.cronExpression === 'string' ? task.cronExpression : null;
+  // Without a name of its own, a task is named after a random id: its schedule says more.
+  const name = task.name && task.name !== task.id ? task.name : `cron ${expression || 'task'}`;
+  const wrapped = function () {
+    const args = arguments;
+    return JobRunner.run(name, () => original.apply(this, args), { schedule: expression || undefined, timezone: task.timezone, trigger: 'cron' });
+  };
+  wrapped[kWrapped] = true;
+  runner.onMatch = wrapped;
+}
+
+function wrapCronJobCallback(job, callback) {
+  if (typeof callback !== 'function' || callback[kWrapped]) return callback;
+  const source = job.cronTime && typeof job.cronTime.source === 'string' ? job.cronTime.source : null;
+  const wrapped = function () {
+    const args = arguments;
+    return JobRunner.run(job.name || `cron ${source || 'job'}`, () => callback.apply(this, args), {
+      schedule: source || undefined,
+      timezone: job.cronTime && typeof job.cronTime.timeZone === 'string' ? job.cronTime.timeZone : undefined,
+      trigger: 'cron'
+    });
+  };
+  wrapped[kWrapped] = true;
+  return wrapped;
+}
+
+// The file an `import` of the package loads, when it differs from what require() loads.
+function esmEntry(packageDir) {
+  try {
+    const pkg = JSON.parse(fs.readFileSync(path.join(packageDir, 'package.json'), 'utf8'));
+    const root = pkg.exports && (pkg.exports['.'] || pkg.exports);
+    let target = null;
+    if (root && typeof root === 'object' && root.import) {
+      target = typeof root.import === 'string' ? root.import : root.import.default || root.import.node || null;
+    }
+    if (!target && typeof pkg.module === 'string') target = pkg.module;
+    if (!target || typeof target !== 'string') return null;
+    const file = path.resolve(packageDir, target);
+    return fs.existsSync(file) ? file : null;
+  } catch (error) {
+    return null;
+  }
+}

package/src/instrumentation/loggers.js

@@ -0,0 +1,135 @@
+/**
+ * Lines written with pino and winston, like console lines: on the timeline of the request (or job
+ * run) they belong to, and on the Logs page. Only lines the logger actually writes are recorded
+ * (a level it has turned off is skipped), and what the logger writes is untouched.
+ *
+ * Both are patched on a prototype every logger shares, so loggers created before PeekInside, child
+ * loggers and loggers imported as ES modules are all covered.
+ */
+
+import { LogCapture } from '../logs/log-capture.js';
+import { Timeline } from '../timeline/timeline.js';
+
+const kPatched = Symbol('peekInside.logger');
+const WINSTON_LEVEL = Symbol.for('level');
+const PINO_LEVELS = { trace: 'debug', debug: 'debug', info: 'info', warn: 'warn', error: 'error', fatal: 'error' };
+const WINSTON_LEVELS = { error: 'error', warn: 'warn', info: 'info', http: 'info', verbose: 'debug', debug: 'debug', silly: 'debug' };
+// Keys of a winston `info` that aren't the line's own data.
+const WINSTON_OWN = new Set(['level', 'message', 'timestamp', 'splat']);
+
+/**
+ * Patches logging libraries.
+ */
+export class LoggerInstrumentation {
+  /**
+   * Patches pino: every logger writes through one prototype method.
+   *
+   * @param {*} pino - The pino module (its factory)
+   * @returns {boolean} Whether it is (now) patched
+   */
+  static pino(pino) {
+    const factory = typeof pino === 'function' ? pino : pino && typeof pino.default === 'function' ? pino.default : null;
+    const writeSym = factory && factory.symbols && factory.symbols.writeSym;
+    if (!writeSym) return false;
+    // Each logger sits on an object of its own, over the prototype that holds the write method.
+    let proto;
+    try {
+      proto = Object.getPrototypeOf(factory({ enabled: false }));
+      while (proto && !Object.prototype.hasOwnProperty.call(proto, writeSym)) proto = Object.getPrototypeOf(proto);
+    } catch (error) {
+      return false;
+    }
+    if (!proto || typeof proto[writeSym] !== 'function') return false;
+    if (proto[kPatched]) return true;
+    const original = proto[writeSym];
+    proto[writeSym] = function (obj, msg, num) {
+      try {
+        const label = this.levels && this.levels.labels ? this.levels.labels[num] : undefined;
+        capture(PINO_LEVELS[label] || (num >= 50 ? 'error' : num >= 40 ? 'warn' : num >= 30 ? 'info' : 'debug'), pinoLine(obj, msg), pinoFields(obj), 'pino');
+      } catch (error) {
+        // The line is written all the same.
+      }
+      return original.call(this, obj, msg, num);
+    };
+    Object.defineProperty(proto, kPatched, { value: true });
+    return true;
+  }
+
+  /**
+   * Patches winston: every logger is a Logger stream whose _transform receives each line.
+   *
+   * @param {*} winston - The winston module
+   * @returns {boolean} Whether it is (now) patched
+   */
+  static winston(winston) {
+    if (!winston || typeof winston.createLogger !== 'function') return false;
+    let proto;
+    try {
+      const probe = winston.createLogger({ silent: true });
+      proto = Object.getPrototypeOf(Object.getPrototypeOf(probe));
+      probe.close();
+    } catch (error) {
+      return false;
+    }
+    if (!proto || typeof proto._transform !== 'function') return false;
+    if (proto[kPatched]) return true;
+    const original = proto._transform;
+    proto._transform = function (info, encoding, callback) {
+      try {
+        const level = info && (info[WINSTON_LEVEL] || info.level);
+        if (!this.silent && typeof level === 'string' && this.isLevelEnabled(level)) {
+          capture(WINSTON_LEVELS[level] || 'info', winstonLine(info), winstonFields(info), 'winston');
+        }
+      } catch (error) {
+        // The line is written all the same.
+      }
+      return original.call(this, info, encoding, callback);
+    };
+    Object.defineProperty(proto, kPatched, { value: true });
+    return true;
+  }
+}
+
+function capture(level, message, fields, source) {
+  const timeline = Timeline.current();
+  if (timeline && timeline.settings.captureLogs) timeline.log(level, message, source);
+  LogCapture.record(level, message, { source, fields });
+}
+
+function pinoLine(obj, msg) {
+  if (typeof msg === 'string') return msg;
+  if (obj instanceof Error) return obj.message;
+  if (obj && typeof obj === 'object' && typeof obj.msg === 'string') return obj.msg;
+  return msg === undefined ? '' : String(msg);
+}
+
+function pinoFields(obj) {
+  if (!obj || typeof obj !== 'object') return undefined;
+  if (obj instanceof Error) return { err: errorFields(obj) };
+  const fields = {};
+  for (const [key, value] of Object.entries(obj)) {
+    if (key === 'msg') continue;
+    fields[key] = value instanceof Error ? errorFields(value) : value;
+  }
+  return fields;
+}
+
+function winstonLine(info) {
+  const message = info.message;
+  if (typeof message === 'string') return message;
+  if (message instanceof Error) return message.message;
+  return message === undefined ? '' : String(message);
+}
+
+function winstonFields(info) {
+  const fields = {};
+  for (const [key, value] of Object.entries(info)) {
+    if (WINSTON_OWN.has(key)) continue;
+    fields[key] = value instanceof Error ? errorFields(value) : value;
+  }
+  return fields;
+}
+
+function errorFields(error) {
+  return { name: error.name, message: error.message, ...(error.code ? { code: error.code } : {}) };
+}