peekinside 0.0.0-stage → 1.2.1

package/src/logs/log-capture.js

@@ -0,0 +1,143 @@
+/**
+ * Log lines for the Logs page: what the app writes with the console, pino or winston, inside a
+ * request, inside a job run, or anywhere else (startup, timers), each linked to its request or job
+ * run when there is one.
+ *
+ * Lines are rate limited per process (100 a second, bursts of 500) and below the `logLevel` option
+ * are left out, so an app that logs heavily costs a bounded amount. PeekInside's own lines are
+ * never recorded.
+ */
+
+import { randomUUID } from 'crypto';
+import { ExceptionCapture } from '../exceptions/exception-capture.js';
+import { truncate } from '../exceptions/limits.js';
+import { TokenBucket } from '../exceptions/rate-limiter.js';
+import { RequestContext } from '../exceptions/request-context.js';
+import { SafeJson } from '../exceptions/safe-json.js';
+import { LOG_LEVELS, LogRecord } from './log-record.js';
+
+const MAX_MESSAGE = 4000;
+const MAX_FIELDS = 4096;
+const RANK = Object.freeze({ debug: 0, info: 1, warn: 2, error: 3 });
+
+/**
+ * One PeekInside instance's log capture.
+ */
+export class LogCapture {
+  /**
+   * @param {Object} options
+   * @param {'debug'|'info'|'warn'|'error'} [options.level='info'] - The lowest level recorded
+   * @param {() => Object|null} options.sink - The instance's transport
+   * @param {() => boolean} options.isEnabled - Whether the instance records right now
+   * @param {TokenBucket} [options.limiter]
+   */
+  constructor({ level = 'info', sink, isEnabled, limiter = new TokenBucket({ rate: 100, burst: 500 }) }) {
+    this.minRank = RANK[LogCapture.level(level)];
+    this._sink = sink;
+    this._isEnabled = isEnabled;
+    this.limiter = limiter;
+  }
+
+  /**
+   * A valid level name, or "info".
+   *
+   * @param {*} value
+   * @returns {'debug'|'info'|'warn'|'error'}
+   */
+  static level(value) {
+    return LOG_LEVELS.includes(value) ? value : 'info';
+  }
+
+  /**
+   * Whether a line at this level would be recorded by anyone right now: lets the console skip
+   * formatting lines nobody records.
+   *
+   * @param {string} level
+   * @returns {boolean}
+   */
+  static wants(level) {
+    const capture = LogCapture.current();
+    return Boolean(capture && capture.recording && RANK[level] >= capture.minRank);
+  }
+
+  /**
+   * Records a line with the capture of the request or job running now, else the most recently
+   * created instance that records. Never throws.
+   *
+   * @param {'debug'|'info'|'warn'|'error'} level
+   * @param {string} message - As written (formatted)
+   * @param {Object} [options]
+   * @param {'console'|'pino'|'winston'} [options.source='console']
+   * @param {*} [options.fields] - Structured data written with the line (pino, winston)
+   * @returns {string|null} The record's id, or null when nothing was recorded
+   */
+  static record(level, message, { source = 'console', fields } = {}) {
+    try {
+      const capture = LogCapture.current();
+      return capture ? capture.record(level, message, { source, fields, context: RequestContext.current() }) : null;
+    } catch (error) {
+      return null;
+    }
+  }
+
+  // The capture of the instance tracking the code running now, else the newest recording one.
+  static current() {
+    const context = RequestContext.current();
+    const owner = context && context.capture ? context.capture.owner : null;
+    if (owner && owner._logs) return owner._logs;
+    const latest = ExceptionCapture.latest();
+    return latest && latest.owner ? latest.owner._logs || null : null;
+  }
+
+  /**
+   * @type {boolean}
+   */
+  get recording() {
+    const sink = this._sink();
+    return Boolean(sink && !sink.disabled && this._isEnabled());
+  }
+
+  /**
+   * @param {string} level
+   * @param {string} message
+   * @param {Object} options
+   * @param {string} options.source
+   * @param {*} [options.fields]
+   * @param {import('../exceptions/request-context.js').RequestContext|null} options.context
+   * @returns {string|null}
+   */
+  record(level, message, { source, fields, context }) {
+    if (!this.recording || !(RANK[level] >= this.minRank)) return null;
+    // Requests local mode doesn't record (server traffic) take their lines with them.
+    if (context && context.blocked) return null;
+    if (!this.limiter.take()) return null;
+    const sink = this._sink();
+    const record = new LogRecord({
+      id: randomUUID(),
+      occurredAt: Date.now(),
+      environment: sink.environment,
+      release: sink.release || null,
+      level,
+      source,
+      requestId: (context && context.requestId) || null,
+      jobRunId: (context && context.jobRunId) || null,
+      summary: { message: truncate(String(message), MAX_MESSAGE), ...structured(fields) }
+    });
+    sink.enqueueRecord(record);
+    return record.id;
+  }
+}
+
+// Structured data as JSON-safe values within MAX_FIELDS; a preview when larger.
+function structured(fields) {
+  if (fields === undefined || fields === null) return {};
+  if (typeof fields === 'object' && !Array.isArray(fields) && Object.keys(fields).length === 0) return {};
+  const { json, truncated } = SafeJson.stringify(fields, MAX_FIELDS);
+  if (json === undefined) return {};
+  if (truncated) return { fields: { truncated: true, preview: json } };
+  try {
+    return { fields: JSON.parse(json) };
+  } catch (error) {
+    return {};
+  }
+}

package/src/logs/log-record.js

@@ -0,0 +1,74 @@
+/**
+ * One log line, as the Logs page lists it (see the data contract): when, its level, where it came
+ * from (console, pino, winston), the request or job run it belongs to, and the line itself.
+ *
+ * PeekInside Cloud receives the line sealed: only the time, level, source and links are readable.
+ */
+
+export const LOG_LEVELS = Object.freeze(['debug', 'info', 'warn', 'error']);
+
+/**
+ * A recorded log line.
+ */
+export class LogRecord {
+  /**
+   * @param {Object} fields
+   * @param {string} fields.id - UUID
+   * @param {number} fields.occurredAt - ms since the epoch
+   * @param {string} fields.environment
+   * @param {string|null} [fields.release=null]
+   * @param {'debug'|'info'|'warn'|'error'} fields.level
+   * @param {'console'|'pino'|'winston'} fields.source
+   * @param {string|null} [fields.requestId=null] - The request it was written during
+   * @param {string|null} [fields.jobRunId=null] - The job run it was written during
+   * @param {{ message: string, fields?: Object|null }} fields.summary - The line (sealed in Cloud)
+   */
+  constructor({ id, occurredAt, environment, release = null, level, source, requestId = null, jobRunId = null, summary }) {
+    this.id = id;
+    this.occurredAt = occurredAt;
+    this.environment = environment;
+    this.release = release;
+    this.level = level;
+    this.source = source;
+    this.requestId = requestId;
+    this.jobRunId = jobRunId;
+    this.summary = summary;
+  }
+
+  /**
+   * The plain form, as the local hub stores it.
+   *
+   * @returns {Object}
+   */
+  toJSON() {
+    return { ...this };
+  }
+
+  /**
+   * The sealed form, for PeekInside Cloud.
+   *
+   * @param {import('../crypto.js').Sealer} sealer
+   * @returns {Promise<Object>}
+   */
+  async seal(sealer) {
+    const { summary, ...meta } = this.toJSON();
+    return { ...meta, summary: await sealer.seal(JSON.stringify(summary), LogRecord.summaryAad(this.id)) };
+  }
+
+  /**
+   * @param {import('../crypto.js').SyncSealer} sealer
+   * @returns {Object}
+   */
+  sealSync(sealer) {
+    const { summary, ...meta } = this.toJSON();
+    return { ...meta, summary: sealer.seal(JSON.stringify(summary), LogRecord.summaryAad(this.id)) };
+  }
+
+  /**
+   * @param {string} id
+   * @returns {string} Additional authenticated data of a sealed line
+   */
+  static summaryAad(id) {
+    return `log:summary:${id}`;
+  }
+}

package/src/project.js

@@ -0,0 +1,76 @@
+/**
+ * Identifies the running app for the local dashboard, so every Node.js project on the machine
+ * gets its own entry: named after its package.json and told apart by its folder.
+ */
+
+import { createHash } from 'crypto';
+import fs from 'fs';
+import path from 'path';
+
+const MAX_NAME_LENGTH = 80;
+
+/**
+ * Which project the running app is: a stable id, a display name and the project's folder.
+ */
+export class ProjectIdentity {
+  /**
+   * @param {Object} fields
+   * @param {string} fields.id - Stable id, also the project's folder name in the hub: a slug of
+   *   the name plus a hash of the folder, so two apps with the same name stay apart
+   * @param {string} fields.name - Display name
+   * @param {string} fields.root - Project folder
+   */
+  constructor({ id, name, root }) {
+    this.id = id;
+    this.name = name;
+    this.root = root;
+  }
+
+  /**
+   * Identifies the running app.
+   *
+   * @param {string} [explicitName] - Name chosen in the options or PEEKINSIDE_PROJECT; otherwise
+   *   the name in package.json, then the folder name
+   * @param {string} [from] - Folder to look for the project from; by default the entry script's
+   *   (or the working directory, when the entry script is a tool in node_modules)
+   * @returns {ProjectIdentity}
+   */
+  static detect(explicitName, from) {
+    const root = findRoot(from);
+    const name = clean(explicitName) || clean(readPackageName(root)) || clean(path.basename(root)) || 'app';
+    const slug = name.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, '').slice(0, 40).replace(/-+$/, '') || 'app';
+    const hash = createHash('sha256').update(`${root}\0${name}`).digest('hex').slice(0, 8);
+    return new ProjectIdentity({ id: `${slug}-${hash}`, name, root });
+  }
+}
+
+// The folder of the nearest package.json above the entry script. Entry scripts inside
+// node_modules (nodemon, ts-node-dev, next...) say nothing about the app, so the working
+// directory is used for those.
+function findRoot(from) {
+  const entry = process.argv[1];
+  const start = from
+    ? path.resolve(from)
+    : entry && !entry.split(path.sep).includes('node_modules')
+      ? path.dirname(path.resolve(entry))
+      : process.cwd();
+
+  for (let dir = start; ; dir = path.dirname(dir)) {
+    if (fs.existsSync(path.join(dir, 'package.json'))) return dir;
+    if (path.dirname(dir) === dir) return start;
+  }
+}
+
+function readPackageName(dir) {
+  try {
+    const name = JSON.parse(fs.readFileSync(path.join(dir, 'package.json'), 'utf8')).name;
+    return typeof name === 'string' ? name : '';
+  } catch (error) {
+    return '';
+  }
+}
+
+// Names end up in the dashboard and the CLI's output: no control characters, bounded length.
+function clean(value) {
+  return typeof value === 'string' ? value.replace(/[\u0000-\u001f\u007f]/g, '').trim().slice(0, MAX_NAME_LENGTH) : '';
+}

package/src/record.js

@@ -0,0 +1,181 @@
+/**
+ * The stored form of a captured exchange, shared by the local hub and PeekInside Cloud:
+ * a few plain numbers for lists, charts and filters, a small summary for the request list,
+ * and the full detail, which the dashboard only loads when someone opens the request.
+ */
+
+const METHOD = /^[A-Z]{1,16}$/;
+// Exceptions listed in a request's detail, so the inspector can link to them.
+const MAX_EXCEPTION_LINKS = 10;
+
+/**
+ * One captured exchange, in the shape both destinations store.
+ *
+ * The local hub receives the plain form (toJSON()). PeekInside Cloud receives the sealed form
+ * (seal()): the numbers stay readable for charts and filters, while the summary and the detail
+ * are encrypted so that only the project's browsers can read them.
+ */
+export class RequestRecord {
+  /**
+   * @param {Object} fields
+   * @param {string} fields.id - Request id
+   * @param {number} fields.occurredAt - When the request started (ms since the epoch)
+   * @param {string} fields.environment - Normalized environment label
+   * @param {string|null} [fields.release=null] - The running release (see release.js)
+   * @param {string} fields.method - HTTP method, or "OTHER" when it isn't a plain token
+   * @param {number|null} fields.status - Status code, or null when no response was sent
+   * @param {number|null} fields.duration - Response time in ms
+   * @param {number} fields.requestSize - Request body size in bytes
+   * @param {number} fields.responseSize - Response body size in bytes
+   * @param {boolean} fields.aborted - Whether the connection closed before the response was sent
+   * @param {number} [fields.exceptionCount=0] - Exceptions recorded during the request (detail.exceptions
+   *   lists the first of them)
+   * @param {Object} [fields.stats] - Timeline counters: queries, queryTime, slowQueries, nPlusOne,
+   *   duplicates, http, httpTime, logs (detail.timeline holds the spans); absent when the timeline is off
+   * @param {{ url: string, path: string, contentType: string, user?: Object }} fields.summary - Shown in
+   *   the request list (with the user, when the app set one)
+   * @param {Object} fields.detail - Everything else: headers, bodies, query, client details
+   */
+  constructor({ id, occurredAt, environment, release = null, method, status, duration, requestSize, responseSize, aborted, exceptionCount = 0, stats, summary, detail }) {
+    this.id = id;
+    this.occurredAt = occurredAt;
+    this.environment = environment;
+    this.release = release;
+    this.method = method;
+    this.status = status;
+    this.duration = duration;
+    this.requestSize = requestSize;
+    this.responseSize = responseSize;
+    this.aborted = aborted;
+    this.exceptionCount = exceptionCount;
+    if (stats) this.stats = stats;
+    this.summary = summary;
+    this.detail = detail;
+  }
+
+  /**
+   * Builds the record of an exchange. Values that would break lists or charts (a method that
+   * isn't a token, a status outside 100-599, negative sizes) are replaced rather than stored.
+   *
+   * @param {import('./capture/exchange-recorder.js').Exchange} exchange - Exchange built by PeekInside
+   * @param {string} environment - Normalized environment label
+   * @param {string|null} [release=null] - The running release
+   * @returns {RequestRecord}
+   */
+  static fromExchange(exchange, environment, release = null) {
+    const response = exchange.response || {};
+    const status = response.statusCode;
+
+    return new RequestRecord({
+      id: exchange.id,
+      occurredAt: exchange.timestamp,
+      environment,
+      release,
+      method: METHOD.test(exchange.method) ? exchange.method : 'OTHER',
+      status: Number.isInteger(status) && status >= 100 && status <= 599 ? status : null,
+      duration: typeof response.responseTime === 'number' && response.responseTime >= 0 ? response.responseTime : null,
+      requestSize: size(exchange.bodySize),
+      responseSize: size(response.size),
+      aborted: Boolean(response.aborted),
+      exceptionCount: size(exchange.exceptionCount),
+      stats: exchange.stats && typeof exchange.stats === 'object' ? exchange.stats : undefined,
+      summary: {
+        url: exchange.url,
+        path: exchange.path,
+        contentType: headerValue(response.headers, 'content-type'),
+        ...(exchange.user ? { user: exchange.user } : {})
+      },
+      detail: {
+        query: exchange.query,
+        params: exchange.params,
+        headers: exchange.headers,
+        body: exchange.body,
+        bodySize: exchange.bodySize,
+        bodyTruncated: exchange.bodyTruncated,
+        bodyEncoding: exchange.bodyEncoding,
+        ip: exchange.ip,
+        userAgent: exchange.userAgent,
+        protocol: exchange.protocol,
+        httpVersion: exchange.httpVersion,
+        response: {
+          statusMessage: response.statusMessage,
+          headers: response.headers,
+          body: response.body,
+          size: response.size,
+          bodyTruncated: response.bodyTruncated,
+          bodyEncoding: response.bodyEncoding
+        },
+        exceptions: Array.isArray(exchange.exceptions) ? exchange.exceptions.slice(0, MAX_EXCEPTION_LINKS) : [],
+        ...(exchange.timeline ? { timeline: exchange.timeline } : {})
+      }
+    });
+  }
+
+  /**
+   * Turns an environment label into the form records store: lowercase letters, digits, "-" and
+   * "_", at most 32 characters. Dashboards group and filter by it, so "Staging " and "staging"
+   * must be the same label.
+   *
+   * @param {*} value - Label chosen by the app
+   * @param {string} fallback - Label used when nothing usable remains
+   * @returns {string}
+   */
+  static normalizeEnvironment(value, fallback) {
+    const normalized = String(value || '')
+      .toLowerCase()
+      .replace(/[^a-z0-9_-]+/g, '-')
+      .replace(/^[^a-z0-9]+/, '')
+      .slice(0, 32);
+    return normalized || fallback;
+  }
+
+  /**
+   * The plain form, as the local hub stores it. Also used by JSON.stringify().
+   *
+   * @returns {Object}
+   */
+  toJSON() {
+    return { ...this };
+  }
+
+  /**
+   * The sealed form, as PeekInside Cloud stores it: the numbers stay readable for charts and
+   * filters, while the summary and the detail are encrypted to the project's public key. Each is
+   * bound to the request id, so they can't be swapped between requests.
+   *
+   * @param {import('./crypto.js').Sealer} sealer - Sealer for the batch this record is sent in
+   * @returns {Promise<Object>} The record, with `summary` and `detail` as sealed strings
+   */
+  async seal(sealer) {
+    const { summary, detail, ...meta } = this;
+    return {
+      ...meta,
+      summary: await sealer.seal(JSON.stringify(summary), `summary:${meta.id}`),
+      detail: await sealer.seal(JSON.stringify(detail), `detail:${meta.id}`)
+    };
+  }
+
+  /**
+   * seal() for a process that is crashing, where nothing asynchronous runs anymore.
+   *
+   * @param {import('./crypto.js').SyncSealer} sealer
+   * @returns {Object}
+   */
+  sealSync(sealer) {
+    const { summary, detail, ...meta } = this;
+    return {
+      ...meta,
+      summary: sealer.seal(JSON.stringify(summary), `summary:${meta.id}`),
+      detail: sealer.seal(JSON.stringify(detail), `detail:${meta.id}`)
+    };
+  }
+}
+
+function headerValue(headers, name) {
+  const value = headers && headers[name];
+  return Array.isArray(value) ? String(value[0]) : value === undefined ? '' : String(value);
+}
+
+function size(value) {
+  return Number.isInteger(value) && value >= 0 && value < 2 ** 31 ? value : 0;
+}

package/src/release.js

@@ -0,0 +1,157 @@
+/**
+ * Which release of the app is running: the deploy a request or an exception came from, so the
+ * dashboard can say when an issue first appeared and what broke after a deploy.
+ *
+ * In order: the `release` option or PEEKINSIDE_RELEASE; else the commit the hosting platform
+ * deployed (Vercel, Netlify, Render, Railway, Heroku, Cloudflare Pages, Google Cloud Run, Fly.io,
+ * and the CI services); else the commit checked out in the project folder; else none.
+ *
+ * Releases are labels, not secrets: PeekInside Cloud stores them readable, like the environment,
+ * so they can be filtered and counted.
+ */
+
+import fs from 'fs';
+import path from 'path';
+
+// Commit (or deploy) variables the platforms set, most specific first.
+const PLATFORM_VARIABLES = [
+  'VERCEL_GIT_COMMIT_SHA',
+  'COMMIT_REF', // Netlify
+  'RENDER_GIT_COMMIT',
+  'RAILWAY_GIT_COMMIT_SHA',
+  'HEROKU_BUILD_COMMIT',
+  'HEROKU_SLUG_COMMIT',
+  'SOURCE_VERSION', // Heroku builds
+  'CF_PAGES_COMMIT_SHA',
+  'K_REVISION', // Google Cloud Run
+  'FLY_IMAGE_REF',
+  'GITHUB_SHA',
+  'CI_COMMIT_SHA', // GitLab
+  'BITBUCKET_COMMIT',
+  'CIRCLE_SHA1',
+  'CODEBUILD_RESOLVED_SOURCE_VERSION', // AWS CodeBuild
+  'COMMIT_SHA' // Google Cloud Build
+];
+
+// What a stored release may contain: the characters of commits, versions and image tags.
+const UNSAFE = /[^A-Za-z0-9._:@/+-]+/g;
+export const MAX_RELEASE_LENGTH = 64;
+const COMMIT = /^[0-9a-f]{40}(?:[0-9a-f]{24})?$/;
+// How far up from the project folder a repository is looked for (monorepos).
+const MAX_GIT_DEPTH = 6;
+
+/**
+ * Finds and normalizes the running release.
+ */
+export class Release {
+  /**
+   * The running release.
+   *
+   * @param {Object} [options]
+   * @param {string} [options.explicit] - The `release` option
+   * @param {Object<string, string|undefined>} [options.env=process.env]
+   * @param {string} [options.root] - The project folder, where the checked-out commit is read
+   * @returns {string|null}
+   */
+  static detect({ explicit, env = process.env, root } = {}) {
+    const chosen = Release.normalize(explicit) || Release.normalize(env.PEEKINSIDE_RELEASE);
+    if (chosen) return chosen;
+    for (const name of PLATFORM_VARIABLES) {
+      const value = Release.normalize(env[name]);
+      if (value) return value;
+    }
+    return root ? Release.normalize(Release.gitCommit(root)) : null;
+  }
+
+  /**
+   * Turns a label into the form records store: at most 64 characters among letters, digits and
+   * `. _ : @ / + -`; anything else becomes "-".
+   *
+   * @param {*} value
+   * @returns {string|null} null when nothing usable remains
+   */
+  static normalize(value) {
+    if (typeof value !== 'string') return null;
+    const clean = value.trim().replace(UNSAFE, '-').replace(/^-+|-+$/g, '').slice(0, MAX_RELEASE_LENGTH);
+    return clean || null;
+  }
+
+  /**
+   * The commit checked out in the git repository holding a folder, read from its files (no git
+   * process is started). Handles branches, detached heads, packed refs and worktrees.
+   *
+   * @param {string} root - A folder inside the repository
+   * @returns {string|null} The commit hash, or null outside a repository
+   */
+  static gitCommit(root) {
+    try {
+      const gitDir = findGitDir(path.resolve(root));
+      if (!gitDir) return null;
+      const head = readText(path.join(gitDir, 'HEAD'));
+      if (!head) return null;
+      if (COMMIT.test(head)) return head;
+      const ref = /^ref:\s*(\S+)$/.exec(head);
+      if (!ref) return null;
+      // A worktree keeps its HEAD; the refs live in the main repository ("commondir").
+      const common = readText(path.join(gitDir, 'commondir'));
+      const refsDir = common ? path.resolve(gitDir, common) : gitDir;
+      for (const dir of new Set([gitDir, refsDir])) {
+        const loose = readText(path.join(dir, ...ref[1].split('/')));
+        if (loose && COMMIT.test(loose)) return loose;
+      }
+      const packed = readText(path.join(refsDir, 'packed-refs')) || '';
+      for (const line of packed.split('\n')) {
+        const [hash, name] = line.trim().split(/\s+/);
+        if (name === ref[1] && COMMIT.test(hash)) return hash;
+      }
+    } catch (error) {
+      // Not a repository we can read: no release from git.
+    }
+    return null;
+  }
+
+  /**
+   * How the dashboard shows a release: commits as their first 7 characters, other labels as they are.
+   *
+   * @param {string|null} release
+   * @returns {string}
+   */
+  static short(release) {
+    if (!release) return '';
+    return COMMIT.test(release) ? release.slice(0, 7) : release;
+  }
+}
+
+// The .git folder of the repository holding `start`: a folder, or a file pointing at one
+// (worktrees, submodules).
+function findGitDir(start) {
+  let dir = start;
+  for (let depth = 0; depth < MAX_GIT_DEPTH; depth++) {
+    const candidate = path.join(dir, '.git');
+    let stat = null;
+    try {
+      stat = fs.statSync(candidate);
+    } catch (error) {
+      stat = null;
+    }
+    if (stat && stat.isDirectory()) return candidate;
+    if (stat && stat.isFile()) {
+      const pointer = /^gitdir:\s*(.+)$/m.exec(readText(candidate) || '');
+      return pointer ? path.resolve(dir, pointer[1].trim()) : null;
+    }
+    const parent = path.dirname(dir);
+    if (parent === dir) return null;
+    dir = parent;
+  }
+  return null;
+}
+
+function readText(file) {
+  try {
+    const stat = fs.statSync(file);
+    if (!stat.isFile() || stat.size > 1024 * 1024) return null;
+    return fs.readFileSync(file, 'utf8').trim();
+  } catch (error) {
+    return null;
+  }
+}