peekinside 0.0.0-stage → 1.2.1

package/src/capture/body-capture.js

@@ -0,0 +1,86 @@
+/**
+ * Bounded collection of body bytes.
+ *
+ * Bodies can be arbitrarily large (uploads, downloads, streams), so only the first `limit` bytes
+ * are kept, while the full size is still counted: the dashboard marks a body as truncated when
+ * the two differ.
+ */
+
+/**
+ * Collects the bytes of one request or response body, up to a limit.
+ */
+export class BodyCapture {
+  /**
+   * @param {number} limit - Most bytes kept (maxBodySize)
+   */
+  constructor(limit) {
+    this.limit = limit;
+    this._chunks = [];
+    this._captured = 0;
+    this._size = 0;
+  }
+
+  /**
+   * Adds a chunk as the app read or wrote it. Anything that is not body data, such as a missing
+   * chunk or a callback passed in its place, is ignored.
+   *
+   * @param {Buffer|Uint8Array|string|*} chunk
+   * @param {string} [encoding] - Encoding of a string chunk (utf8 when missing or unknown)
+   * @returns {void}
+   */
+  push(chunk, encoding) {
+    if (chunk === undefined || chunk === null || typeof chunk === 'function') {
+      return;
+    }
+    let buffer;
+    try {
+      buffer = toBuffer(chunk, encoding);
+    } catch (error) {
+      return;
+    }
+    this._size += buffer.length;
+    if (this._captured >= this.limit) {
+      return;
+    }
+    const take = Math.min(buffer.length, this.limit - this._captured);
+    // A copy: the app may reuse its buffer once written, and a slice would keep all of it alive.
+    this._chunks.push(Buffer.from(buffer.subarray(0, take)));
+    this._captured += take;
+  }
+
+  /**
+   * Full body size in bytes, including bytes beyond the limit.
+   *
+   * @type {number}
+   */
+  get size() {
+    return this._size;
+  }
+
+  /**
+   * Whether bytes beyond the limit were left out.
+   *
+   * @type {boolean}
+   */
+  get truncated() {
+    return this._size > this._captured;
+  }
+
+  /**
+   * @returns {Buffer} The bytes kept
+   */
+  buffer() {
+    return Buffer.concat(this._chunks, this._captured);
+  }
+}
+
+function toBuffer(chunk, encoding) {
+  if (Buffer.isBuffer(chunk)) {
+    return chunk;
+  }
+  if (chunk instanceof Uint8Array) {
+    return Buffer.from(chunk.buffer, chunk.byteOffset, chunk.byteLength);
+  }
+  const charset = typeof encoding === 'string' && Buffer.isEncoding(encoding) ? encoding : 'utf8';
+  return Buffer.from(String(chunk), charset);
+}

package/src/capture/body-decoder.js

@@ -0,0 +1,188 @@
+/**
+ * Turns captured body bytes into what the dashboard shows: decompressed, and as text when the
+ * body is text, or only its size when it is binary.
+ *
+ * Also turns a body that a framework parsed before PeekInside could see the bytes into text,
+ * from the parsed value.
+ */
+
+import zlib from 'zlib';
+import { BodyCapture } from './body-capture.js';
+import { headerValue } from './http-message.js';
+
+const TEXTUAL_TYPE = /^(text\/|application\/(json|xml|javascript|ecmascript|x-www-form-urlencoded|graphql|x-ndjson|csv|yaml|x-yaml|toml)\b)|\+(json|xml)\b|^image\/svg\+xml/i;
+const BINARY_TYPE = /^(image|audio|video|font)\/|^application\/(octet-stream|pdf|zip|gzip|x-gzip|x-tar|wasm|x-protobuf|protobuf|grpc|vnd\.ms-|vnd\.openxmlformats)/i;
+
+/**
+ * @typedef {Object} BodyContent
+ * @property {string} text - The body as text; empty for binary or missing bodies
+ * @property {number} size - Full body size in bytes, as sent
+ * @property {boolean} truncated - Whether `text` holds only the start of the body
+ * @property {'none'|'utf8'|'binary'|'unavailable'} encoding - "unavailable" when the body was
+ *   never read by the app, so there was nothing to capture
+ */
+
+/**
+ * The content of an absent body.
+ *
+ * @type {Readonly<BodyContent>}
+ */
+export const EMPTY_BODY = Object.freeze({ text: '', size: 0, truncated: false, encoding: 'none' });
+
+/**
+ * Decodes captured bodies, keeping at most `limit` bytes of text.
+ */
+export class BodyDecoder {
+  /**
+   * @param {number} limit - Most bytes of text kept per body, after decompression
+   */
+  constructor(limit) {
+    this.limit = limit;
+  }
+
+  /**
+   * Decodes captured bytes: decompressed per Content-Encoding, then kept as text unless they are
+   * binary.
+   *
+   * @param {BodyCapture|null} capture - Captured body
+   * @param {Object} headers - Headers of the same message, with lowercase names
+   * @returns {BodyContent}
+   */
+  decode(capture, headers) {
+    if (!capture || capture.size === 0) {
+      return EMPTY_BODY;
+    }
+
+    let buffer = capture.buffer();
+    let truncated = capture.truncated;
+    let trustContentType = true;
+
+    const contentEncoding = String(headerValue(headers, 'content-encoding') || '').trim().toLowerCase();
+    if (contentEncoding && contentEncoding !== 'identity') {
+      const decoded = this._decompress(buffer, contentEncoding);
+      if (decoded) {
+        buffer = decoded.buffer;
+        truncated = truncated || decoded.truncated;
+      } else {
+        // Not decodable (or already decoded by middleware below us): judge by content
+        trustContentType = false;
+      }
+    }
+
+    const contentType = String(headerValue(headers, 'content-type') || '');
+    if (!BodyDecoder.isTextual(contentType, buffer, trustContentType)) {
+      return { text: '', size: capture.size, truncated, encoding: 'binary' };
+    }
+    return { text: buffer.toString('utf8'), size: capture.size, truncated, encoding: 'utf8' };
+  }
+
+  /**
+   * Content of a body known only from the value a framework parsed it into, because the app read
+   * the bytes before PeekInside was tracking the request.
+   *
+   * @param {*} value - Parsed body: an object, a string, a Buffer, or undefined when never read
+   * @param {Object} headers - Request headers
+   * @returns {BodyContent} Objects are shown as JSON; the size is Content-Length when known
+   */
+  decodeValue(value, headers) {
+    const declared = Number(headers['content-length']) || 0;
+
+    if (value === undefined || value === null) {
+      // The application never read the body, so there was nothing to capture.
+      return { text: '', size: declared, truncated: false, encoding: 'unavailable' };
+    }
+    if (Buffer.isBuffer(value)) {
+      const capture = new BodyCapture(this.limit);
+      capture.push(value);
+      return this.decode(capture, headers);
+    }
+
+    let text;
+    if (typeof value === 'string') {
+      text = value;
+    } else {
+      try {
+        text = JSON.stringify(value);
+      } catch (error) {
+        text = String(value);
+      }
+    }
+
+    const bytes = Buffer.byteLength(text);
+    const truncated = bytes > this.limit;
+    return {
+      text: truncated ? Buffer.from(text).subarray(0, this.limit).toString('utf8') : text,
+      size: declared || bytes,
+      truncated,
+      encoding: 'utf8'
+    };
+  }
+
+  /**
+   * Whether a body is text: by its Content-Type when that is trusted and conclusive, otherwise by
+   * looking at the bytes.
+   *
+   * @param {string} contentType
+   * @param {Buffer} buffer - The body, decompressed
+   * @param {boolean} [trustContentType=true] - False when the bytes may not match the headers
+   * @returns {boolean}
+   */
+  static isTextual(contentType, buffer, trustContentType = true) {
+    if (trustContentType && contentType) {
+      if (TEXTUAL_TYPE.test(contentType)) return true;
+      if (BINARY_TYPE.test(contentType)) return false;
+    }
+    return BodyDecoder.looksLikeText(buffer);
+  }
+
+  /**
+   * Heuristic for bodies without a conclusive Content-Type: text has no NUL bytes and few control
+   * characters in its first 2 KB.
+   *
+   * @param {Buffer} buffer
+   * @returns {boolean}
+   */
+  static looksLikeText(buffer) {
+    const length = Math.min(buffer.length, 2048);
+    let suspicious = 0;
+    for (let i = 0; i < length; i++) {
+      const byte = buffer[i];
+      if (byte === 0) return false;
+      if (byte < 7 || (byte > 13 && byte < 32)) suspicious++;
+    }
+    return suspicious <= length * 0.05;
+  }
+
+  // Decompresses gzip, deflate (zlib or raw) and brotli. Output is capped, so a small compressed
+  // body can't expand into gigabytes in the app's memory (a "zip bomb"), and a body cut off at
+  // the capture limit still yields the part that was captured (sync flush, not finish).
+  // Returns null when the body can't be decoded.
+  _decompress(buffer, encoding) {
+    const limit = this.limit;
+    const maxOutputLength = Math.max(limit * 16, 1024 * 1024);
+    const options = { finishFlush: zlib.constants.Z_SYNC_FLUSH, maxOutputLength };
+
+    try {
+      let output;
+      if (encoding === 'gzip' || encoding === 'x-gzip') {
+        output = zlib.gunzipSync(buffer, options);
+      } else if (encoding === 'deflate') {
+        try {
+          output = zlib.inflateSync(buffer, options);
+        } catch (error) {
+          output = zlib.inflateRawSync(buffer, options);
+        }
+      } else if (encoding === 'br') {
+        output = zlib.brotliDecompressSync(buffer, { finishFlush: zlib.constants.BROTLI_OPERATION_FLUSH, maxOutputLength });
+      } else {
+        return null;
+      }
+
+      return output.length > limit
+        ? { buffer: output.subarray(0, limit), truncated: true }
+        : { buffer: output, truncated: false };
+    } catch (error) {
+      return null;
+    }
+  }
+}

package/src/capture/exchange-recorder.js

@@ -0,0 +1,263 @@
+/**
+ * Records one request/response exchange: starts the clock, taps both bodies, and once the
+ * response has been sent (or the connection closed first) builds the exchange object that a
+ * transport turns into a stored record.
+ *
+ * Exceptions recorded during the request use the recorder too: its id (known from the start) and
+ * a snapshot of the request as received so far.
+ */
+
+import { randomUUID } from 'crypto';
+import http from 'http';
+import { performance } from 'perf_hooks';
+import { BodyDecoder, EMPTY_BODY } from './body-decoder.js';
+import { hasBody, parseQuery } from './http-message.js';
+import { RequestBodyTap } from './request-body-tap.js';
+import { ResponseTap } from './response-tap.js';
+
+const kBodySource = Symbol('peekInside.bodySource');
+
+/**
+ * @typedef {Object} Exchange
+ * @property {string} id - Random UUID
+ * @property {string} method
+ * @property {string} url - Request target as received (originalUrl in Express)
+ * @property {string} path - Pathname of `url`
+ * @property {Object<string, string|string[]>} query
+ * @property {Object<string, string>} params - Route parameters set by the framework, if any
+ * @property {Object<string, string|string[]>} headers - Request headers
+ * @property {string} body - Request body as text (see bodyEncoding)
+ * @property {number} bodySize
+ * @property {boolean} bodyTruncated
+ * @property {'none'|'utf8'|'binary'|'unavailable'} bodyEncoding
+ * @property {string} ip
+ * @property {string} userAgent
+ * @property {'http'|'https'} protocol
+ * @property {string} httpVersion
+ * @property {number} timestamp - When the request started (ms since the epoch)
+ * @property {Object} response - statusCode (null when nothing was sent), statusMessage, headers,
+ *   body, size, bodyTruncated, bodyEncoding, responseTime (ms) and aborted
+ * @property {number} [exceptionCount] - Exceptions recorded during the request, added by PeekInside
+ * @property {Array<{ id: string, name: string, message: string }>} [exceptions] - The first of them
+ * @property {Object} [stats] - Timeline counters (queries, outgoing calls, logs), added by PeekInside
+ * @property {Object} [timeline] - The request's timeline, when something was recorded on it
+ */
+
+/**
+ * Records a single exchange. Create one per request, then call start() before the app reads the
+ * request body.
+ */
+export class ExchangeRecorder {
+  /**
+   * Tells the recorder where a framework keeps a request's parsed body. It is used when the app
+   * read the body before tracking started (a body parser registered before PeekInside), so the
+   * bytes themselves were never seen.
+   *
+   * @param {import('http').IncomingMessage} req
+   * @param {() => *} getter - Returns the parsed body, such as Koa's ctx.request.body
+   * @returns {void}
+   */
+  static setBodySource(req, getter) {
+    req[kBodySource] = getter;
+  }
+
+  /**
+   * @param {import('http').IncomingMessage} req
+   * @param {import('http').ServerResponse} res
+   * @param {{ url: string, pathname: string, search: string }} target - The request target, split
+   * @param {Object} options
+   * @param {() => number} options.maxBodySize - Returns the current maxBodySize; it is read when
+   *   the bodies are tapped and again when they are decoded, so later option changes apply
+   * @param {(exchange: Exchange) => void} options.onExchange - Receives the finished exchange
+   */
+  constructor(req, res, { url, pathname, search }, { maxBodySize, onExchange }) {
+    this.req = req;
+    this.res = res;
+    this.url = url;
+    this.pathname = pathname;
+    this.search = search;
+    this._maxBodySize = maxBodySize;
+    this._onExchange = onExchange;
+    this._recorded = false;
+
+    /**
+     * Id of the request record. Created now rather than when the exchange is built, so that
+     * exceptions recorded during the request can refer to it.
+     * @type {string}
+     */
+    this.id = randomUUID();
+  }
+
+  /**
+   * Whether the exchange has been built (the response finished, or the connection closed).
+   *
+   * @type {boolean}
+   */
+  get recorded() {
+    return this._recorded;
+  }
+
+  /**
+   * Starts the clock, taps both bodies and waits for the response to finish.
+   *
+   * @returns {ExchangeRecorder} This recorder
+   */
+  start() {
+    const res = this.res;
+    this.startedAt = Date.now();
+    // The timeline's clock: span offsets count from the same instant as occurredAt.
+    this.perfStart = performance.now();
+    this._start = process.hrtime.bigint();
+    this._requestTap = new RequestBodyTap(this.req, this._maxBodySize());
+    this._responseTap = new ResponseTap(res, this._maxBodySize());
+
+    this._onFinish = () => this._finish(false);
+    // 'close' without 'finish' means the client went away before the response was sent.
+    this._onClose = () => this._finish(!res.writableFinished);
+    res.on('finish', this._onFinish);
+    res.on('close', this._onClose);
+    return this;
+  }
+
+  /**
+   * Builds the exchange as it stands now.
+   *
+   * @param {boolean} aborted - Whether the connection closed before the response was sent
+   * @returns {Exchange}
+   */
+  buildExchange(aborted) {
+    const { req, res } = this;
+    const duration = Number(process.hrtime.bigint() - this._start) / 1e6;
+    const decoder = new BodyDecoder(this._maxBodySize());
+    const requestHeaders = { ...req.headers };
+    const responseHeaders = this._responseTap.headers();
+    const requestContent = this._requestContent(decoder, requestHeaders);
+    // HEAD responses never have a body, whatever the app wrote.
+    const responseContent = req.method === 'HEAD'
+      ? EMPTY_BODY
+      : decoder.decode(this._responseTap.body, responseHeaders);
+    const statusCode = aborted && !res.headersSent ? null : res.statusCode;
+
+    return {
+      id: this.id,
+      method: req.method,
+      url: this.url,
+      path: this.pathname,
+      query: parseQuery(this.search),
+      params: req.params && typeof req.params === 'object' ? { ...req.params } : {},
+      headers: requestHeaders,
+      body: requestContent.text,
+      bodySize: requestContent.size,
+      bodyTruncated: requestContent.truncated,
+      bodyEncoding: requestContent.encoding,
+      ip: req.ip || (req.socket && req.socket.remoteAddress) || '',
+      userAgent: requestHeaders['user-agent'] || '',
+      protocol: req.socket && req.socket.encrypted ? 'https' : 'http',
+      httpVersion: req.httpVersion || '1.1',
+      timestamp: this.startedAt,
+      response: {
+        statusCode,
+        statusMessage: statusCode === null ? '' : (res.statusMessage || http.STATUS_CODES[statusCode] || ''),
+        headers: responseHeaders,
+        body: responseContent.text,
+        size: responseContent.size,
+        bodyTruncated: responseContent.truncated,
+        bodyEncoding: responseContent.encoding,
+        responseTime: Math.round(duration * 100) / 100,
+        aborted: Boolean(aborted)
+      }
+    };
+  }
+
+  /**
+   * The request as an exception record shows it: what has been received so far, with a shorter
+   * body than the request record keeps.
+   *
+   * @param {number} bodyLimit - Most bytes of body text
+   * @returns {{ method: string, url: string, headers: Object, query: Object, params: Object,
+   *   body: string, bodyTruncated: boolean, ip: string, userAgent: string }}
+   */
+  snapshot(bodyLimit) {
+    const req = this.req;
+    const headers = { ...req.headers };
+    const content = this._requestContent(new BodyDecoder(bodyLimit), headers);
+    return {
+      method: String(req.method || ''),
+      url: this.url,
+      headers,
+      query: parseQuery(this.search),
+      params: stringParams(req.params),
+      body: content.encoding === 'utf8' ? content.text : '',
+      bodyTruncated: content.encoding === 'utf8' && content.truncated,
+      ip: req.ip || (req.socket && req.socket.remoteAddress) || '',
+      userAgent: headers['user-agent'] || ''
+    };
+  }
+
+  /**
+   * Builds the exchange right away, for a process that is about to exit: a response still in
+   * progress is recorded as aborted. Afterwards the recorder records nothing more.
+   *
+   * @returns {Exchange|null} null when the exchange was already recorded
+   */
+  takeExchange() {
+    if (this._recorded) {
+      return null;
+    }
+    this._recorded = true;
+    this.res.removeListener('finish', this._onFinish);
+    this.res.removeListener('close', this._onClose);
+    return this.buildExchange(!this.res.writableFinished);
+  }
+
+  // Runs once, on 'finish' or 'close', whichever comes first.
+  _finish(aborted) {
+    if (this._recorded) {
+      return;
+    }
+    this._recorded = true;
+    this.res.removeListener('finish', this._onFinish);
+    this.res.removeListener('close', this._onClose);
+
+    try {
+      this._onExchange(this.buildExchange(aborted));
+    } catch (error) {
+      // Ignore: a malformed exchange must not affect the application.
+    }
+  }
+
+  // The request body as the app read it, or, when the app read it before tracking started,
+  // what the framework parsed it into.
+  _requestContent(decoder, headers) {
+    const req = this.req;
+    const tapped = this._requestTap.body;
+    if (tapped.size > 0) {
+      return decoder.decode(tapped, headers);
+    }
+    if (!hasBody(headers)) {
+      return EMPTY_BODY;
+    }
+
+    let parsed;
+    try {
+      parsed = typeof req[kBodySource] === 'function' ? req[kBodySource]() : undefined;
+    } catch (error) {
+      parsed = undefined;
+    }
+    if (parsed === undefined) {
+      parsed = req.body !== undefined ? req.body : req.rawBody;
+    }
+    return decoder.decodeValue(parsed, headers);
+  }
+}
+
+// Route parameters as strings (Express keeps them as strings; other routers may not).
+function stringParams(params) {
+  const result = {};
+  if (!params || typeof params !== 'object') return result;
+  for (const key of Object.keys(params)) {
+    const value = params[key];
+    if (value !== undefined && value !== null && typeof value !== 'object') result[key] = String(value);
+  }
+  return result;
+}

package/src/capture/http-message.js

@@ -0,0 +1,138 @@
+/**
+ * Pure helpers for the parts of HTTP messages that PeekInside records: request targets, query
+ * strings and header objects.
+ *
+ * Query keys and header names come from the network, so the objects built from them define
+ * every key as an own property: a key such as "__proto__" stays a plain entry instead of
+ * changing the object's prototype.
+ */
+
+/**
+ * Splits a request target into pathname and search, without normalizing it: the dashboard shows
+ * the path exactly as the client sent it.
+ *
+ * @param {string} url - Request target, such as "/users?page=2" or an absolute URL (sent to proxies)
+ * @returns {{ pathname: string, search: string }} `search` includes the "?" when present
+ */
+export function splitUrl(url) {
+  let target = typeof url === 'string' && url !== '' ? url : '/';
+  if (/^[a-z][a-z\d+.-]*:\/\//i.test(target)) {
+    // Absolute-form target, as sent to proxies
+    try {
+      const parsed = new URL(target);
+      target = parsed.pathname + parsed.search;
+    } catch (error) {
+      target = '/';
+    }
+  }
+  const index = target.indexOf('?');
+  return index === -1
+    ? { pathname: target, search: '' }
+    : { pathname: target.slice(0, index), search: target.slice(index) };
+}
+
+/**
+ * Parses a query string. Repeated keys become arrays, in order.
+ *
+ * @param {string} search - Query string, including the leading "?"
+ * @returns {Object<string, string|string[]>}
+ */
+export function parseQuery(search) {
+  const query = {};
+  if (!search || search.length < 2) {
+    return query;
+  }
+
+  let params;
+  try {
+    params = new URLSearchParams(search.slice(1));
+  } catch (error) {
+    return query;
+  }
+
+  for (const [key, value] of params) {
+    const existing = Object.prototype.hasOwnProperty.call(query, key) ? query[key] : undefined;
+    setOwn(query, key, existing === undefined ? value : [].concat(existing, value));
+  }
+  return query;
+}
+
+/**
+ * Turns headers in any form writeHead() accepts into an object with lowercase names.
+ *
+ * @param {Object|Array} headers - An object, a flat array [name, value, ...] or an array of pairs
+ * @returns {Object<string, *>}
+ */
+export function normalizeHeaders(headers) {
+  const normalized = {};
+  const add = (name, value) => {
+    if (name !== undefined && name !== null && value !== undefined) {
+      setOwn(normalized, String(name).toLowerCase(), value);
+    }
+  };
+
+  if (Array.isArray(headers)) {
+    if (headers.length > 0 && Array.isArray(headers[0])) {
+      for (const pair of headers) add(pair[0], pair[1]);
+    } else {
+      for (let i = 0; i + 1 < headers.length; i += 2) add(headers[i], headers[i + 1]);
+    }
+  } else if (headers && typeof headers === 'object') {
+    for (const name of Object.keys(headers)) add(name, headers[name]);
+  }
+  return normalized;
+}
+
+/**
+ * Merges header objects; later sources win.
+ *
+ * @param {...(Object|null|undefined)} sources
+ * @returns {Object<string, *>}
+ */
+export function mergeHeaders(...sources) {
+  const merged = {};
+  for (const source of sources) {
+    if (!source) continue;
+    for (const name of Object.keys(source)) {
+      setOwn(merged, name, source[name]);
+    }
+  }
+  return merged;
+}
+
+/**
+ * @param {Object} headers - Headers with lowercase names
+ * @param {string} name - Lowercase header name
+ * @returns {*} The header's value, or its first value when it was sent several times
+ */
+export function headerValue(headers, name) {
+  const value = headers[name];
+  return Array.isArray(value) ? value[0] : value;
+}
+
+/**
+ * Whether request headers announce a body: chunked, or a positive Content-Length.
+ *
+ * @param {Object} headers - Request headers
+ * @returns {boolean}
+ */
+export function hasBody(headers) {
+  if (headers['transfer-encoding'] !== undefined) {
+    return true;
+  }
+  const length = Number(headers['content-length']);
+  return Number.isFinite(length) && length > 0;
+}
+
+/**
+ * Sets a property as an own, enumerable entry, even for keys such as "__proto__" that would
+ * otherwise change the object's prototype.
+ *
+ * @param {Object} target
+ * @param {string} key
+ * @param {*} value
+ * @returns {void}
+ */
+export function setOwn(target, key, value) {
+  Object.defineProperty(target, key, { value, enumerable: true, writable: true, configurable: true });
+}