snappy-prerender 2.6.0 → 2.7.0

package/src/core/crawl.js

@@ -0,0 +1,300 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { renderRoute } from './render.js';
+import { resolveRouteConfig } from './route-config.js';
+import {
+  crawlRoots,
+  filterRoutes,
+  isIncludedRoute,
+  matchesPattern,
+  normaliseRoute,
+  withinDepth,
+} from './routes.js';
+import { createPool, mapLimit } from './scheduler.js';
+
+/**
+ * Runs the crawl and render pool. Each rendered page contributes its output, any new
+ * same-origin routes, and the resources recorded for the link-hint features.
+ */
+export async function renderAllRoutes({
+  browser,
+  server,
+  config,
+  log,
+  report,
+  outputDir,
+  sourceSeeds = [],
+}) {
+  const visited = new Set();
+  const scheduled = new Set();
+  const rendered = new Map();
+  const cap = config.maxRoutes ?? Number.POSITIVE_INFINITY;
+  let nextOrder = 0;
+
+  // Queues routes up to maxRoutes, recording exactly which candidates did not fit.
+  // A seed used only to reach wildcard routes is given output: false and is rendered
+  // for its links without being written.
+  const accept = (routes, outputFor = () => true) => {
+    const accepted = [];
+    const dropped = [];
+    for (const route of routes) {
+      if (scheduled.has(route)) continue;
+      if (scheduled.size >= cap) {
+        dropped.push(route);
+        continue;
+      }
+      scheduled.add(route);
+      accepted.push({ route, output: outputFor(route), order: nextOrder });
+      nextOrder += 1;
+    }
+    if (dropped.length > 0) {
+      const previous = report.truncated?.droppedRoutes ?? [];
+      report.truncated = {
+        limit: config.maxRoutes,
+        routes: scheduled.size,
+        droppedRoutes: [...new Set([...previous, ...dropped])].sort(),
+      };
+    }
+    return accepted;
+  };
+
+  const renderOne = async ({ route, output, depth = 0, order = null }) => {
+    if (visited.has(route)) return null;
+    visited.add(route);
+    log.debug(`rendering ${route}${output ? '' : ' as a crawl seed'}`);
+    const routeConfig = resolveRouteConfig(config, route);
+    const result = await renderWithRetries({
+      browser,
+      server,
+      route,
+      routeConfig,
+      config,
+      log,
+      report,
+      outputDir,
+      screenshot: config.saveAs !== 'html',
+    });
+    if (!result) return null;
+    result.depth = depth;
+    result.order = order;
+    try {
+      const notFoundRoute = config.notFoundRoute ? normaliseRoute(config.notFoundRoute) : null;
+      const isNotFoundRoute = notFoundRoute !== null && route === notFoundRoute;
+      if (result.statusCode === 404 && !isNotFoundRoute) {
+        report.errors.push({ route, message: 'returned 404' });
+        log.error(`  route ${route} returned 404`);
+        return null;
+      }
+      if (isNotFoundRoute) {
+        result.notFound = true;
+        if (!report.notFoundRoutes.includes(route)) report.notFoundRoutes.push(route);
+        if (result.statusCode !== null && result.statusCode !== 404) {
+          log.warn(`  ignored unknown prerender-status-code ${result.statusCode} on ${route}`);
+        }
+      } else if (result.statusCode !== null) {
+        log.warn(`  ignored unknown prerender-status-code ${result.statusCode} on ${route}`);
+      }
+      if (output && result.empty) {
+        report.emptyPages.push(route);
+        if (config.failOnEmptyPage) {
+          report.errors.push({ route, message: 'rendered an empty page' });
+          log.error(`  empty page ${route}: no app content was found`);
+          return null;
+        }
+        log.warn(`  empty page ${route}: no app content was found`);
+      }
+      if (output) {
+        rendered.set(route, result);
+        report.routes.push(route);
+        for (const to of result.links) report.links.push({ from: route, to });
+      }
+      for (const message of result.pageErrors) {
+        report.pageErrors.push({ route, message });
+        log.warn(`  page error on ${route}: ${message}`);
+      }
+      for (const message of result.consoleErrors) {
+        report.consoleErrors.push({ route, message });
+      }
+      if (result.consoleErrors.length > 0) {
+        log.debug(`  console error on ${route}: ${result.consoleErrors[0]}`);
+        if (config.failOnConsoleError) {
+          report.errors.push({ route, message: `console error: ${result.consoleErrors[0]}` });
+        }
+      }
+      for (const failure of result.failedRequests) {
+        report.failedRequests.push({ route, ...failure });
+        const detail = failure.status ? `HTTP ${failure.status}` : failure.error;
+        log.warn(`  failed request on ${route}: ${failure.url} (${detail})`);
+      }
+      if (result.failedRequests.length > 0 && config.failOnFailedRequest) {
+        const first = result.failedRequests[0];
+        report.errors.push({
+          route,
+          message: `failed request: ${first.url}${first.status ? ` (HTTP ${first.status})` : ''}`,
+        });
+      }
+      return { result, routeConfig };
+    } catch (error) {
+      report.errors.push({ route, message: error.message });
+      log.error(`  failed ${route}: ${error.message}`);
+      return null;
+    }
+  };
+
+  const outputFor = (route) => !config.includeProvided || isIncludedRoute(route, config.include);
+  const candidates = [];
+  if (sourceSeeds.length > 0) candidates.push(...sourceSeeds);
+  if (sourceSeeds.length === 0 || config.includeProvided) {
+    candidates.push(...crawlRoots(config));
+  }
+  const roots = accept([...new Set(sortRoutes(candidates, config))], outputFor);
+
+  const deterministic = config.maxRoutes !== null || config.routePriority.length > 0;
+  if (deterministic) {
+    await renderWaves({ config, accept, renderOne, roots, report, log });
+  } else {
+    const pool = createPool({
+      concurrency: config.concurrency,
+      handler: async ({ route, depth, output, order }) => {
+        const renderedRoute = await renderOne({ route, output, depth, order });
+        if (!renderedRoute) return;
+        const { result, routeConfig } = renderedRoute;
+        if (routeConfig.crawl && withinDepth(depth + 1, routeConfig.maxDepth)) {
+          const discovered = filterRoutes(result.links, {
+            include: config.includeProvided ? config.include : [],
+            exclude: config.exclude,
+          }).filter((candidate) => !scheduled.has(candidate));
+          const accepted = accept(sortRoutes(discovered, config));
+          pool.push(accepted.map((item) => ({ ...item, depth: depth + 1 })));
+        }
+      },
+    });
+    pool.push(roots.map((item) => ({ ...item, depth: 0 })));
+    const { errors } = await pool.run();
+    for (const { item, error } of errors) {
+      report.errors.push({ route: item.route, message: error.message });
+      log.error(`  pool error on ${item.route}: ${error.message}`);
+    }
+  }
+  return rendered;
+}
+
+/**
+ * Renders one route with the configured retry policy. A retryable failure is retried
+ * after retryDelay; the final failure records the attempt count and, when traceOnError
+ * is on, a Playwright trace written under the output directory.
+ */
+async function renderWithRetries({
+  browser,
+  server,
+  route,
+  routeConfig,
+  config,
+  log,
+  report,
+  outputDir,
+  screenshot,
+}) {
+  const attempts = config.retries + 1;
+  for (let attempt = 1; attempt <= attempts; attempt += 1) {
+    const traceFile =
+      config.traceOnError && !config.dryRun && attempt === attempts
+        ? await prepareTraceFile(outputDir, route, attempt)
+        : null;
+    try {
+      const result = await renderRoute({
+        browser,
+        origin: server.origin,
+        route,
+        config: routeConfig,
+        screenshot,
+        traceFile,
+      });
+      if (attempt > 1) log.info(`  ${route}: succeeded on attempt ${attempt}`);
+      result.attempts = attempt;
+      return result;
+    } catch (error) {
+      if (error.tracePath) {
+        report.traces.push({ route, attempt, file: error.tracePath });
+        log.warn(`  trace for ${route}: ${error.tracePath}`);
+      }
+      if (error.retryable !== false && attempt < attempts) {
+        log.warn(
+          `  attempt ${attempt}/${attempts} failed for ${route}: ${error.message}; retrying`,
+        );
+        await delay(config.retryDelay);
+        continue;
+      }
+      report.errors.push({ route, message: error.message, attempts: attempt });
+      log.error(`  failed ${route} after ${attempt} attempt(s): ${error.message}`);
+      return null;
+    }
+  }
+  return null;
+}
+
+// Creates the trace directory and returns the archive path for one failed attempt.
+async function prepareTraceFile(outputDir, route, attempt) {
+  const slug = route
+    .replace(/^\/+|\/+$/g, '')
+    .replace(/[^a-z0-9._-]+/gi, '-')
+    .toLowerCase();
+  const dir = path.join(outputDir, 'snappy-traces');
+  await fs.mkdir(dir, { recursive: true });
+  return path.join(dir, `${slug || 'root'}-${attempt}.zip`);
+}
+
+// Waits between retry attempts.
+function delay(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+/**
+ * Deterministic BFS crawl used when maxRoutes or routePriority is set. Each depth level
+ * is sorted, capped and rendered as a wave, so the accepted set never depends on which
+ * concurrent page happened to finish first.
+ */
+async function renderWaves({ config, accept, renderOne, roots, report, log }) {
+  let frontier = roots.map((item) => ({ ...item, depth: 0 }));
+  let depth = 0;
+  while (frontier.length > 0) {
+    const { results, errors } = await mapLimit(frontier, config.concurrency, async (item) => ({
+      item,
+      rendered: await renderOne(item),
+    }));
+    for (const { item, error } of errors) {
+      report.errors.push({ route: item.route, message: error.message });
+      log.error(`  pool error on ${item.route}: ${error.message}`);
+    }
+    const discovered = [];
+    for (const { rendered: renderedRoute } of results) {
+      if (!renderedRoute) continue;
+      if (!renderedRoute.routeConfig.crawl) continue;
+      if (!withinDepth(depth + 1, config.maxDepth)) continue;
+      discovered.push(
+        ...filterRoutes(renderedRoute.result.links, {
+          include: config.includeProvided ? config.include : [],
+          exclude: config.exclude,
+        }),
+      );
+    }
+    if (!withinDepth(depth + 1, config.maxDepth)) break;
+    const next = accept(sortRoutes(discovered, config));
+    frontier = next.map((item) => ({ ...item, depth: depth + 1 }));
+    depth += 1;
+  }
+}
+
+// Orders routes by routePriority first, then by path for a stable frontier.
+function sortRoutes(routes, config) {
+  return [...routes].sort((a, b) => sortRoute(a, b, config));
+}
+
+function sortRoute(a, b, config) {
+  const rank = (route) => {
+    const index = config.routePriority.findIndex((pattern) => matchesPattern(route, pattern));
+    return index === -1 ? Number.MAX_SAFE_INTEGER : index;
+  };
+  const difference = rank(a) - rank(b);
+  return difference === 0 ? a.localeCompare(b) : difference;
+}

package/src/core/csp.js

@@ -0,0 +1,127 @@
+import { createHash } from 'node:crypto';
+import { parse, serialize } from 'parse5';
+
+// Types that execute as JavaScript. A script with any other type is data, not code.
+const EXECUTABLE_TYPES = new Set([
+  '',
+  'text/javascript',
+  'application/javascript',
+  'application/ecmascript',
+  'text/ecmascript',
+  'module',
+]);
+const CSP_MARKER = 'data-snappy-csp';
+
+/**
+ * Adds a hash-based Content-Security-Policy meta tag covering every inline script and
+ * style in the final document. A policy this function wrote on an earlier run is
+ * replaced; a policy the app wrote is kept untouched, so the prerenderer never merges
+ * two policies silently.
+ */
+export function applyCspHashes(html) {
+  const document = parse(html);
+  const head = findHead(document);
+  if (!head) return { html, hashedScripts: 0, hashedStyles: 0, existingPolicy: false };
+  removeGeneratedPolicy(head);
+  const existing = findPolicyMeta(head);
+  if (existing) return { html, hashedScripts: 0, hashedStyles: 0, existingPolicy: true };
+
+  const scriptHashes = [];
+  const styleHashes = [];
+  walk(document, (node) => {
+    if (node.tagName === 'script' && isExecutableInline(node)) {
+      scriptHashes.push(hashOf(textContent(node)));
+    }
+    if (node.tagName === 'style') {
+      styleHashes.push(hashOf(textContent(node)));
+    }
+  });
+
+  const directives = [
+    `script-src 'self'${hashList(scriptHashes)}`,
+    `style-src 'self'${hashList(styleHashes)}`,
+  ];
+  head.childNodes.unshift(metaNode(directives.join('; ')));
+  return {
+    html: serialize(document),
+    hashedScripts: scriptHashes.length,
+    hashedStyles: styleHashes.length,
+    existingPolicy: false,
+  };
+}
+
+// A script is executable inline code when it has no src and no data type.
+function isExecutableInline(node) {
+  if (attribute(node, 'src').length > 0) return false;
+  const type = attribute(node, 'type').trim().toLowerCase();
+  return EXECUTABLE_TYPES.has(type);
+}
+
+// Renders the hash list, deduplicated and in document order.
+function hashList(hashes) {
+  return [...new Set(hashes)].map((hash) => ` '${hash}'`).join('');
+}
+
+// Base64 SHA-256, the form a CSP hash source uses.
+function hashOf(text) {
+  return `sha256-${createHash('sha256').update(text).digest('base64')}`;
+}
+
+function findHead(document) {
+  const html = (document.childNodes ?? []).find((node) => node.tagName === 'html');
+  return (html?.childNodes ?? []).find((node) => node.tagName === 'head') ?? null;
+}
+
+function findPolicyMeta(head) {
+  return (head.childNodes ?? []).some(
+    (node) =>
+      node.tagName === 'meta' &&
+      attribute(node, 'http-equiv').toLowerCase() === 'content-security-policy' &&
+      !hasAttribute(node, CSP_MARKER),
+  );
+}
+
+// Removes the policy this prerenderer wrote on an earlier run, so it is recomputed against
+// the current document instead of being mistaken for an app-owned policy.
+function removeGeneratedPolicy(head) {
+  head.childNodes = (head.childNodes ?? []).filter(
+    (node) => !(node.tagName === 'meta' && hasAttribute(node, CSP_MARKER)),
+  );
+}
+
+function metaNode(content) {
+  return {
+    nodeName: 'meta',
+    tagName: 'meta',
+    attrs: [
+      { name: 'http-equiv', value: 'Content-Security-Policy' },
+      { name: 'content', value: content },
+      { name: CSP_MARKER, value: '' },
+    ],
+    namespaceURI: 'http://www.w3.org/1999/xhtml',
+    childNodes: [],
+    parentNode: null,
+  };
+}
+
+function walk(node, visit) {
+  visit(node);
+  for (const child of node.childNodes ?? []) walk(child, visit);
+}
+
+function textContent(node) {
+  let text = '';
+  for (const child of node.childNodes ?? []) {
+    if (child.nodeName === '#text') text += child.value;
+    else text += textContent(child);
+  }
+  return text;
+}
+
+function attribute(node, name) {
+  return node.attrs?.find((entry) => entry.name === name)?.value ?? '';
+}
+
+function hasAttribute(node, name) {
+  return Boolean(node.attrs?.some((entry) => entry.name === name));
+}

package/src/core/frameworks.js

@@ -0,0 +1,94 @@
+const REACT = (container) =>
+  Object.keys(container).some(
+    (key) => key.startsWith('__reactContainer') || key === '_reactRootContainer',
+  );
+const VUE = (container) =>
+  Boolean(
+    container.__vue_app__ ||
+      container.__vueParentComponent ||
+      container._vnode ||
+      container.__vue__,
+  );
+const PREACT = (container) =>
+  Boolean(
+    container.__k ||
+      container._children ||
+      container.firstElementChild?.__k ||
+      container.firstElementChild?._children,
+  );
+const SVELTE = (container) =>
+  Boolean(container.__svelte_meta || container.querySelector?.('[data-svelte-h]'));
+
+const DETECTORS = { react: REACT, vue: VUE, preact: PREACT, svelte: SVELTE };
+const AUTO_ORDER = ['react', 'vue', 'preact', 'svelte'];
+
+/**
+ * Detects the framework that mounted into a container from its DOM markers. This is
+ * observation only: no framework is imported or executed by the prerenderer.
+ */
+export function detectFramework(container, requested = 'auto') {
+  if (!container) return { name: null, mounted: false };
+  if (requested !== 'auto') {
+    return { name: requested, mounted: Boolean(DETECTORS[requested]?.(container)) };
+  }
+  const markers = {
+    react: REACT(container),
+    vue: VUE(container),
+    preact: PREACT(container),
+    svelte: SVELTE(container),
+  };
+  return frameworkFromMarkers(markers, requested);
+}
+
+/**
+ * Selects a framework from a plain marker map produced inside the page, where DOM
+ * object identity is not available to Node.
+ */
+export function frameworkFromMarkers(markers, requested = 'auto') {
+  if (requested !== 'auto') {
+    return { name: requested, mounted: markers?.[requested] === true };
+  }
+  for (const name of AUTO_ORDER) {
+    if (markers?.[name] === true) return { name, mounted: true };
+  }
+  return { name: null, mounted: false };
+}
+
+const COMMON_PATTERNS = [
+  /react\.dev\/errors\/(418|419|421|422|423|424|425)/i,
+  /Minified React error #(418|419|421|422|423|424|425)\b/i,
+];
+
+const FRAMEWORK_PATTERNS = {
+  react: [
+    ...COMMON_PATTERNS,
+    /hydrat/i,
+    /text content did not match/i,
+    /prop [`"].+?[`"] did not match/i,
+    /server-rendered HTML/i,
+    /did not expect server HTML/i,
+    /expected server HTML to contain/i,
+  ],
+  preact: [
+    ...COMMON_PATTERNS,
+    /hydrat/i,
+    /text content did not match/i,
+    /did not expect server HTML/i,
+  ],
+  vue: [/hydration (?:node|text|children) mismatch/i, /hydration-mismatch/i, /hydrat/i],
+  svelte: [/hydration_mismatch/i, /hydration failed/i],
+};
+
+/**
+ * Classifies a console or page error as a hydration failure for the detected framework.
+ * Patterns stay narrow so an unrelated application error is not treated as a build gate.
+ * With `auto`, every framework set is tried because the framework was not established.
+ */
+export function isHydrationError(message, framework = 'auto') {
+  const text = String(message);
+  const patterns =
+    framework === 'auto'
+      ? Object.values(FRAMEWORK_PATTERNS).flat()
+      : [...(FRAMEWORK_PATTERNS[framework] ?? [])];
+  return patterns.some((pattern) => pattern.test(text));
+}

package/src/core/hydration-diff.js

@@ -0,0 +1,93 @@
+import { parse } from 'parse5';
+
+const CONTAINER_IDS = new Set(['root', 'app', '__next']);
+const STABLE_ATTRIBUTES = ['id', 'class', 'href', 'src', 'type', 'role', 'name'];
+const MAX_NODES = 5000;
+const MAX_DEPTH = 20;
+
+/**
+ * Finds the first meaningful difference between the prerendered document and the live
+ * post-boot container. Bounded so a large app cannot turn verification into a memory
+ * problem.
+ */
+export function compareContainers(expectedHtml, liveHtml) {
+  const expected = containerOf(parse(expectedHtml));
+  const actual = containerOf(parse(liveHtml));
+  if (!expected || !actual) return null;
+  const state = { visited: 0 };
+  return diffNode(expected, actual, 'body', 0, state);
+}
+
+// Finds the app container in a parsed document, falling back to the body's first child.
+function containerOf(document) {
+  const html = (document.childNodes ?? []).find((node) => node.tagName === 'html');
+  const body = (html?.childNodes ?? []).find((node) => node.tagName === 'body');
+  if (!body) return null;
+  const container = (body.childNodes ?? []).find(
+    (node) => node.tagName && CONTAINER_IDS.has(attribute(node, 'id')),
+  );
+  return container ?? (body.childNodes ?? []).find((node) => node.tagName) ?? null;
+}
+
+// Compares one node pair and descends into the first differing child.
+function diffNode(expected, actual, path, depth, state) {
+  if (state.visited > MAX_NODES || depth > MAX_DEPTH) return null;
+  state.visited += 1;
+  const here = `${path}>${expected.tagName ?? expected.nodeName}`;
+  if (!actual || expected.tagName !== actual.tagName) {
+    return {
+      path,
+      kind: 'tag',
+      expected: expected.tagName ?? expected.nodeName,
+      actual: actual?.tagName ?? actual?.nodeName ?? null,
+    };
+  }
+  for (const name of STABLE_ATTRIBUTES) {
+    const left = attribute(expected, name);
+    const right = attribute(actual, name);
+    if (left !== right)
+      return { path: here, kind: 'attribute', name, expected: left, actual: right };
+  }
+  const expectedChildren = elementChildren(expected);
+  const actualChildren = elementChildren(actual);
+  for (let index = 0; index < expectedChildren.length; index += 1) {
+    const child = expectedChildren[index];
+    const other = actualChildren[index];
+    if (!other) {
+      return {
+        path: here,
+        kind: 'missing',
+        expected: child.tagName ?? child.nodeName,
+        actual: null,
+      };
+    }
+    const difference = diffNode(child, other, `${here}[${index}]`, depth + 1, state);
+    if (difference) return difference;
+  }
+  if (actualChildren.length > expectedChildren.length) {
+    const extra = actualChildren[expectedChildren.length];
+    return { path: here, kind: 'extra', expected: null, actual: extra.tagName ?? extra.nodeName };
+  }
+  const expectedText = directText(expected);
+  const actualText = directText(actual);
+  if (expectedText !== actualText) {
+    return { path: here, kind: 'text', expected: expectedText, actual: actualText };
+  }
+  return null;
+}
+
+function elementChildren(node) {
+  return (node.childNodes ?? []).filter((child) => child.tagName);
+}
+
+function directText(node) {
+  return (node.childNodes ?? [])
+    .filter((child) => child.nodeName === '#text')
+    .map((child) => child.value)
+    .join('')
+    .trim();
+}
+
+function attribute(node, name) {
+  return node.attrs?.find((entry) => entry.name === name)?.value ?? '';
+}