snappy-prerender 2.6.0 → 2.8.0

package/src/core/crawl.js

@@ -0,0 +1,362 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import {
+  activeLocale,
+  isLocalePrefixedRoute,
+  localeByCode,
+  localeRoute,
+  stripLocalePrefix,
+} from './locales.js';
+import { renderRoute } from './render.js';
+import { resolveRouteConfig } from './route-config.js';
+import {
+  crawlRoots,
+  filterRoutes,
+  isIncludedRoute,
+  matchesPattern,
+  normaliseRoute,
+  withinDepth,
+} from './routes.js';
+import { createPool, mapLimit } from './scheduler.js';
+
+/**
+ * Runs the crawl and render pool. Each rendered page contributes its output, any new
+ * same-origin routes, and the resources recorded for the link-hint features.
+ */
+export async function renderAllRoutes({
+  browser,
+  server,
+  config,
+  log,
+  report,
+  outputDir,
+  sourceSeeds = [],
+  caches = null,
+  localePlan = null,
+}) {
+  const visited = new Set();
+  const scheduled = new Set();
+  const rendered = new Map();
+  const routeLocales = new Map();
+  const cap = config.maxRoutes ?? Number.POSITIVE_INFINITY;
+  let nextOrder = 0;
+  const defaultLocale = localePlan ? localeByCode(localePlan, localePlan.default) : null;
+  const notFoundRoute = config.notFoundRoute ? normaliseRoute(config.notFoundRoute) : null;
+
+  // Expands a base route into every locale variant it publishes. The not-found route is
+  // deliberately not localized: a host needs one 404 document.
+  const variantsFor = (route) => {
+    if (!localePlan) return [{ route, locale: null, baseRoute: route }];
+    if (localePlan.mode === 'single') {
+      return [{ route, locale: activeLocale(localePlan), baseRoute: route }];
+    }
+    const baseRoute = defaultLocale ? stripLocalePrefix(route, defaultLocale) : route;
+    const isNotFound = notFoundRoute !== null && normaliseRoute(baseRoute) === notFoundRoute;
+    const variants = [];
+    for (const locale of localePlan.list) {
+      if (isNotFound && locale !== defaultLocale) continue;
+      const localized = isNotFound
+        ? normaliseRoute(config.notFoundRoute)
+        : localeRoute(locale, baseRoute);
+      variants.push({ route: localized, locale, baseRoute });
+    }
+    return variants;
+  };
+
+  // Queues routes up to maxRoutes, recording exactly which candidates did not fit.
+  // A seed used only to reach wildcard routes is given output: false and is rendered
+  // for its links without being written.
+  const accept = (routes, outputFor = () => true) => {
+    const accepted = [];
+    const dropped = [];
+    for (const route of routes) {
+      if (isLocalePrefixedRoute(route, localePlan)) continue;
+      for (const variant of variantsFor(route)) {
+        if (scheduled.has(variant.route)) continue;
+        if (scheduled.size >= cap) {
+          dropped.push(variant.route);
+          continue;
+        }
+        scheduled.add(variant.route);
+        routeLocales.set(variant.route, variant);
+        accepted.push({
+          route: variant.route,
+          output: outputFor(variant.baseRoute),
+          order: nextOrder,
+          locale: variant.locale,
+          baseRoute: variant.baseRoute,
+        });
+        nextOrder += 1;
+      }
+    }
+    if (dropped.length > 0) {
+      const previous = report.truncated?.droppedRoutes ?? [];
+      report.truncated = {
+        limit: config.maxRoutes,
+        routes: scheduled.size,
+        droppedRoutes: [...new Set([...previous, ...dropped])].sort(),
+      };
+    }
+    return accepted;
+  };
+
+  const renderOne = async ({
+    route,
+    output,
+    depth = 0,
+    order = null,
+    locale = null,
+    baseRoute = route,
+  }) => {
+    if (visited.has(route)) return null;
+    visited.add(route);
+    log.debug(`rendering ${route}${output ? '' : ' as a crawl seed'}`);
+    const routeConfig = resolveRouteConfig(config, route);
+    const result = await renderWithRetries({
+      browser,
+      server,
+      route,
+      routeConfig,
+      config,
+      log,
+      report,
+      outputDir,
+      screenshot: config.saveAs !== 'html',
+      caches,
+      locale,
+    });
+    if (!result) return null;
+    result.depth = depth;
+    result.order = order;
+    try {
+      const notFoundRoute = config.notFoundRoute ? normaliseRoute(config.notFoundRoute) : null;
+      const isNotFoundRoute = notFoundRoute !== null && normaliseRoute(baseRoute) === notFoundRoute;
+      if (result.statusCode === 404 && !isNotFoundRoute) {
+        report.errors.push({ route, message: 'returned 404' });
+        log.error(`  route ${route} returned 404`);
+        return null;
+      }
+      if (isNotFoundRoute) {
+        result.notFound = true;
+        if (!report.notFoundRoutes.includes(route)) report.notFoundRoutes.push(route);
+        if (result.statusCode !== null && result.statusCode !== 404) {
+          log.warn(`  ignored unknown prerender-status-code ${result.statusCode} on ${route}`);
+        }
+      } else if (result.statusCode !== null) {
+        log.warn(`  ignored unknown prerender-status-code ${result.statusCode} on ${route}`);
+      }
+      if (output && result.empty) {
+        report.emptyPages.push(route);
+        if (config.failOnEmptyPage) {
+          report.errors.push({ route, message: 'rendered an empty page' });
+          log.error(`  empty page ${route}: no app content was found`);
+          return null;
+        }
+        log.warn(`  empty page ${route}: no app content was found`);
+      }
+      if (output) {
+        rendered.set(route, result);
+        report.routes.push(route);
+        for (const to of result.links) report.links.push({ from: route, to });
+      }
+      for (const message of result.pageErrors) {
+        report.pageErrors.push({ route, message });
+        log.warn(`  page error on ${route}: ${message}`);
+      }
+      for (const message of result.consoleErrors) {
+        report.consoleErrors.push({ route, message });
+      }
+      if (result.consoleErrors.length > 0) {
+        log.debug(`  console error on ${route}: ${result.consoleErrors[0]}`);
+        if (config.failOnConsoleError) {
+          report.errors.push({ route, message: `console error: ${result.consoleErrors[0]}` });
+        }
+      }
+      for (const failure of result.failedRequests) {
+        report.failedRequests.push({ route, ...failure });
+        const detail = failure.status ? `HTTP ${failure.status}` : failure.error;
+        log.warn(`  failed request on ${route}: ${failure.url} (${detail})`);
+      }
+      if (result.failedRequests.length > 0 && config.failOnFailedRequest) {
+        const first = result.failedRequests[0];
+        report.errors.push({
+          route,
+          message: `failed request: ${first.url}${first.status ? ` (HTTP ${first.status})` : ''}`,
+        });
+      }
+      return { result, routeConfig };
+    } catch (error) {
+      report.errors.push({ route, message: error.message });
+      log.error(`  failed ${route}: ${error.message}`);
+      return null;
+    }
+  };
+
+  const outputFor = (route) => !config.includeProvided || isIncludedRoute(route, config.include);
+  const candidates = [];
+  if (sourceSeeds.length > 0) candidates.push(...sourceSeeds);
+  if (sourceSeeds.length === 0 || config.includeProvided) {
+    candidates.push(...crawlRoots(config));
+  }
+  const roots = accept(
+    [...new Set(sortRoutes(candidates, config))].filter(
+      (route) => !isLocalePrefixedRoute(route, localePlan),
+    ),
+    outputFor,
+  );
+
+  const deterministic = config.maxRoutes !== null || config.routePriority.length > 0;
+  if (deterministic) {
+    await renderWaves({ config, accept, renderOne, roots, report, log });
+  } else {
+    const pool = createPool({
+      concurrency: config.concurrency,
+      handler: async ({ route, depth, output, order, locale, baseRoute }) => {
+        const renderedRoute = await renderOne({ route, output, depth, order, locale, baseRoute });
+        if (!renderedRoute) return;
+        const { result, routeConfig } = renderedRoute;
+        if (routeConfig.crawl && withinDepth(depth + 1, routeConfig.maxDepth)) {
+          const discovered = filterRoutes(result.links, {
+            include: config.includeProvided ? config.include : [],
+            exclude: config.exclude,
+          })
+            .filter((candidate) => !isLocalePrefixedRoute(candidate, localePlan))
+            .filter((candidate) => !scheduled.has(candidate));
+          const accepted = accept(sortRoutes(discovered, config));
+          pool.push(accepted.map((item) => ({ ...item, depth: depth + 1 })));
+        }
+      },
+    });
+    pool.push(roots.map((item) => ({ ...item, depth: 0 })));
+    const { errors } = await pool.run();
+    for (const { item, error } of errors) {
+      report.errors.push({ route: item.route, message: error.message });
+      log.error(`  pool error on ${item.route}: ${error.message}`);
+    }
+  }
+  return { rendered, routeLocales };
+}
+
+/**
+ * Renders one route with the configured retry policy. A retryable failure is retried
+ * after retryDelay; the final failure records the attempt count and, when traceOnError
+ * is on, a Playwright trace written under the output directory.
+ */
+async function renderWithRetries({
+  browser,
+  server,
+  route,
+  routeConfig,
+  config,
+  log,
+  report,
+  outputDir,
+  screenshot,
+  caches,
+  locale,
+}) {
+  const attempts = config.retries + 1;
+  for (let attempt = 1; attempt <= attempts; attempt += 1) {
+    const traceFile =
+      config.traceOnError && !config.dryRun && attempt === attempts
+        ? await prepareTraceFile(outputDir, route, attempt)
+        : null;
+    try {
+      const result = await renderRoute({
+        browser,
+        origin: server.origin,
+        route,
+        locale,
+        config: routeConfig,
+        screenshot,
+        traceFile,
+        caches,
+      });
+      if (attempt > 1) log.info(`  ${route}: succeeded on attempt ${attempt}`);
+      result.attempts = attempt;
+      return result;
+    } catch (error) {
+      if (error.tracePath) {
+        report.traces.push({ route, attempt, file: error.tracePath });
+        log.warn(`  trace for ${route}: ${error.tracePath}`);
+      }
+      if (error.retryable !== false && attempt < attempts) {
+        log.warn(
+          `  attempt ${attempt}/${attempts} failed for ${route}: ${error.message}; retrying`,
+        );
+        await delay(config.retryDelay);
+        continue;
+      }
+      report.errors.push({ route, message: error.message, attempts: attempt });
+      log.error(`  failed ${route} after ${attempt} attempt(s): ${error.message}`);
+      return null;
+    }
+  }
+  return null;
+}
+
+// Creates the trace directory and returns the archive path for one failed attempt.
+async function prepareTraceFile(outputDir, route, attempt) {
+  const slug = route
+    .replace(/^\/+|\/+$/g, '')
+    .replace(/[^a-z0-9._-]+/gi, '-')
+    .toLowerCase();
+  const dir = path.join(outputDir, 'snappy-traces');
+  await fs.mkdir(dir, { recursive: true });
+  return path.join(dir, `${slug || 'root'}-${attempt}.zip`);
+}
+
+// Waits between retry attempts.
+function delay(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+/**
+ * Deterministic BFS crawl used when maxRoutes or routePriority is set. Each depth level
+ * is sorted, capped and rendered as a wave, so the accepted set never depends on which
+ * concurrent page happened to finish first.
+ */
+async function renderWaves({ config, accept, renderOne, roots, report, log }) {
+  let frontier = roots.map((item) => ({ ...item, depth: 0 }));
+  let depth = 0;
+  while (frontier.length > 0) {
+    const { results, errors } = await mapLimit(frontier, config.concurrency, async (item) => ({
+      item,
+      rendered: await renderOne(item),
+    }));
+    for (const { item, error } of errors) {
+      report.errors.push({ route: item.route, message: error.message });
+      log.error(`  pool error on ${item.route}: ${error.message}`);
+    }
+    const discovered = [];
+    for (const { rendered: renderedRoute } of results) {
+      if (!renderedRoute) continue;
+      if (!renderedRoute.routeConfig.crawl) continue;
+      if (!withinDepth(depth + 1, config.maxDepth)) continue;
+      discovered.push(
+        ...filterRoutes(renderedRoute.result.links, {
+          include: config.includeProvided ? config.include : [],
+          exclude: config.exclude,
+        }),
+      );
+    }
+    if (!withinDepth(depth + 1, config.maxDepth)) break;
+    const next = accept(sortRoutes(discovered, config));
+    frontier = next.map((item) => ({ ...item, depth: depth + 1 }));
+    depth += 1;
+  }
+}
+
+// Orders routes by routePriority first, then by path for a stable frontier.
+function sortRoutes(routes, config) {
+  return [...routes].sort((a, b) => sortRoute(a, b, config));
+}
+
+function sortRoute(a, b, config) {
+  const rank = (route) => {
+    const index = config.routePriority.findIndex((pattern) => matchesPattern(route, pattern));
+    return index === -1 ? Number.MAX_SAFE_INTEGER : index;
+  };
+  const difference = rank(a) - rank(b);
+  return difference === 0 ? a.localeCompare(b) : difference;
+}

package/src/core/csp.js

@@ -0,0 +1,138 @@
+import { createHash } from 'node:crypto';
+import { parse, serialize } from 'parse5';
+
+// Types that execute as JavaScript. A script with any other type is data, not code.
+const EXECUTABLE_TYPES = new Set([
+  '',
+  'text/javascript',
+  'application/javascript',
+  'application/ecmascript',
+  'text/ecmascript',
+  'module',
+]);
+const CSP_MARKER = 'data-snappy-csp';
+
+/**
+ * Adds a hash-based Content-Security-Policy meta tag covering every inline script and
+ * style in the final document. A policy this function wrote on an earlier run is
+ * replaced; a policy the app wrote is kept untouched, so the prerenderer never merges
+ * two policies silently.
+ */
+export function applyCspHashes(html) {
+  const document = parse(html);
+  const result = applyCspHashesToDocument(document);
+  return {
+    html: result.existingPolicy ? html : serialize(document),
+    ...result,
+  };
+}
+
+/**
+ * Applies the hash policy to an already parsed document. Returns counts and whether a
+ * user-owned policy was left untouched.
+ */
+export function applyCspHashesToDocument(document) {
+  const head = findHead(document);
+  if (!head) return { hashedScripts: 0, hashedStyles: 0, existingPolicy: false };
+  removeGeneratedPolicy(head);
+  const existing = findPolicyMeta(head);
+  if (existing) return { hashedScripts: 0, hashedStyles: 0, existingPolicy: true };
+
+  const scriptHashes = [];
+  const styleHashes = [];
+  walk(document, (node) => {
+    if (node.tagName === 'script' && isExecutableInline(node)) {
+      scriptHashes.push(hashOf(textContent(node)));
+    }
+    if (node.tagName === 'style') {
+      styleHashes.push(hashOf(textContent(node)));
+    }
+  });
+
+  const directives = [
+    `script-src 'self'${hashList(scriptHashes)}`,
+    `style-src 'self'${hashList(styleHashes)}`,
+  ];
+  head.childNodes.unshift(metaNode(directives.join('; ')));
+  return {
+    hashedScripts: scriptHashes.length,
+    hashedStyles: styleHashes.length,
+    existingPolicy: false,
+  };
+}
+
+// A script is executable inline code when it has no src and no data type.
+function isExecutableInline(node) {
+  if (attribute(node, 'src').length > 0) return false;
+  const type = attribute(node, 'type').trim().toLowerCase();
+  return EXECUTABLE_TYPES.has(type);
+}
+
+// Renders the hash list, deduplicated and in document order.
+function hashList(hashes) {
+  return [...new Set(hashes)].map((hash) => ` '${hash}'`).join('');
+}
+
+// Base64 SHA-256, the form a CSP hash source uses.
+function hashOf(text) {
+  return `sha256-${createHash('sha256').update(text).digest('base64')}`;
+}
+
+function findHead(document) {
+  const html = (document.childNodes ?? []).find((node) => node.tagName === 'html');
+  return (html?.childNodes ?? []).find((node) => node.tagName === 'head') ?? null;
+}
+
+function findPolicyMeta(head) {
+  return (head.childNodes ?? []).some(
+    (node) =>
+      node.tagName === 'meta' &&
+      attribute(node, 'http-equiv').toLowerCase() === 'content-security-policy' &&
+      !hasAttribute(node, CSP_MARKER),
+  );
+}
+
+// Removes the policy this prerenderer wrote on an earlier run, so it is recomputed against
+// the current document instead of being mistaken for an app-owned policy.
+function removeGeneratedPolicy(head) {
+  head.childNodes = (head.childNodes ?? []).filter(
+    (node) => !(node.tagName === 'meta' && hasAttribute(node, CSP_MARKER)),
+  );
+}
+
+function metaNode(content) {
+  return {
+    nodeName: 'meta',
+    tagName: 'meta',
+    attrs: [
+      { name: 'http-equiv', value: 'Content-Security-Policy' },
+      { name: 'content', value: content },
+      { name: CSP_MARKER, value: '' },
+    ],
+    namespaceURI: 'http://www.w3.org/1999/xhtml',
+    childNodes: [],
+    parentNode: null,
+  };
+}
+
+function walk(node, visit) {
+  visit(node);
+  for (const child of node.childNodes ?? []) walk(child, visit);
+}
+
+function textContent(node) {
+  let text = '';
+  for (const child of node.childNodes ?? []) {
+    if (child.nodeName === '#text') text += child.value;
+    else text += textContent(child);
+  }
+  return text;
+}
+
+function attribute(node, name) {
+  return node.attrs?.find((entry) => entry.name === name)?.value ?? '';
+}
+
+function hasAttribute(node, name) {
+  return Boolean(node.attrs?.some((entry) => entry.name === name));
+}

package/src/core/destination.js

@@ -1,5 +1,8 @@
 import fs from 'node:fs/promises';
 import path from 'node:path';
+import { mapLimit } from './scheduler.js';
+
+const COPY_CONCURRENCY = 8;
 
 /**
  * Prepares the directory the run writes into. When a destination is configured the
@@ -58,13 +61,26 @@ export async function prepareDestination({
 async function copyTree(source, target) {
   await fs.mkdir(target, { recursive: true });
   const entries = await fs.readdir(source, { withFileTypes: true });
-  let skipped = 0;
-  for (const entry of entries) {
+  const { results, errors } = await mapLimit(entries, COPY_CONCURRENCY, async (entry) => {
     const from = path.join(source, entry.name);
     const to = path.join(target, entry.name);
-    if (entry.isDirectory()) skipped += await copyTree(from, to);
-    else if (entry.isFile()) await fs.copyFile(from, to);
-    else skipped += 1;
-  }
-  return skipped;
+    if (entry.isDirectory()) return copyTree(from, to);
+    if (!entry.isFile()) return 1;
+    const [sourceStats, targetStats] = await Promise.all([
+      fs.stat(from),
+      fs.stat(to).catch(() => null),
+    ]);
+    if (
+      targetStats?.isFile() &&
+      targetStats.size === sourceStats.size &&
+      targetStats.mtimeMs === sourceStats.mtimeMs
+    ) {
+      return 0;
+    }
+    await fs.copyFile(from, to);
+    await fs.utimes(to, sourceStats.atime, sourceStats.mtime);
+    return 0;
+  });
+  if (errors.length > 0) throw errors[0].error;
+  return results.reduce((total, skipped) => total + (skipped ?? 0), 0);
 }

package/src/core/frameworks.js

@@ -0,0 +1,94 @@
+const REACT = (container) =>
+  Object.keys(container).some(
+    (key) => key.startsWith('__reactContainer') || key === '_reactRootContainer',
+  );
+const VUE = (container) =>
+  Boolean(
+    container.__vue_app__ ||
+      container.__vueParentComponent ||
+      container._vnode ||
+      container.__vue__,
+  );
+const PREACT = (container) =>
+  Boolean(
+    container.__k ||
+      container._children ||
+      container.firstElementChild?.__k ||
+      container.firstElementChild?._children,
+  );
+const SVELTE = (container) =>
+  Boolean(container.__svelte_meta || container.querySelector?.('[data-svelte-h]'));
+
+const DETECTORS = { react: REACT, vue: VUE, preact: PREACT, svelte: SVELTE };
+const AUTO_ORDER = ['react', 'vue', 'preact', 'svelte'];
+
+/**
+ * Detects the framework that mounted into a container from its DOM markers. This is
+ * observation only: no framework is imported or executed by the prerenderer.
+ */
+export function detectFramework(container, requested = 'auto') {
+  if (!container) return { name: null, mounted: false };
+  if (requested !== 'auto') {
+    return { name: requested, mounted: Boolean(DETECTORS[requested]?.(container)) };
+  }
+  const markers = {
+    react: REACT(container),
+    vue: VUE(container),
+    preact: PREACT(container),
+    svelte: SVELTE(container),
+  };
+  return frameworkFromMarkers(markers, requested);
+}
+
+/**
+ * Selects a framework from a plain marker map produced inside the page, where DOM
+ * object identity is not available to Node.
+ */
+export function frameworkFromMarkers(markers, requested = 'auto') {
+  if (requested !== 'auto') {
+    return { name: requested, mounted: markers?.[requested] === true };
+  }
+  for (const name of AUTO_ORDER) {
+    if (markers?.[name] === true) return { name, mounted: true };
+  }
+  return { name: null, mounted: false };
+}
+
+const COMMON_PATTERNS = [
+  /react\.dev\/errors\/(418|419|421|422|423|424|425)/i,
+  /Minified React error #(418|419|421|422|423|424|425)\b/i,
+];
+
+const FRAMEWORK_PATTERNS = {
+  react: [
+    ...COMMON_PATTERNS,
+    /hydrat/i,
+    /text content did not match/i,
+    /prop [`"].+?[`"] did not match/i,
+    /server-rendered HTML/i,
+    /did not expect server HTML/i,
+    /expected server HTML to contain/i,
+  ],
+  preact: [
+    ...COMMON_PATTERNS,
+    /hydrat/i,
+    /text content did not match/i,
+    /did not expect server HTML/i,
+  ],
+  vue: [/hydration (?:node|text|children) mismatch/i, /hydration-mismatch/i, /hydrat/i],
+  svelte: [/hydration_mismatch/i, /hydration failed/i],
+};
+
+/**
+ * Classifies a console or page error as a hydration failure for the detected framework.
+ * Patterns stay narrow so an unrelated application error is not treated as a build gate.
+ * With `auto`, every framework set is tried because the framework was not established.
+ */
+export function isHydrationError(message, framework = 'auto') {
+  const text = String(message);
+  const patterns =
+    framework === 'auto'
+      ? Object.values(FRAMEWORK_PATTERNS).flat()
+      : [...(FRAMEWORK_PATTERNS[framework] ?? [])];
+  return patterns.some((pattern) => pattern.test(text));
+}