snappy-prerender 2.6.0 → 2.7.0

package/src/core/init.js

@@ -2,6 +2,7 @@ import fs from 'node:fs/promises';
 import { createRequire } from 'node:module';
 import path from 'node:path';
 import { pathToFileURL } from 'node:url';
+import { DEFAULTS } from './config.js';
 import { normaliseRoute } from './routes.js';
 
 export const CONFIG_FILE = 'snappy.config.js';
@@ -15,6 +16,47 @@ const RUNNERS = [
   ['package-lock.json', 'npm'],
 ];
 
+// react-snap options whose name differs here, mapped to this package's option.
+const REACT_SNAP_RENAMED = {
+  source: 'sourceDir',
+  puppeteerArgs: 'browserArgs',
+  puppeteerIgnoreHTTPSErrors: 'ignoreHTTPSErrors',
+  skipThirdPartyRequests: 'blockThirdParty',
+  fixInsertRule: 'captureRuntimeStyles',
+  fixFormFields: 'captureFormState',
+  http2PushManifest: 'preloadManifest',
+};
+
+// react-snap options that keep their name and meaning.
+const REACT_SNAP_PASSTHROUGH = new Set([
+  'include',
+  'exclude',
+  'crawl',
+  'concurrency',
+  'timeout',
+  'waitFor',
+  'inlineCss',
+  'minifyHtml',
+  'minifyCss',
+  'cacheAjaxRequests',
+  'preconnectThirdParty',
+  'preloadImages',
+  'removeBlobs',
+  'removeStyleTags',
+  'removeScriptTags',
+  'asyncScriptTags',
+  'userAgent',
+  'viewport',
+  'saveAs',
+  'destination',
+]);
+
+// react-snap options that cannot be carried over, with the reason shown to the user.
+const REACT_SNAP_IGNORED = {
+  port: 'the prerenderer serves the build itself on an ephemeral port',
+  sourceMaps: 'react-snap declares it but never uses it',
+};
+
 /**
  * Reads what the project already says about itself: the framework that decides the build
  * directory, whether Vite is in use, which package runner it uses, and whether this package
@@ -26,13 +68,61 @@ export async function detectProjectDefaults(dir = process.cwd()) {
   const usesVite = Boolean(dependencies.vite) || (await anyExists(dir, VITE_CONFIGS));
   const usesCreateReactApp = Boolean(dependencies['react-scripts']) && !usesVite;
   const vite = usesVite ? await detectViteConfig(dir) : null;
-  return {
+  const detected = {
     sourceDir: vite?.sourceDir ?? (usesCreateReactApp ? 'build' : 'dist'),
     base: vite?.base ?? homepageBase(manifest?.homepage) ?? '/',
     usesVite,
     runner: await detectRunner(dir),
     installed: PACKAGE_NAME in dependencies,
   };
+  const reactSnap = mapReactSnapOptions(manifest?.reactSnap);
+  if (reactSnap) detected.reactSnap = reactSnap;
+  return detected;
+}
+
+/**
+ * Maps a legacy react-snap block onto this package's options. The result lists what
+ * carried over, what was deliberately dropped and what was not recognised, so the
+ * wizard can explain the migration instead of guessing.
+ */
+export function mapReactSnapOptions(block) {
+  if (typeof block !== 'object' || block === null || Array.isArray(block)) return null;
+  const values = {};
+  const mapping = [];
+  const ignored = [];
+  const unknown = [];
+  for (const [key, value] of Object.entries(block)) {
+    if (Object.hasOwn(REACT_SNAP_IGNORED, key)) {
+      ignored.push({ key, reason: REACT_SNAP_IGNORED[key] });
+      continue;
+    }
+    const target = REACT_SNAP_RENAMED[key] ?? (REACT_SNAP_PASSTHROUGH.has(key) ? key : null);
+    if (target === null) {
+      unknown.push(key);
+      continue;
+    }
+    // JSON cannot carry functions or RegExp, but a hand-written manifest may.
+    if (!isSerialisable(value)) {
+      unknown.push(`${key} (unsupported value)`);
+      continue;
+    }
+    const mappedValue =
+      target === 'include' && typeof value === 'string' ? parseRouteList(value) : value;
+    values[target] = mappedValue;
+    mapping.push({ from: key, to: target, value: mappedValue });
+  }
+  return { values, mapping, ignored, unknown };
+}
+
+// Reports whether a value can be written into a JavaScript config file.
+function isSerialisable(value) {
+  if (value === null) return true;
+  if (['string', 'number', 'boolean'].includes(typeof value)) return true;
+  if (Array.isArray(value)) return value.every(isSerialisable);
+  if (typeof value === 'object' && Object.getPrototypeOf(value) === Object.prototype) {
+    return Object.values(value).every(isSerialisable);
+  }
+  return false;
 }
 
 /**
@@ -87,46 +177,74 @@ export function parseRouteList(input) {
   return routes.length > 0 ? routes : ['/'];
 }
 
+/**
+ * Turns the wizard's answers into the options object the generated config will export.
+ * Only values that change a default are included, and only options the resolver knows.
+ */
+export function configValuesFromAnswers(answers = {}) {
+  const values = { sourceDir: answers.sourceDir ?? 'dist' };
+  if (answers.base && answers.base !== '/') values.base = answers.base;
+  // A supplied include list acts as an allowlist, so writing the default would stop discovery.
+  const includeValue = answers.include ?? ['/'];
+  const include = parseRouteList(
+    Array.isArray(includeValue) ? includeValue.join(',') : includeValue,
+  );
+  const includeIsDefault = include.length === 1 && include[0] === '/';
+  if (!includeIsDefault) values.include = include;
+  if (answers.crawl === false) values.crawl = false;
+  if (answers.csp === 'strict' || answers.csp === 'hash') values.csp = answers.csp;
+  if (answers.externalScripts === true) values.externalScripts = true;
+  if (answers.criticalCssPreload !== undefined && answers.criticalCssPreload !== 'media') {
+    values.criticalCssPreload = answers.criticalCssPreload;
+  }
+
+  const metadata = METADATA_KEYS.filter((key) => answers[key]).map((key) => [key, answers[key]]);
+  if (metadata.length > 0) values.metadata = Object.fromEntries(metadata);
+
+  for (const [key, value] of Object.entries(answers.extraOptions ?? {})) {
+    if (!Object.hasOwn(DEFAULTS, key) || key === 'metadata') continue;
+    if (Object.hasOwn(values, key)) continue;
+    if (!isSerialisable(value)) continue;
+    values[key] = value;
+  }
+  return values;
+}
+
 /**
  * Renders the config file for the wizard's answers, writing only the options the answers
  * actually set so every line in the file does something.
  */
 export function renderConfigFile(answers = {}) {
+  const values = configValuesFromAnswers(answers);
   const lines = [
     '// Created by `snappy-prerender init`. Every option is documented in the snappy-prerender README.',
     'export default {',
-    `  sourceDir: ${quote(answers.sourceDir ?? 'dist')},`,
   ];
-  if (answers.base && answers.base !== '/') lines.push(`  base: ${quote(answers.base)},`);
-  // A supplied include list acts as an allowlist, so writing the default would stop discovery.
-  // It is only written when the answer actually narrows the set of routes.
-  const include = parseRouteList((answers.include ?? ['/']).join(','));
-  const includeIsDefault = include.length === 1 && include[0] === '/';
-  if (!includeIsDefault) lines.push(`  include: [${include.map(quote).join(', ')}],`);
-  if (answers.crawl === false) lines.push('  crawl: false,');
-  // The CSP dial and the parts it was tuned with, so a strict-CSP project keeps the choice.
-  if (answers.csp === 'strict') lines.push('  csp: "strict",');
-  if (answers.externalScripts === true) lines.push('  externalScripts: true,');
-  if (answers.criticalCssPreload !== undefined && answers.criticalCssPreload !== 'media') {
-    // A boolean stays a literal; a mode name is quoted.
-    const value =
-      typeof answers.criticalCssPreload === 'boolean'
-        ? String(answers.criticalCssPreload)
-        : quote(answers.criticalCssPreload);
-    lines.push(`  criticalCssPreload: ${value},`);
-  }
-
-  const metadata = METADATA_KEYS.filter((key) => answers[key]).map((key) => [key, answers[key]]);
-  if (metadata.length > 0) {
+  for (const [key, value] of Object.entries(values)) {
+    if (key !== 'metadata') {
+      lines.push(`  ${key}: ${serializeValue(value)},`);
+      continue;
+    }
     lines.push('  metadata: {');
-    for (const [key, value] of metadata) lines.push(`    ${key}: ${quote(value)},`);
+    for (const [metadataKey, metadataValue] of Object.entries(value)) {
+      lines.push(`    ${metadataKey}: ${serializeValue(metadataValue)},`);
+    }
     lines.push('  },');
   }
-
   lines.push('};', '');
   return lines.join('\n');
 }
 
+// Writes a JSON-compatible value as JavaScript source with a stable, readable shape.
+function serializeValue(value) {
+  if (Array.isArray(value)) return `[${value.map(serializeValue).join(', ')}]`;
+  if (value !== null && typeof value === 'object') {
+    const entries = Object.entries(value).map(([key, entry]) => `${key}: ${serializeValue(entry)}`);
+    return `{ ${entries.join(', ')} }`;
+  }
+  return JSON.stringify(value);
+}
+
 /**
  * Writes the config file into the project, refusing to replace an existing one unless the
  * caller asked for it, so a wizard run can never silently destroy hand-written settings.
@@ -140,11 +258,6 @@ export async function writeConfigFile({ dir, contents, force = false }) {
   return target;
 }
 
-// Quotes a string for the generated module.
-function quote(value) {
-  return JSON.stringify(String(value));
-}
-
 // Reads a JSON file, treating a missing or malformed file as absent.
 async function readJson(file) {
   try {

package/src/core/normalize.js

@@ -117,6 +117,13 @@ function shouldStripMetadata(node, options, stats) {
 function shouldRemove(node, stats, options) {
   if (node.tagName === 'script' && hasAttribute(node, 'data-snappy-meta', null)) return false;
   if (node.tagName === 'style' && hasAttribute(node, 'data-snappy-inline', null)) return false;
+  if (
+    node.tagName === 'meta' &&
+    attributeValue(node, 'name').toLowerCase() === 'prerender-status-code'
+  ) {
+    stats.removedElements += 1;
+    return true;
+  }
   const reason =
     (node.tagName === 'style' && hasAttribute(node, 'id', FREEZE_STYLE_ID)) ||
     hasAttribute(node, REMOVE_ATTRIBUTE, null) ||

package/src/core/page-meta.js

@@ -0,0 +1,108 @@
+import { parse } from 'parse5';
+
+const DATA_URL = /^(?:[a-z][a-z0-9+.-]*:|\/\/|#)/i;
+const SVG_NAMESPACE = 'http://www.w3.org/2000/svg';
+
+/**
+ * Reads the metadata and structural markers from a final HTML document. This runs
+ * after post-processing so a transform that changes the head is reflected in the
+ * sitemap, the audit and the report.
+ */
+export function collectPageMeta(html) {
+  const document = parse(html);
+  const meta = {
+    title: null,
+    titleCount: 0,
+    description: null,
+    canonical: null,
+    robots: null,
+    noindex: false,
+    h1Count: 0,
+    alternates: [],
+    images: [],
+    inlineStyleAttributes: 0,
+  };
+  const visitedLinks = new Set();
+
+  const walk = (node) => {
+    // An SVG <title> names a drawing, not the document, so it must not count as a page title.
+    if (node.tagName === 'title' && node.namespaceURI !== SVG_NAMESPACE) {
+      meta.titleCount += 1;
+      if (meta.title === null) meta.title = textContent(node);
+    }
+    if (node.tagName === 'meta') {
+      const name = attribute(node, 'name').toLowerCase();
+      const content = attribute(node, 'content');
+      if (name === 'description' && meta.description === null) meta.description = content;
+      if (name === 'robots') {
+        meta.robots ??= content;
+        if (/\b(noindex|none)\b/i.test(content)) meta.noindex = true;
+      }
+    }
+    if (node.tagName === 'link') {
+      const rel = attribute(node, 'rel').toLowerCase();
+      if (rel === 'canonical' && meta.canonical === null) meta.canonical = attribute(node, 'href');
+      if (rel.split(/\s+/).includes('alternate')) {
+        const hreflang = attribute(node, 'hreflang');
+        const href = attribute(node, 'href');
+        if (hreflang && href) {
+          const key = `${hreflang}|${href}`;
+          if (!visitedLinks.has(key)) {
+            visitedLinks.add(key);
+            meta.alternates.push({ hreflang, href });
+          }
+        }
+      }
+    }
+    if (node.tagName === 'h1') meta.h1Count += 1;
+    if (node.tagName === 'img') {
+      addImage(meta, attribute(node, 'src'));
+      for (const candidate of srcsetCandidates(attribute(node, 'srcset')))
+        addImage(meta, candidate);
+    }
+    if (node.tagName === 'source' && node.parentNode?.tagName === 'picture') {
+      addImage(meta, attribute(node, 'src'));
+      for (const candidate of srcsetCandidates(attribute(node, 'srcset')))
+        addImage(meta, candidate);
+    }
+    for (const attr of node.attrs ?? []) {
+      if (attr.name === 'style' && attr.value.trim().length > 0) meta.inlineStyleAttributes += 1;
+    }
+    for (const child of node.childNodes ?? []) walk(child);
+  };
+
+  walk(document);
+  return meta;
+}
+
+// Adds a usable image URL once. Data URLs have no host to list in a sitemap.
+function addImage(meta, value) {
+  if (!value || DATA_URL.test(value) || meta.images.includes(value)) return;
+  meta.images.push(value);
+}
+
+// Extracts URL candidates from a srcset, ignoring their density or width descriptors.
+function srcsetCandidates(value) {
+  if (!value) return [];
+  return value
+    .split(',')
+    .map((candidate) => candidate.trim().split(/\s+/)[0])
+    .filter(
+      (candidate) => candidate.length > 0 && !DATA_URL.test(candidate) && !/[<>]/.test(candidate),
+    );
+}
+
+// Reads an attribute value, defaulting to an empty string.
+function attribute(node, name) {
+  return node.attrs?.find((entry) => entry.name === name)?.value ?? '';
+}
+
+// Concatenates the text beneath a node.
+function textContent(node) {
+  let text = '';
+  for (const child of node.childNodes ?? []) {
+    if (child.nodeName === '#text') text += child.value;
+    else text += textContent(child);
+  }
+  return text.trim();
+}

package/src/core/preload-links.js

@@ -0,0 +1,120 @@
+import path from 'node:path';
+import { parse, serialize } from 'parse5';
+import { toPublicPath } from './routes.js';
+
+/**
+ * Injects static preload hints into the final HTML. Static hosts cannot send the
+ * preload-manifest headers, so this is the page-level equivalent for runtime chunks,
+ * fonts and the LCP image.
+ */
+export function injectPreloadLinks(
+  html,
+  {
+    scripts = [],
+    fonts = new Map(),
+    lcpImage = null,
+    base = '/',
+    ignoreForPreload = [],
+    skipScripts = new Set(),
+  } = {},
+) {
+  const document = parse(html);
+  const head = findHead(document);
+  if (!head) return html;
+  const existingScripts = new Set();
+  const hints = new Set();
+
+  const walk = (node) => {
+    if (node.tagName === 'script') {
+      const src = attribute(node, 'src');
+      if (src) existingScripts.add(src);
+    }
+    if (node.tagName === 'link') {
+      const href = attribute(node, 'href');
+      if (href && attribute(node, 'rel').split(/\s+/).includes('preload')) hints.add(href);
+    }
+    for (const child of node.childNodes ?? []) walk(child);
+  };
+  walk(document);
+
+  const ignored = new Set(ignoreForPreload);
+  let added = 0;
+  const add = (element, href) => {
+    if (hints.has(href) || ignored.has(path.posix.basename(href))) return;
+    hints.add(href);
+    head.childNodes.push(element);
+    added += 1;
+  };
+
+  for (const script of scripts) {
+    const href = toPublicPath(script, base);
+    if (existingScripts.has(href) || skipScripts.has(href)) continue;
+    add(
+      linkNode([
+        { name: 'rel', value: 'modulepreload' },
+        { name: 'href', value: href },
+      ]),
+      href,
+    );
+  }
+  for (const [font, contentType] of fonts) {
+    const href = toPublicPath(font, base);
+    const attrs = [
+      { name: 'rel', value: 'preload' },
+      { name: 'as', value: 'font' },
+      { name: 'href', value: href },
+      { name: 'crossorigin', value: '' },
+    ];
+    if (contentType) attrs.push({ name: 'type', value: contentType });
+    add(linkNode(attrs), href);
+  }
+  if (lcpImage) {
+    const href = toPublicPath(lcpImage, base);
+    add(
+      linkNode([
+        { name: 'rel', value: 'preload' },
+        { name: 'as', value: 'image' },
+        { name: 'href', value: href },
+        { name: 'fetchpriority', value: 'high' },
+      ]),
+      href,
+    );
+  }
+  return added > 0 ? serialize(document) : html;
+}
+
+// Collects script src values from a document, used to tell a script a transform
+// removed from a chunk the app loads at runtime.
+export function collectScriptSources(html) {
+  const document = parse(html);
+  const sources = new Set();
+  const walk = (node) => {
+    if (node.tagName === 'script') {
+      const src = attribute(node, 'src');
+      if (src) sources.add(src);
+    }
+    for (const child of node.childNodes ?? []) walk(child);
+  };
+  walk(document);
+  return sources;
+}
+
+function findHead(document) {
+  const html = (document.childNodes ?? []).find((node) => node.tagName === 'html');
+  return (html?.childNodes ?? []).find((node) => node.tagName === 'head') ?? null;
+}
+
+function linkNode(attrs) {
+  return {
+    nodeName: 'link',
+    tagName: 'link',
+    attrs,
+    namespaceURI: 'http://www.w3.org/1999/xhtml',
+    childNodes: [],
+    parentNode: null,
+  };
+}
+
+function attribute(node, name) {
+  return node.attrs?.find((entry) => entry.name === name)?.value ?? '';
+}

package/src/core/render.js

@@ -2,7 +2,7 @@ import { captureFormState } from './form-state.js';
 import { inlineStylesheets } from './inline-css.js';
 import { isPageEmpty } from './page-content.js';
 import { createResourceCollector } from './resources.js';
-import { toAppRoute, toServerUrl } from './routes.js';
+import { normaliseRoute, toAppRoute, toServerUrl } from './routes.js';
 import { captureRuntimeStyles } from './runtime-styles.js';
 import { freezeAnimations, scrollThroughPage, settlePage, trackNetwork } from './settle.js';
 import { separateTextNodes } from './text-separators.js';
@@ -26,7 +26,14 @@ function escapeForScript(value) {
  * storage from leaking between routes. When a screenshot path is supplied the route
  * is captured as an image instead of HTML.
  */
-export async function renderRoute({ browser, origin, route, config, screenshot = false }) {
+export async function renderRoute({
+  browser,
+  origin,
+  route,
+  config,
+  screenshot = false,
+  traceFile = null,
+}) {
   const context = await browser.newContext({
     viewport: config.viewport,
     storageState: config.storageState ?? undefined,
@@ -35,6 +42,11 @@ export async function renderRoute({ browser, origin, route, config, screenshot =
     reducedMotion: config.freezeAnimations ? 'reduce' : 'no-preference',
     serviceWorkers: 'block',
   });
+  let tracing = false;
+  if (traceFile) {
+    await context.tracing.start({ screenshots: false, snapshots: true, sources: false });
+    tracing = true;
+  }
   const page = await context.newPage();
   const pageErrors = [];
   const consoleErrors = [];
@@ -78,19 +90,51 @@ export async function renderRoute({ browser, origin, route, config, screenshot =
     maxCachedBytes: config.maxCachedBytes,
   });
   await collector.install(page);
+  await page.addInitScript(() => {
+    window.__snappyLcp = [];
+    try {
+      const observer = new PerformanceObserver((list) => {
+        for (const entry of list.getEntries()) {
+          const element = entry.element;
+          window.__snappyLcp.push({
+            url: entry.url || element?.currentSrc || element?.src || null,
+          });
+        }
+      });
+      observer.observe({ type: 'largest-contentful-paint', buffered: true });
+    } catch {
+      // Older engines simply do not report LCP; preload injection falls back to no hint.
+    }
+  });
   const tracker = trackNetwork(page);
   try {
     const response = await page.goto(toServerUrl(route, config.base, origin), {
       waitUntil: 'domcontentloaded',
       timeout: config.timeout,
     });
+    const notFoundRoute = config.notFoundRoute ? normaliseRoute(config.notFoundRoute) : null;
+    const isNotFoundRoute = notFoundRoute !== null && normaliseRoute(route) === notFoundRoute;
     if (response && response.status() >= 400) {
-      throw new Error(
-        `route responded with HTTP ${response.status()}${response.statusText() ? ` ${response.statusText()}` : ''}`,
-      );
+      // A configured 404 route may legitimately answer with HTTP 404; every other status
+      // is a failure, and the marker cannot be read before the navigation completes.
+      if (!(response.status() === 404 && isNotFoundRoute)) {
+        const error = new Error(
+          `route responded with HTTP ${response.status()}${response.statusText() ? ` ${response.statusText()}` : ''}`,
+        );
+        // A server error can be transient and is retried; a client error is the app's own.
+        error.retryable = response.status() >= 500;
+        throw error;
+      }
     }
     await settlePage(page, config, tracker);
     const empty = await isPageEmpty(page);
+    const statusCode = await page.evaluate(() => {
+      const marker = document.querySelector('meta[name="prerender-status-code"]');
+      if (!marker) return null;
+      const value = Number.parseInt(marker.getAttribute('content') ?? '', 10);
+      return Number.isFinite(value) ? value : null;
+    });
+    const finalUrl = page.url();
 
     if (config.captureRuntimeStyles) await captureRuntimeStyles(page);
     if (config.captureFormState) await captureFormState(page);
@@ -106,6 +150,16 @@ export async function renderRoute({ browser, origin, route, config, screenshot =
     if (metadataDefaults) await injectMetadataDefaults(page, metadataDefaults);
     if (config.freezeAnimations) await freezeAnimations(page);
     if (config.scrollToBottom) await scrollThroughPage(page, config);
+    const lcpUrl = await page.evaluate(() => window.__snappyLcp?.at(-1)?.url ?? null);
+    let lcpImage = null;
+    if (lcpUrl) {
+      try {
+        const parsed = new URL(lcpUrl);
+        if (parsed.origin === origin) lcpImage = `${parsed.pathname}${parsed.search}`;
+      } catch {
+        // Ignore malformed performance entries.
+      }
+    }
     await separateTextNodes(page);
 
     const links = await collectLinks(page, config.base, origin);
@@ -126,6 +180,10 @@ export async function renderRoute({ browser, origin, route, config, screenshot =
         stateScript,
         screenshot: buffer,
         empty,
+        statusCode,
+        httpStatus: response?.status() ?? null,
+        finalUrl,
+        lcpImage,
       };
     }
 
@@ -141,13 +199,37 @@ export async function renderRoute({ browser, origin, route, config, screenshot =
       inlineCssSkipped,
       stateScript,
       empty,
+      statusCode,
+      httpStatus: response?.status() ?? null,
+      finalUrl,
+      lcpImage,
     };
+  } catch (error) {
+    error.retryable ??= isRetryableRenderError(error);
+    if (tracing) {
+      tracing = false;
+      await context.tracing.stop({ path: traceFile }).then(
+        () => {
+          error.tracePath = traceFile;
+        },
+        // A failed trace must never replace the render error that explains the build failure.
+        () => {},
+      );
+    }
+    throw error;
   } finally {
     tracker.stop();
+    if (tracing) await context.tracing.stop().catch(() => {});
     await context.close();
   }
 }
 
+// Navigation and network failures are transient; a bad selector or an HTTP status is not.
+function isRetryableRenderError(error) {
+  if (error?.name === 'TimeoutError') return true;
+  return /net::ERR_|timed? ?out|ECONNRESET|ECONNREFUSED|socket hang up/i.test(error?.message ?? '');
+}
+
 /**
  * Injects captured JSON responses as window.snapStore and anything the app exposes
  * through window.snapSaveState, before the first script runs. The client can then