snappy-prerender 2.6.0 → 2.8.0

package/src/core/preload-links.js

@@ -0,0 +1,133 @@
+import path from 'node:path';
+import { parse, serialize } from 'parse5';
+import { toPublicPath } from './routes.js';
+
+/**
+ * Injects static preload hints into the final HTML. Static hosts cannot send the
+ * preload-manifest headers, so this is the page-level equivalent for runtime chunks,
+ * fonts and the LCP image.
+ */
+export function injectPreloadLinks(html, options = {}) {
+  if (options.scripts?.length === 0 && options.fonts?.size === 0 && !options.lcpImage) {
+    return html;
+  }
+  const document = parse(html);
+  const added = injectPreloadLinksInto(document, options);
+  return added > 0 ? serialize(document) : html;
+}
+
+/**
+ * Injects preload hints into a parsed document and reports how many were added.
+ * Keeping this separate lets the write pipeline parse the page once for all final
+ * post-processing steps.
+ */
+export function injectPreloadLinksInto(
+  document,
+  {
+    scripts = [],
+    fonts = new Map(),
+    lcpImage = null,
+    base = '/',
+    ignoreForPreload = [],
+    skipScripts = new Set(),
+  } = {},
+) {
+  const head = findHead(document);
+  if (!head) return 0;
+  const existingScripts = new Set();
+  const hints = new Set();
+
+  const walk = (node) => {
+    if (node.tagName === 'script') {
+      const src = attribute(node, 'src');
+      if (src) existingScripts.add(src);
+    }
+    if (node.tagName === 'link') {
+      const href = attribute(node, 'href');
+      if (href && attribute(node, 'rel').split(/\s+/).includes('preload')) hints.add(href);
+    }
+    for (const child of node.childNodes ?? []) walk(child);
+  };
+  walk(document);
+
+  const ignored = new Set(ignoreForPreload);
+  let added = 0;
+  const add = (element, href) => {
+    if (hints.has(href) || ignored.has(path.posix.basename(href))) return;
+    hints.add(href);
+    head.childNodes.push(element);
+    added += 1;
+  };
+
+  for (const script of [...scripts].sort()) {
+    const href = toPublicPath(script, base);
+    if (existingScripts.has(href) || skipScripts.has(href)) continue;
+    add(
+      linkNode([
+        { name: 'rel', value: 'modulepreload' },
+        { name: 'href', value: href },
+      ]),
+      href,
+    );
+  }
+  for (const [font, contentType] of [...fonts.entries()].sort(([left], [right]) =>
+    left.localeCompare(right),
+  )) {
+    const href = toPublicPath(font, base);
+    const attrs = [
+      { name: 'rel', value: 'preload' },
+      { name: 'as', value: 'font' },
+      { name: 'href', value: href },
+      { name: 'crossorigin', value: '' },
+    ];
+    if (contentType) attrs.push({ name: 'type', value: contentType });
+    add(linkNode(attrs), href);
+  }
+  if (lcpImage) {
+    const href = toPublicPath(lcpImage, base);
+    add(
+      linkNode([
+        { name: 'rel', value: 'preload' },
+        { name: 'as', value: 'image' },
+        { name: 'href', value: href },
+        { name: 'fetchpriority', value: 'high' },
+      ]),
+      href,
+    );
+  }
+  return added;
+}
+
+// Collects script src values from an already parsed document.
+export function collectScriptSourcesFromDocument(document) {
+  const sources = new Set();
+  const walk = (node) => {
+    if (node.tagName === 'script') {
+      const src = attribute(node, 'src');
+      if (src) sources.add(src);
+    }
+    for (const child of node.childNodes ?? []) walk(child);
+  };
+  walk(document);
+  return sources;
+}
+
+function findHead(document) {
+  const html = (document.childNodes ?? []).find((node) => node.tagName === 'html');
+  return (html?.childNodes ?? []).find((node) => node.tagName === 'head') ?? null;
+}
+
+function linkNode(attrs) {
+  return {
+    nodeName: 'link',
+    tagName: 'link',
+    attrs,
+    namespaceURI: 'http://www.w3.org/1999/xhtml',
+    childNodes: [],
+    parentNode: null,
+  };
+}
+
+function attribute(node, name) {
+  return node.attrs?.find((entry) => entry.name === name)?.value ?? '';
+}

package/src/core/preload-manifest.js

@@ -12,8 +12,12 @@ const PAGE_ORIGIN = 'http://snappy.local';
  * output rather than the render-time resource list keeps the manifest correct after
  * inline CSS, tag surgery, or a failed write.
  */
-export function collectPageResources(html, { base = '/', route = '/' } = {}) {
-  const document = parse(html);
+export function collectPageResources(html, options = {}) {
+  return collectPageResourcesFromDocument(parse(html), options);
+}
+
+// Reads referenced resources from an already parsed document.
+export function collectPageResourcesFromDocument(document, { base = '/', route = '/' } = {}) {
   const scripts = [];
   const styles = [];
   const pageUrl = new URL(toPublicPath(route, base), PAGE_ORIGIN);

package/src/core/render.js

@@ -1,24 +1,13 @@
+import { installClock, installClockScripts } from './clock.js';
 import { captureFormState } from './form-state.js';
 import { inlineStylesheets } from './inline-css.js';
-import { isPageEmpty } from './page-content.js';
+import { localeContextOptions, localeStorageState, seedLocaleContext } from './locale-context.js';
+import { readPageState } from './page-content.js';
 import { createResourceCollector } from './resources.js';
-import { toAppRoute, toServerUrl } from './routes.js';
+import { normaliseRoute, toAppRoute, toServerUrl } from './routes.js';
 import { captureRuntimeStyles } from './runtime-styles.js';
 import { freezeAnimations, scrollThroughPage, settlePage, trackNetwork } from './settle.js';
-import { separateTextNodes } from './text-separators.js';
-
-const SCRIPT_ESCAPES = {
-  '<': '\\u003C',
-  '>': '\\u003E',
-  '/': '\\u002F',
-  '\u2028': '\\u2028',
-  '\u2029': '\\u2029',
-};
-
-// Serialises a value for embedding in a script element without allowing tag breakout.
-function escapeForScript(value) {
-  return JSON.stringify(value).replace(/[<>/\u2028\u2029]/g, (char) => SCRIPT_ESCAPES[char]);
-}
+import { readRenderSnapshot } from './text-separators.js';
 
 /**
  * Renders one route in a fresh browser context and returns the serialised DOM plus
@@ -26,16 +15,36 @@ function escapeForScript(value) {
  * storage from leaking between routes. When a screenshot path is supplied the route
  * is captured as an image instead of HTML.
  */
-export async function renderRoute({ browser, origin, route, config, screenshot = false }) {
+export async function renderRoute({
+  browser,
+  origin,
+  route,
+  locale = null,
+  config,
+  screenshot = false,
+  traceFile = null,
+  caches = null,
+}) {
   const context = await browser.newContext({
     viewport: config.viewport,
-    storageState: config.storageState ?? undefined,
+    storageState: localeStorageState(config, locale),
     userAgent: config.userAgent ?? undefined,
     ignoreHTTPSErrors: config.ignoreHTTPSErrors,
     reducedMotion: config.freezeAnimations ? 'reduce' : 'no-preference',
     serviceWorkers: 'block',
+    ...localeContextOptions(config, locale),
   });
+  let tracing = false;
+  if (traceFile) {
+    await context.tracing.start({ screenshots: false, snapshots: true, sources: false });
+    tracing = true;
+  }
   const page = await context.newPage();
+  await installClock(page, config.fixedTime ?? null);
+  await installClockScripts(page, {
+    fixedTime: config.fixedTime ?? null,
+    randomSeed: config.randomSeed ?? null,
+  });
   const pageErrors = [];
   const consoleErrors = [];
   const failedRequests = [];
@@ -63,6 +72,15 @@ export async function renderRoute({ browser, origin, route, config, screenshot =
   const metadataDefaults = config.metadata
     ? { base: config.base, flatOutput: config.flatOutput, ...config.metadata }
     : null;
+  await seedLocaleContext({
+    page,
+    context,
+    origin,
+    config,
+    route,
+    locale,
+    phase: 'render',
+  });
   if (metadataDefaults) {
     await page.addInitScript((defaults) => {
       window.__SNAPPY_META__ = defaults;
@@ -78,25 +96,58 @@ export async function renderRoute({ browser, origin, route, config, screenshot =
     maxCachedBytes: config.maxCachedBytes,
   });
   await collector.install(page);
+  await page.addInitScript(() => {
+    window.__snappyLcp = [];
+    try {
+      const observer = new PerformanceObserver((list) => {
+        for (const entry of list.getEntries()) {
+          const element = entry.element;
+          window.__snappyLcp.push({
+            url: entry.url || element?.currentSrc || element?.src || null,
+          });
+        }
+      });
+      observer.observe({ type: 'largest-contentful-paint', buffered: true });
+    } catch {
+      // Older engines simply do not report LCP; preload injection falls back to no hint.
+    }
+  });
   const tracker = trackNetwork(page);
   try {
     const response = await page.goto(toServerUrl(route, config.base, origin), {
       waitUntil: 'domcontentloaded',
       timeout: config.timeout,
     });
+    const notFoundRoute = config.notFoundRoute ? normaliseRoute(config.notFoundRoute) : null;
+    const isNotFoundRoute = notFoundRoute !== null && normaliseRoute(route) === notFoundRoute;
     if (response && response.status() >= 400) {
-      throw new Error(
-        `route responded with HTTP ${response.status()}${response.statusText() ? ` ${response.statusText()}` : ''}`,
-      );
+      // A configured 404 route may legitimately answer with HTTP 404; every other status
+      // is a failure, and the marker cannot be read before the navigation completes.
+      if (!(response.status() === 404 && isNotFoundRoute)) {
+        const error = new Error(
+          `route responded with HTTP ${response.status()}${response.statusText() ? ` ${response.statusText()}` : ''}`,
+        );
+        // A server error can be transient and is retried; a client error is the app's own.
+        error.retryable = response.status() >= 500;
+        throw error;
+      }
     }
     await settlePage(page, config, tracker);
-    const empty = await isPageEmpty(page);
+    const { empty, statusCode } = await readPageState(page);
+    const finalUrl = page.url();
 
     if (config.captureRuntimeStyles) await captureRuntimeStyles(page);
     if (config.captureFormState) await captureFormState(page);
     if (config.inlineCss === true || config.inlineCss === 'inline') {
-      const inlined = await inlineStylesheets({ page, minifyCssOptions: config.minifyCss });
+      const inlined = await inlineStylesheets({
+        page,
+        minifyCssOptions: config.minifyCss,
+        cache: caches?.inlineCss ?? null,
+      });
       inlineCssSkipped = inlined.skipped;
+      if (caches?.inlineCss) {
+        for (const sheet of inlined.fetched ?? []) caches.inlineCss.set(sheet.href, sheet.css);
+      }
     }
     await collector.settled();
     const stateScript = await injectCapturedState(page, collector.json, {
@@ -106,9 +157,17 @@ export async function renderRoute({ browser, origin, route, config, screenshot =
     if (metadataDefaults) await injectMetadataDefaults(page, metadataDefaults);
     if (config.freezeAnimations) await freezeAnimations(page);
     if (config.scrollToBottom) await scrollThroughPage(page, config);
-    await separateTextNodes(page);
-
-    const links = await collectLinks(page, config.base, origin);
+    const { lcpUrl, hrefs } = await readRenderSnapshot(page);
+    let lcpImage = null;
+    if (lcpUrl) {
+      try {
+        const parsed = new URL(lcpUrl);
+        if (parsed.origin === origin) lcpImage = `${parsed.pathname}${parsed.search}`;
+      } catch {
+        // Ignore malformed performance entries.
+      }
+    }
+    const links = collectLinks(hrefs, config.base, origin);
 
     if (screenshot) {
       const buffer = await page.screenshot({
@@ -126,6 +185,10 @@ export async function renderRoute({ browser, origin, route, config, screenshot =
         stateScript,
         screenshot: buffer,
         empty,
+        statusCode,
+        httpStatus: response?.status() ?? null,
+        finalUrl,
+        lcpImage,
       };
     }
 
@@ -141,13 +204,37 @@ export async function renderRoute({ browser, origin, route, config, screenshot =
       inlineCssSkipped,
       stateScript,
       empty,
+      statusCode,
+      httpStatus: response?.status() ?? null,
+      finalUrl,
+      lcpImage,
     };
+  } catch (error) {
+    error.retryable ??= isRetryableRenderError(error);
+    if (tracing) {
+      tracing = false;
+      await context.tracing.stop({ path: traceFile }).then(
+        () => {
+          error.tracePath = traceFile;
+        },
+        // A failed trace must never replace the render error that explains the build failure.
+        () => {},
+      );
+    }
+    throw error;
   } finally {
     tracker.stop();
+    if (tracing) await context.tracing.stop().catch(() => {});
     await context.close();
   }
 }
 
+// Navigation and network failures are transient; a bad selector or an HTTP status is not.
+function isRetryableRenderError(error) {
+  if (error?.name === 'TimeoutError') return true;
+  return /net::ERR_|timed? ?out|ECONNRESET|ECONNREFUSED|socket hang up/i.test(error?.message ?? '');
+}
+
 /**
  * Injects captured JSON responses as window.snapStore and anything the app exposes
  * through window.snapSaveState, before the first script runs. The client can then
@@ -229,16 +316,21 @@ async function injectCapturedState(page, cache, { externalScripts = false, base
  * executable script, so a strict script-src applies to nothing here.
  */
 async function injectMetadataDefaults(page, defaults) {
-  const payload = await page.evaluate((fallback) => {
+  await page.evaluate((fallback) => {
     document.querySelectorAll('script[data-snappy-meta]').forEach((script) => script.remove());
     const current = typeof window === 'undefined' ? null : window.__SNAPPY_META__;
     const recorded = typeof window === 'undefined' ? null : window.__snappyHeadOriginals;
     const merged = { ...(current ?? fallback) };
     if (recorded) merged.originals = recorded;
-    return Object.keys(merged).length > 0 ? merged : fallback;
-  }, defaults);
-  const element = escapeForScript(payload);
-  await page.evaluate((text) => {
+    const payload = Object.keys(merged).length > 0 ? merged : fallback;
+    const escapes = {
+      '<': '\\u003C',
+      '>': '\\u003E',
+      '/': '\\u002F',
+      '\u2028': '\\u2028',
+      '\u2029': '\\u2029',
+    };
+    const text = JSON.stringify(payload).replace(/[<>/\u2028\u2029]/g, (char) => escapes[char]);
     const script = document.createElement('script');
     script.setAttribute('type', 'application/json');
     script.setAttribute('data-snappy-meta', '');
@@ -246,7 +338,7 @@ async function injectMetadataDefaults(page, defaults) {
     const first = document.scripts[0];
     if (first?.parentNode) first.parentNode.insertBefore(script, first);
     else document.head.appendChild(script);
-  }, element);
+  }, defaults);
 }
 
 /**
@@ -254,8 +346,7 @@ async function injectMetadataDefaults(page, defaults) {
  * app-relative routes for the crawler. Off-origin links are ignored so external
  * URLs can never be mistaken for routes and written into the output directory.
  */
-async function collectLinks(page, base, origin) {
-  const hrefs = await page.$$eval('a[href]', (anchors) => anchors.map((anchor) => anchor.href));
+function collectLinks(hrefs, base, origin) {
   const routes = [];
   for (const href of hrefs) {
     let parsed;
@@ -268,7 +359,7 @@ async function collectLinks(page, base, origin) {
     const route = toAppRoute(href, base);
     if (route) routes.push(route);
   }
-  return routes;
+  return [...new Set(routes)];
 }
 
 // Parses an HTTP URL, returning null for data, blob and other non-network URLs.

package/src/core/report.js

@@ -0,0 +1,116 @@
+import path from 'node:path';
+import { writeFileIfChanged } from './output.js';
+
+/**
+ * Writes the machine-readable run report. The JSON is deliberately limited to route,
+ * audit and artifact data; configuration and captured state are never serialised.
+ */
+export async function writeReport({ report, config, log }) {
+  if (!config.reportPath) return null;
+  const contents = `${JSON.stringify(buildReport(report, config), null, 2)}\n`;
+  if (config.dryRun) {
+    log.info(`Dry run: would write report to ${config.reportPath}`);
+    return { file: config.reportPath, status: 'dry-run' };
+  }
+  const target = path.resolve(config.reportPath);
+  const written = await writeFileIfChanged({
+    dir: path.dirname(target),
+    file: path.basename(target),
+    contents,
+  });
+  log.info(`Wrote report to ${config.reportPath}`);
+  return { file: config.reportPath, status: written.status };
+}
+
+/**
+ * Shapes the public report document. `routes` here is an array of per-route records;
+ * the programmatic `report.routes` stays a list of route strings. URLs are reduced to
+ * origin and path, and the configured storage-state path is redacted everywhere, so a
+ * report can be attached to a build log without leaking credentials or session data.
+ */
+export function buildReport(report, config = null) {
+  const audit = report.audit ?? { errors: [], warnings: [], linksChecked: 0 };
+  const document = {
+    version: 1,
+    generatedAt: new Date().toISOString(),
+    durationMs: report.durationMs,
+    timings: report.timings,
+    ok: report.ok,
+    clock: report.clock ?? null,
+    locales: report.locales ?? null,
+    seoLocale: report.seoLocale ?? null,
+    summary: {
+      routes: report.routeDetails.length,
+      files: report.files.filter((file) => file.status !== 'dry-run').length,
+      errors: uniqueErrorCount(report, audit),
+      warnings: audit.warnings.length,
+    },
+    routes: report.routeDetails,
+    verification: report.verification
+      ? {
+          ok: report.verification.ok,
+          modes: report.verification.modes,
+          frameworks: report.verification.frameworks,
+          skipped: report.verification.skipped ?? 0,
+          policySkipped: report.verification.policySkipped ?? 0,
+          allowed: report.verification.allowed ?? 0,
+          allowedRerender: report.verification.allowedRerender ?? 0,
+          errors: report.verification.errors,
+        }
+      : null,
+    audit,
+    sitemap: report.sitemap,
+    preloadManifest: report.preloadManifest,
+    truncated: report.truncated,
+    notFoundRoutes: report.notFoundRoutes,
+    duplicateOutputs: report.duplicatePages,
+    routeSources: report.routeSources ?? [],
+    cspSkipped: report.cspSkipped ?? [],
+    traces: report.traces ?? [],
+  };
+  return sanitiseValue(document, config);
+}
+
+// Counts route errors once even when the audit mirrors them as findings.
+function uniqueErrorCount(report, audit) {
+  const seen = new Set();
+  for (const error of [...(report.errors ?? []), ...(audit.errors ?? [])]) {
+    seen.add(`${error.route ?? ''}|${error.message ?? ''}`);
+  }
+  return seen.size;
+}
+
+// Recursively redacts URLs and the configured storage-state path from report strings.
+function sanitiseValue(value, config) {
+  if (Array.isArray(value)) return value.map((entry) => sanitiseValue(entry, config));
+  if (value !== null && typeof value === 'object') {
+    return Object.fromEntries(
+      Object.entries(value).map(([key, entry]) => [key, sanitiseValue(entry, config)]),
+    );
+  }
+  if (typeof value !== 'string') return value;
+  let text = value;
+  if (config?.storageState) {
+    for (const secret of [config.storageState, path.resolve(config.storageState)]) {
+      if (text.includes(secret)) text = text.split(secret).join('[redacted]');
+    }
+  }
+  if (!/https?:\/\//i.test(text)) return text;
+  return text.replace(/https?:\/\/[^\s"'<>]+/gi, redactUrl);
+}
+
+// Removes credentials, query strings and fragments from an absolute URL.
+function redactUrl(raw) {
+  const trailing = /[.,;)\]]+$/.exec(raw)?.[0] ?? '';
+  const urlText = trailing ? raw.slice(0, -trailing.length) : raw;
+  try {
+    const url = new URL(urlText);
+    url.username = '';
+    url.password = '';
+    url.search = '';
+    url.hash = '';
+    return `${url.href}${trailing}`;
+  } catch {
+    return raw;
+  }
+}

package/src/core/resources.js

@@ -18,6 +18,7 @@ export function createResourceCollector({
   const images = new Set();
   const scripts = new Set();
   const styles = new Set();
+  const fonts = new Map();
   const json = new Map();
   const pending = [];
 
@@ -26,6 +27,7 @@ export function createResourceCollector({
     images,
     scripts,
     styles,
+    fonts,
     json,
     // Waits for in-flight JSON body reads so the cache is complete before injection.
     async settled() {
@@ -57,7 +59,9 @@ export function createResourceCollector({
         if (resourceType === 'image') images.add(path);
         else if (resourceType === 'script') scripts.add(path);
         else if (resourceType === 'stylesheet') styles.add(path);
-        else if (collectJson && isJson(response)) {
+        else if (resourceType === 'font') {
+          fonts.set(path, response.headers()['content-type'] ?? null);
+        } else if (collectJson && isJson(response)) {
           const length = Number(response.headers()['content-length'] ?? Number.NaN);
           if (Number.isFinite(length) && length > maxCachedBytes) return;
           const capture = response

package/src/core/route-config.js

@@ -0,0 +1,17 @@
+import { matchesPattern } from './routes.js';
+
+/**
+ * Resolves one route's effective config by applying matching routeOverrides in order.
+ * Later entries win per key; the base config is never mutated.
+ */
+export function resolveRouteConfig(config, route) {
+  let resolved = config;
+  for (const override of config.routeOverrides ?? []) {
+    if (!matchesPattern(route, override.match)) continue;
+    const values = Object.fromEntries(
+      Object.entries(override).filter(([key, value]) => key !== 'match' && value !== undefined),
+    );
+    resolved = { ...resolved, ...values };
+  }
+  return resolved;
+}