snappy-prerender 2.6.0-rc.3 → 2.6.0

package/src/core/server.js

@@ -8,18 +8,32 @@ import sirv from 'sirv';
  * an ephemeral loopback port so parallel runs never collide.
  */
 export async function startStaticServer({ dir, base }) {
-  const handler = sirv(dir, { dev: true, etag: true, single: true });
+  const handler = sirv(dir, { dev: true, etag: true });
+  const indexHandler = sirv(dir, { dev: true, etag: true });
   const server = http.createServer((request, response) => {
     if (base !== '/') {
       const stripped = stripBase(request.url, base);
       if (stripped === null) {
-        response.statusCode = 404;
-        response.end('Not found');
+        notFound(response);
         return;
       }
       request.url = stripped;
     }
-    handler(request, response);
+    handler(request, response, () => {
+      // Navigation requests can be SPA routes that only exist in the router. Assets,
+      // fetches and websockets ask for something else and must see their real 404.
+      if (!acceptsHtml(request)) {
+        notFound(response);
+        return;
+      }
+      const original = request.url;
+      request.url = '/index.html';
+      try {
+        indexHandler(request, response, () => notFound(response));
+      } finally {
+        request.url = original;
+      }
+    });
   });
 
   await new Promise((resolve, reject) => {
@@ -37,6 +51,18 @@ export async function startStaticServer({ dir, base }) {
   };
 }
 
+// Reports whether a request is a document navigation rather than an asset or fetch.
+function acceptsHtml(request) {
+  const accept = request.headers.accept ?? '';
+  return accept.includes('text/html') || accept.includes('application/xhtml+xml');
+}
+
+// Answers a request no handler could serve.
+function notFound(response) {
+  response.statusCode = 404;
+  response.end('Not found');
+}
+
 /**
  * Removes the configured base prefix from a request URL, keeping any query string
  * intact. Returns null when the request falls outside the base path.

package/src/core/sitemap.js

@@ -1,37 +1,55 @@
 import { normaliseRoute, toPublicPath } from './routes.js';
+import { pagePath } from './url-policy.js';
 
 const XML_ESCAPES = { '&': '&amp;', '<': '&lt;', '>': '&gt;', '"': '&quot;', "'": '&apos;' };
+export const SITEMAP_MAX_URLS = 50000;
 
 /**
- * Builds the sitemap for the routes a run wrote. Locs are derived from the output file
- * names, so a directory route becomes /about/ and a flat one becomes /about.html, then
- * prefixed with the public base path. Returns null when there is no public site URL or
- * nothing to list, so callers can skip the write entirely.
+ * Builds the sitemap for the routes a run wrote. Locs use the same page path policy as
+ * canonical links, so a directory route keeps its slash, a flat route stays
+ * extensionless, and a noindex page is never listed. More than maxUrls entries are
+ * split into numbered sitemaps behind a sitemap index.
  */
-export function buildSitemap({ files, config, generatedAt = new Date() }) {
+export function buildSitemap({ files, config, maxUrls = SITEMAP_MAX_URLS }) {
   const siteUrl = config.metadata?.siteUrl?.replace(/\/$/, '');
   if (!siteUrl) return null;
   const notFound = config.notFoundRoute ? normaliseRoute(config.notFoundRoute) : null;
 
   const locs = new Set();
   for (const file of files) {
-    if (!file.file.endsWith('.html')) continue;
+    if (!file.file.endsWith('.html') || file.noindex) continue;
     if (file.route && notFound && normaliseRoute(file.route) === notFound) continue;
-    locs.add(`${siteUrl}${toPublicPath(publicPath(file.file), config.base)}`);
+    const path = file.route
+      ? pagePath(file.route, {
+          flatOutput: config.flatOutput,
+          trailingSlash: config.metadata?.trailingSlash,
+        })
+      : publicPath(file.file);
+    locs.add(`${siteUrl}${toPublicPath(path, config.base)}`);
   }
   if (locs.size === 0) return null;
 
-  const lastmod = generatedAt.toISOString().slice(0, 10);
-  const urls = [...locs]
-    .sort()
-    .map(
-      (loc) =>
-        `  <url>\n    <loc>${escapeXml(loc)}</loc>\n    <lastmod>${lastmod}</lastmod>\n  </url>`,
-    )
-    .join('\n');
+  const sorted = [...locs].sort();
+  const chunks = [];
+  for (let index = 0; index < sorted.length; index += maxUrls) {
+    chunks.push(sorted.slice(index, index + maxUrls));
+  }
+
+  if (chunks.length === 1) {
+    return { files: [{ name: 'sitemap.xml', contents: urlSet(chunks[0]) }], routes: sorted.length };
+  }
+
+  const entries = chunks.map((chunk, index) => {
+    const name = `sitemap-${index + 1}.xml`;
+    const loc = `${siteUrl}${toPublicPath(`/${name}`, config.base)}`;
+    return { name, contents: urlSet(chunk), loc };
+  });
   return {
-    contents: `<?xml version="1.0" encoding="UTF-8"?>\n<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n${urls}\n</urlset>\n`,
-    routes: locs.size,
+    files: [
+      { name: 'sitemap.xml', contents: sitemapIndex(entries) },
+      ...entries.map(({ name, contents }) => ({ name, contents })),
+    ],
+    routes: sorted.length,
   };
 }
 
@@ -56,6 +74,20 @@ export function withSitemapDirective(robots, sitemapUrl) {
   return `${robots}${separator}Sitemap: ${sitemapUrl}\n`;
 }
 
+// Builds one urlset document.
+function urlSet(locs) {
+  const urls = locs.map((loc) => `  <url>\n    <loc>${escapeXml(loc)}</loc>\n  </url>`).join('\n');
+  return `<?xml version="1.0" encoding="UTF-8"?>\n<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n${urls}\n</urlset>\n`;
+}
+
+// Builds the sitemap index that points at the numbered sitemaps.
+function sitemapIndex(entries) {
+  const sitemaps = entries
+    .map((entry) => `  <sitemap>\n    <loc>${escapeXml(entry.loc)}</loc>\n  </sitemap>`)
+    .join('\n');
+  return `<?xml version="1.0" encoding="UTF-8"?>\n<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n${sitemaps}\n</sitemapindex>\n`;
+}
+
 // Escapes the characters that would break a sitemap element.
 function escapeXml(value) {
   return value.replace(/[&<>"']/g, (char) => XML_ESCAPES[char]);

package/src/core/url-policy.js

@@ -0,0 +1,32 @@
+import { normaliseRoute } from './routes.js';
+
+/**
+ * Builds the public path for a route using the same policy for canonical, social and
+ * sitemap URLs. Directory output keeps the trailing slash it is served with; flat
+ * output stays extensionless so `/about` remains the URL even though the file is
+ * `about.html`. An explicit trailingSlash convention always wins.
+ */
+export function pagePath(route, { flatOutput = false, trailingSlash = 'preserve' } = {}) {
+  const normalised = normaliseRoute(route);
+  if (normalised === '/') return '/';
+  if (trailingSlash === 'always') return `${normalised}/`;
+  if (trailingSlash === 'never') return normalised;
+  return flatOutput ? normalised : `${normalised}/`;
+}
+
+/**
+ * Applies a trailing slash convention to an absolute URL, leaving the root and asset
+ * URLs alone. Used for an explicit canonical that is already a full URL.
+ */
+export function withTrailingSlash(url, convention) {
+  if (convention !== 'always' && convention !== 'never') return url;
+  const [path, query] = url.split('?');
+  const isRoot = /^https?:\/\/[^/]+\/?$/i.test(path);
+  if (convention === 'always') {
+    const next = path.endsWith('/') ? path : `${path}/`;
+    return query ? `${next}?${query}` : next;
+  }
+  if (isRoot) return query ? `${path.replace(/\/$/, '/')}?${query}` : path.replace(/\/?$/, '/');
+  const next = path.replace(/\/+$/, '');
+  return query ? `${next}?${query}` : next;
+}

package/src/core/verify.js

@@ -1,11 +1,14 @@
 import { blockThirdPartyRequests } from './blocking.js';
+import { DEFAULTS } from './config.js';
+import { isPageEmpty } from './page-content.js';
 import { toServerUrl } from './routes.js';
 import { mapLimit } from './scheduler.js';
 import { settlePage, trackNetwork } from './settle.js';
 
 const HYDRATION_PATTERNS = [
   /hydrat/i,
-  /did not match/i,
+  /text content did not match/i,
+  /prop [`"].+?[`"] did not match/i,
   /server-rendered HTML/i,
   // React 16 and 17 phrase their mismatch warnings differently and never use the
   // minified codes, which only exist from React 18 on.
@@ -94,8 +97,11 @@ async function verifyRoute({ browser, origin, route, config }) {
   const context = await browser.newContext({
     viewport: config.viewport,
     storageState: config.storageState ?? undefined,
-    userAgent: config.userAgent ?? undefined,
+    userAgent:
+      config.userAgent === DEFAULTS.userAgent ? undefined : (config.userAgent ?? undefined),
     ignoreHTTPSErrors: config.ignoreHTTPSErrors,
+    reducedMotion: config.freezeAnimations ? 'reduce' : 'no-preference',
+    serviceWorkers: 'block',
   });
   const page = await context.newPage();
   const hydrationErrors = [];
@@ -117,11 +123,13 @@ async function verifyRoute({ browser, origin, route, config }) {
       await blockThirdPartyRequests(page, origin, config.allowedHosts);
     }
     await page.addInitScript(bootProbe);
-    await page.goto(toServerUrl(route, config.base, origin), {
+    const response = await page.goto(toServerUrl(route, config.base, origin), {
       waitUntil: 'domcontentloaded',
       timeout: config.timeout,
     });
+    const status = response?.status() ?? null;
     await settlePage(page, config, tracker);
+    const empty = await isPageEmpty(page);
 
     const boot = await page.evaluate((selector) => {
       const probe = window.__snappyBoot ?? {};
@@ -138,10 +146,12 @@ async function verifyRoute({ browser, origin, route, config }) {
 
     return {
       route,
+      status,
+      empty,
       hydrationErrors,
       consoleErrors,
       pageErrors,
-      ok: hydrationErrors.length === 0,
+      ok: hydrationErrors.length === 0 && (status === null || status < 400) && !empty,
       mode: classifyBoot(boot),
       markupPreserved: boot.markupPreserved,
     };

package/src/core/write.js

@@ -0,0 +1,278 @@
+import { createHash } from 'node:crypto';
+import { findBuildOriginUrls } from './build-origin.js';
+import { inlineCriticalCss } from './inline-css.js';
+import { minifyHtml } from './minify.js';
+import { normaliseHtml } from './normalize.js';
+import {
+  findOutputCollisions,
+  routeToScreenshotFile,
+  writeBufferIfChanged,
+  writeFileIfChanged,
+  writeRouteHtml,
+} from './output.js';
+import { collectPageResources } from './preload-manifest.js';
+import { normaliseRoute, toPublicPath } from './routes.js';
+
+/**
+ * Writes every rendered route and returns the routes that can be verified. A write
+ * failure is recorded against its route instead of aborting the remaining writes.
+ */
+export async function writeAllRoutes({
+  rendered,
+  outputDir,
+  config,
+  log,
+  report,
+  state,
+  manifestEntries,
+}) {
+  const sorted = [...report.routes].sort();
+  const conflicted = reportOutputCollisions({ routes: sorted, config, log, report });
+
+  await writeStateScripts({ rendered, outputDir, config, log, report });
+
+  const verifiable = [];
+  const hashes = new Map();
+  for (const route of sorted) {
+    if (conflicted.has(route)) continue;
+    try {
+      const { file, resources } = await writeRouteOutput({
+        route,
+        result: rendered.get(route),
+        outputDir,
+        config,
+        log,
+        state,
+        report,
+      });
+      report.files.push(file);
+      if (resources) manifestEntries.push({ route, ...resources });
+      if (file.hash) {
+        const group = hashes.get(file.hash) ?? [];
+        group.push(route);
+        hashes.set(file.hash, group);
+      }
+      verifiable.push(route);
+    } catch (error) {
+      report.errors.push({ route, message: `write failed: ${error.message}` });
+      log.error(`  write failed for ${route}: ${error.message}`);
+    }
+  }
+  reportDuplicatePages({ groups: [...hashes.values()], config, log, report });
+  return verifiable;
+}
+
+/**
+ * Reports routes that produced byte-identical HTML. The usual cause is an SPA fallback
+ * serving the shell for a path the app does not route, which would otherwise publish a
+ * copy of another page under a dead URL.
+ */
+function reportDuplicatePages({ groups, config, log, report }) {
+  for (const routes of groups) {
+    if (routes.length < 2) continue;
+    report.duplicatePages.push({ routes });
+    log.warn(
+      `identical output for ${routes.join(', ')}: check for a dead link served by the SPA fallback`,
+    );
+    if (config.failOnDuplicateContent) {
+      for (const route of routes) {
+        report.errors.push({
+          route,
+          message: `identical output to ${routes.filter((other) => other !== route).join(', ')}`,
+        });
+      }
+    }
+  }
+}
+
+/**
+ * Writes the captured-state scripts routes reference when externalScripts is on. The payload
+ * is content-addressed, so routes that inject the same state share one file and a rerun
+ * leaves it untouched.
+ */
+async function writeStateScripts({ rendered, outputDir, config, log, report }) {
+  const seen = new Set();
+  for (const [route, result] of rendered) {
+    const stateScript = result?.stateScript;
+    if (!stateScript || seen.has(stateScript.file)) continue;
+    seen.add(stateScript.file);
+    if (config.dryRun) {
+      report.files.push({
+        route,
+        file: stateScript.file,
+        status: 'dry-run',
+        bytes: Buffer.byteLength(stateScript.contents),
+      });
+      continue;
+    }
+    try {
+      const written = await writeFileIfChanged({
+        dir: outputDir,
+        file: stateScript.file,
+        contents: stateScript.contents,
+      });
+      report.files.push({ route, ...written });
+      log.debug(`  wrote ${stateScript.file}: ${written.status}`);
+    } catch (error) {
+      report.errors.push({ route, message: `write failed: ${error.message}` });
+      log.error(`  write failed for ${stateScript.file}: ${error.message}`);
+    }
+  }
+}
+
+/**
+ * Records an error for every route involved in an output file collision and returns
+ * the routes to skip, so an ambiguous write can never overwrite another route.
+ */
+function reportOutputCollisions({ routes, config, log, report }) {
+  const conflicted = new Set();
+  for (const [file, colliding] of findOutputCollisions(routes, config)) {
+    for (const route of colliding) {
+      conflicted.add(route);
+      const others = colliding.filter((other) => other !== route).join(', ');
+      report.errors.push({
+        route,
+        message: `output file collision: ${file} is also written by ${others}`,
+      });
+    }
+    log.error(`  output file collision on ${file}: ${colliding.join(', ')}`);
+  }
+  return conflicted;
+}
+
+/**
+ * Applies the configured post-processing to one rendered route and writes it: DOM
+ * cleanups and link hints, critical CSS extraction, then optional minification.
+ */
+async function writeRouteOutput({ route, result, outputDir, config, log, state, report }) {
+  if (result.screenshot) {
+    const file = routeToScreenshotFile(route, config);
+    if (config.dryRun) {
+      return {
+        file: {
+          route,
+          file,
+          status: 'dry-run',
+          bytes: result.screenshot.length,
+        },
+        resources: null,
+      };
+    }
+    const written = await writeBufferIfChanged({
+      dir: outputDir,
+      file,
+      buffer: result.screenshot,
+    });
+    return { file: { route, ...written }, resources: null };
+  }
+
+  const { html, stats } = normaliseHtml(result.html, {
+    removeStyleTags: config.removeStyleTags,
+    removeScriptTags: config.removeScriptTags,
+    asyncScriptTags: config.asyncScriptTags,
+    removeBlobs: config.removeBlobs,
+    removeAttributes: config.removeAttributes,
+    stripBuildOriginMetadata: config.stripBuildOriginMetadata,
+    noindex:
+      config.notFoundRoute !== null &&
+      normaliseRoute(route) === normaliseRoute(config.notFoundRoute),
+    preconnectOrigins: config.preconnectThirdParty
+      ? [...result.collector.thirdPartyOrigins].sort()
+      : [],
+    preloadImages: config.preloadImages
+      ? [...result.collector.images].sort().map((image) => toPublicPath(image, config.base))
+      : [],
+  });
+  if (
+    stats.removedElements > 0 ||
+    stats.removedAttributes > 0 ||
+    stats.dedupedStyles > 0 ||
+    stats.hints > 0 ||
+    stats.textSeparators > 0
+  ) {
+    log.debug(
+      `  cleaned ${route}: ${stats.removedElements} element(s) removed, ${stats.removedAttributes} attribute(s) removed, ${stats.dedupedStyles} duplicate style(s), ${stats.hints} hint(s), ${stats.textSeparators} text separator(s)`,
+    );
+  }
+  if ((result.inlineCssSkipped ?? []).length > 0) {
+    log.debug(`  inline css: left ${result.inlineCssSkipped.length} stylesheet(s) as links`);
+  }
+
+  let output = html;
+  if (config.inlineCss === 'critical') {
+    output = await applyCriticalCss({ output, outputDir, config, log, state });
+  }
+  if (config.minifyHtml) {
+    output = await minifyHtml(output, config.minifyHtml, config.minifyCss);
+  }
+  warnOnBuildOriginMetadata({
+    origins: stats.metadataOrigins,
+    removed: stats.removedMetadata,
+    route,
+    log,
+    state,
+  });
+  if (stats.titleElements > 1) report.duplicateTitles.push(route);
+  if (config.failOnBuildOriginMetadata) {
+    const origins = findBuildOriginUrls(output);
+    if (origins.length > 0) {
+      const message = `metadata still points at the build server: ${origins.join(', ')}`;
+      report.errors.push({ route, message });
+      log.error(`  ${route}: ${message}`);
+    }
+  }
+  const file = await writeRouteHtml({ dir: outputDir, route, html: output, config });
+  const resources = config.preloadManifest
+    ? collectPageResources(output, { base: config.base, route })
+    : null;
+  return {
+    file: {
+      ...file,
+      // Hash the captured DOM before post-processing adds route-specific extras such as
+      // the not-found noindex, so a dead link serving the 404 page is still detected.
+      hash: createHash('sha256')
+        .update(result.html ?? output)
+        .digest('hex'),
+      noindex: stats.noindex,
+    },
+    resources,
+  };
+}
+
+/**
+ * Inlines critical CSS, or explains once why it cannot: the strategy reads stylesheets from
+ * the output directory, which a command server does not have to serve.
+ */
+async function applyCriticalCss({ output, outputDir, config, log, state }) {
+  if (!config.serveCmd) {
+    return inlineCriticalCss({
+      html: output,
+      outputDir,
+      base: config.base,
+      preload: config.criticalCssPreload,
+    });
+  }
+  if (!state.warnedCriticalCssSkipped) {
+    state.warnedCriticalCssSkipped = true;
+    log.warn(
+      "inlineCss: 'critical' is skipped with serveCmd: stylesheets are read from the output directory, which the command server does not have to serve",
+    );
+  }
+  return output;
+}
+
+/**
+ * Warns once when metadata contains the build server's origin, which means the page was
+ * composed without a public siteUrl and og:url, og:image or canonical would ship wrong.
+ */
+function warnOnBuildOriginMetadata({ origins, removed = 0, route, log, state }) {
+  if (state.warnedBuildOriginMetadata || origins.length === 0) return;
+  state.warnedBuildOriginMetadata = true;
+  log.warn(
+    `metadata on ${route} points at the build server (${origins.join(', ')}); ${
+      removed > 0
+        ? `removed ${removed} affected tag(s). Set metadata.siteUrl so canonical, og:url and og:image match the deployed site`
+        : 'set metadata.siteUrl so og:url, og:image and canonical match the deployed site'
+    }`,
+  );
+}

package/src/head/index.js

@@ -45,7 +45,7 @@ export function Head(props) {
  */
 export function setHead(id, props) {
   sequence += 1;
-  registerHead(id, sequence, buildHeadEntries(props, readDefaults()));
+  registerHead(id, sequence, buildHeadEntries(props, readMetadataDefaults()));
 }
 
 /**

package/src/head/tags.js

@@ -1,4 +1,4 @@
-import { collectBuildOrigins } from '../core/metadata-origin.js';
+import { pagePath, withTrailingSlash } from '../core/url-policy.js';
 
 const ABSOLUTE = /^https?:\/\//i;
 const PROTOCOL_RELATIVE = /^\/\//;
@@ -34,7 +34,7 @@ export function buildHeadEntries(props = {}, defaults = {}) {
     entries.push(metaProp('og:type', props.type ?? 'website'));
     if (siteName) entries.push(metaProp('og:site_name', siteName));
     if (props.locale) entries.push(metaProp('og:locale', props.locale));
-    const url = pageUrl(props.canonical ?? currentPath(), defaults);
+    const url = pageUrl(props.canonical ?? defaultPagePath(defaults), defaults);
     if (url) entries.push(metaProp('og:url', url));
     if (image) entries.push(metaProp('og:image', absoluteUrl(image, defaults)));
   }
@@ -45,7 +45,7 @@ export function buildHeadEntries(props = {}, defaults = {}) {
   }
 
   if (props.canonical !== false) {
-    const url = pageUrl(props.canonical ?? currentPath(), defaults);
+    const url = pageUrl(props.canonical ?? defaultPagePath(defaults), defaults);
     if (url)
       entries.push({ kind: 'link', key: 'rel:canonical', attrs: { rel: 'canonical', href: url } });
   }
@@ -88,14 +88,6 @@ export function headKey(entry) {
   return `meta:${attrsKey(entry.attrs)}`;
 }
 
-/**
- * Collects the absolute metadata URLs in a document that point at the machine the
- * prerenderer ran on, which happens when no public siteUrl was configured.
- */
-export function findBuildOriginUrls(html) {
-  return collectBuildOrigins(html);
-}
-
 // Applies the site's title template unless the title already carries its suffix or is the
 // site name itself, so an app that composed its own title never gets it suffixed twice.
 function applyTitleTemplate(title, template, siteName) {
@@ -139,23 +131,24 @@ function encodeSpaces(url) {
   return url.replace(/ /g, '%20');
 }
 
-// Applies the site's trailing slash convention to a URL, leaving the root alone.
-function withTrailingSlash(url, convention) {
-  if (convention !== 'always' && convention !== 'never') return url;
-  const [path, query] = url.split('?');
-  const isRoot = /^https?:\/\/[^/]+\/?$/i.test(path);
-  if (convention === 'always') {
-    const next = path.endsWith('/') ? path : `${path}/`;
-    return query ? `${next}?${query}` : next;
-  }
-  if (isRoot) return query ? `${path.replace(/\/$/, '/')}?${query}` : path.replace(/\/?$/, '/');
-  const next = path.replace(/\/+$/, '');
-  return query ? `${next}?${query}` : next;
+// Reads the route being rendered, which under prerendering is the page being captured,
+// and removes the public base path so absoluteUrl does not add it a second time.
+function currentPath(defaults = {}) {
+  const pathname = typeof location === 'undefined' ? '/' : location.pathname;
+  const base = defaults.base ?? '/';
+  if (base === '/') return pathname;
+  const prefix = base.replace(/\/$/, '');
+  if (pathname === prefix) return '/';
+  if (pathname.startsWith(`${prefix}/`)) return pathname.slice(prefix.length);
+  return pathname;
 }
 
-// Reads the route being rendered, which under prerendering is the page being captured.
-function currentPath() {
-  return typeof location === 'undefined' ? '/' : location.pathname;
+// Builds the default canonical path from the current route and the output's URL policy.
+function defaultPagePath(defaults) {
+  return pagePath(currentPath(defaults), {
+    flatOutput: defaults.flatOutput === true,
+    trailingSlash: defaults.trailingSlash,
+  });
 }
 
 function metaName(name, content) {