snappy-prerender 2.6.0-rc.3 → 2.7.0

package/src/core/server.js

@@ -8,18 +8,32 @@ import sirv from 'sirv';
  * an ephemeral loopback port so parallel runs never collide.
  */
 export async function startStaticServer({ dir, base }) {
-  const handler = sirv(dir, { dev: true, etag: true, single: true });
+  const handler = sirv(dir, { dev: true, etag: true });
+  const indexHandler = sirv(dir, { dev: true, etag: true });
   const server = http.createServer((request, response) => {
     if (base !== '/') {
       const stripped = stripBase(request.url, base);
       if (stripped === null) {
-        response.statusCode = 404;
-        response.end('Not found');
+        notFound(response);
         return;
       }
       request.url = stripped;
     }
-    handler(request, response);
+    handler(request, response, () => {
+      // Navigation requests can be SPA routes that only exist in the router. Assets,
+      // fetches and websockets ask for something else and must see their real 404.
+      if (!acceptsHtml(request)) {
+        notFound(response);
+        return;
+      }
+      const original = request.url;
+      request.url = '/index.html';
+      try {
+        indexHandler(request, response, () => notFound(response));
+      } finally {
+        request.url = original;
+      }
+    });
   });
 
   await new Promise((resolve, reject) => {
@@ -37,6 +51,18 @@ export async function startStaticServer({ dir, base }) {
   };
 }
 
+// Reports whether a request is a document navigation rather than an asset or fetch.
+function acceptsHtml(request) {
+  const accept = request.headers.accept ?? '';
+  return accept.includes('text/html') || accept.includes('application/xhtml+xml');
+}
+
+// Answers a request no handler could serve.
+function notFound(response) {
+  response.statusCode = 404;
+  response.end('Not found');
+}
+
 /**
  * Removes the configured base prefix from a request URL, keeping any query string
  * intact. Returns null when the request falls outside the base path.

package/src/core/sitemap.js

@@ -1,37 +1,64 @@
 import { normaliseRoute, toPublicPath } from './routes.js';
+import { pagePath } from './url-policy.js';
 
 const XML_ESCAPES = { '&': '&amp;', '<': '&lt;', '>': '&gt;', '"': '&quot;', "'": '&apos;' };
+export const SITEMAP_MAX_URLS = 50000;
 
 /**
- * Builds the sitemap for the routes a run wrote. Locs are derived from the output file
- * names, so a directory route becomes /about/ and a flat one becomes /about.html, then
- * prefixed with the public base path. Returns null when there is no public site URL or
- * nothing to list, so callers can skip the write entirely.
+ * Builds the sitemap for the routes a run wrote. Locs use the same page path policy as
+ * canonical links, so a directory route keeps its slash, a flat route stays
+ * extensionless, and a noindex page is never listed. More than maxUrls entries are
+ * split into numbered sitemaps behind a sitemap index.
  */
-export function buildSitemap({ files, config, generatedAt = new Date() }) {
+export function buildSitemap({ files, config, maxUrls = SITEMAP_MAX_URLS }) {
   const siteUrl = config.metadata?.siteUrl?.replace(/\/$/, '');
   if (!siteUrl) return null;
   const notFound = config.notFoundRoute ? normaliseRoute(config.notFoundRoute) : null;
 
-  const locs = new Set();
+  const entries = [];
   for (const file of files) {
-    if (!file.file.endsWith('.html')) continue;
+    if (!file.file.endsWith('.html') || file.noindex) continue;
     if (file.route && notFound && normaliseRoute(file.route) === notFound) continue;
-    locs.add(`${siteUrl}${toPublicPath(publicPath(file.file), config.base)}`);
+    const path = file.route
+      ? pagePath(file.route, {
+          flatOutput: config.flatOutput,
+          trailingSlash: config.metadata?.trailingSlash,
+        })
+      : publicPath(file.file);
+    entries.push({
+      loc: `${siteUrl}${toPublicPath(path, config.base)}`,
+      alternates: normaliseAlternates(file.alternates ?? [], siteUrl, config.base),
+      images: config.imageSitemap ? normaliseImages(file.images ?? [], siteUrl, config.base) : [],
+    });
   }
-  if (locs.size === 0) return null;
-
-  const lastmod = generatedAt.toISOString().slice(0, 10);
-  const urls = [...locs]
-    .sort()
-    .map(
-      (loc) =>
-        `  <url>\n    <loc>${escapeXml(loc)}</loc>\n    <lastmod>${lastmod}</lastmod>\n  </url>`,
-    )
-    .join('\n');
+  if (entries.length === 0) return null;
+
+  entries.sort((a, b) => a.loc.localeCompare(b.loc));
+  const chunks = [];
+  for (let index = 0; index < entries.length; index += maxUrls) {
+    chunks.push(entries.slice(index, index + maxUrls));
+  }
+
+  if (chunks.length === 1) {
+    return {
+      files: [{ name: 'sitemap.xml', contents: urlSet(chunks[0]) }],
+      routes: entries.length,
+      locs: entries.map((entry) => entry.loc),
+    };
+  }
+
+  const chunksWithNames = chunks.map((chunk, index) => {
+    const name = `sitemap-${index + 1}.xml`;
+    const loc = `${siteUrl}${toPublicPath(`/${name}`, config.base)}`;
+    return { name, contents: urlSet(chunk), loc };
+  });
   return {
-    contents: `<?xml version="1.0" encoding="UTF-8"?>\n<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n${urls}\n</urlset>\n`,
-    routes: locs.size,
+    files: [
+      { name: 'sitemap.xml', contents: sitemapIndex(chunksWithNames) },
+      ...chunksWithNames.map(({ name, contents }) => ({ name, contents })),
+    ],
+    routes: entries.length,
+    locs: entries.map((entry) => entry.loc),
   };
 }
 
@@ -56,6 +83,87 @@ export function withSitemapDirective(robots, sitemapUrl) {
   return `${robots}${separator}Sitemap: ${sitemapUrl}\n`;
 }
 
+// Builds one urlset document.
+function urlSet(entries) {
+  const hasAlternates = entries.some((entry) => entry.alternates.length > 0);
+  const hasImages = entries.some((entry) => entry.images.length > 0);
+  const namespaces = [
+    'xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"',
+    hasAlternates ? 'xmlns:xhtml="http://www.w3.org/1999/xhtml"' : null,
+    hasImages ? 'xmlns:image="http://www.google.com/schemas/sitemap-image/1.1"' : null,
+  ]
+    .filter(Boolean)
+    .join(' ');
+  const urls = entries
+    .map((entry) => {
+      const lines = [`    <loc>${escapeXml(entry.loc)}</loc>`];
+      for (const alternate of entry.alternates) {
+        lines.push(
+          `    <xhtml:link rel="alternate" hreflang="${escapeXml(alternate.hreflang)}" href="${escapeXml(alternate.href)}"/>`,
+        );
+      }
+      for (const image of entry.images) {
+        lines.push(
+          `    <image:image>\n      <image:loc>${escapeXml(image)}</image:loc>\n    </image:image>`,
+        );
+      }
+      return `  <url>\n${lines.join('\n')}\n  </url>`;
+    })
+    .join('\n');
+  return `<?xml version="1.0" encoding="UTF-8"?>\n<urlset ${namespaces}>\n${urls}\n</urlset>\n`;
+}
+
+// Resolves hreflang links and removes duplicates.
+function normaliseAlternates(alternates, siteUrl, base) {
+  const seen = new Set();
+  const result = [];
+  for (const alternate of alternates) {
+    const href = absoluteSitemapUrl(alternate.href, siteUrl, base);
+    if (!href) continue;
+    const key = `${alternate.hreflang}|${href}`;
+    if (seen.has(key)) continue;
+    seen.add(key);
+    result.push({ hreflang: alternate.hreflang, href });
+  }
+  return result;
+}
+
+// Resolves image URLs to absolute same-origin URLs.
+function normaliseImages(images, siteUrl, base) {
+  const seen = new Set();
+  const result = [];
+  for (const image of images) {
+    const absolute = absoluteSitemapUrl(image, siteUrl, base);
+    if (!absolute || new URL(absolute).origin !== new URL(siteUrl).origin) continue;
+    if (seen.has(absolute)) continue;
+    seen.add(absolute);
+    result.push(absolute);
+  }
+  return result;
+}
+
+// Resolves a site-relative path against the public base, leaving absolute URLs alone.
+function absoluteSitemapUrl(value, siteUrl, base) {
+  if (!value) return null;
+  const raw = String(value);
+  if (/^https?:\/\//i.test(raw)) return raw;
+  if (raw.startsWith('//')) return `https:${raw}`;
+  const basePrefix = base === '/' ? '' : base.slice(0, -1);
+  if (basePrefix && (raw === basePrefix || raw.startsWith(`${basePrefix}/`))) {
+    return `${siteUrl}${raw}`;
+  }
+  const path = raw.startsWith('/') ? raw : `/${raw}`;
+  return `${siteUrl}${basePrefix}${path}`;
+}
+
+// Builds the sitemap index that points at the numbered sitemaps.
+function sitemapIndex(entries) {
+  const sitemaps = entries
+    .map((entry) => `  <sitemap>\n    <loc>${escapeXml(entry.loc)}</loc>\n  </sitemap>`)
+    .join('\n');
+  return `<?xml version="1.0" encoding="UTF-8"?>\n<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n${sitemaps}\n</sitemapindex>\n`;
+}
+
 // Escapes the characters that would break a sitemap element.
 function escapeXml(value) {
   return value.replace(/[&<>"']/g, (char) => XML_ESCAPES[char]);

package/src/core/url-policy.js

@@ -0,0 +1,32 @@
+import { normaliseRoute } from './routes.js';
+
+/**
+ * Builds the public path for a route using the same policy for canonical, social and
+ * sitemap URLs. Directory output keeps the trailing slash it is served with; flat
+ * output stays extensionless so `/about` remains the URL even though the file is
+ * `about.html`. An explicit trailingSlash convention always wins.
+ */
+export function pagePath(route, { flatOutput = false, trailingSlash = 'preserve' } = {}) {
+  const normalised = normaliseRoute(route);
+  if (normalised === '/') return '/';
+  if (trailingSlash === 'always') return `${normalised}/`;
+  if (trailingSlash === 'never') return normalised;
+  return flatOutput ? normalised : `${normalised}/`;
+}
+
+/**
+ * Applies a trailing slash convention to an absolute URL, leaving the root and asset
+ * URLs alone. Used for an explicit canonical that is already a full URL.
+ */
+export function withTrailingSlash(url, convention) {
+  if (convention !== 'always' && convention !== 'never') return url;
+  const [path, query] = url.split('?');
+  const isRoot = /^https?:\/\/[^/]+\/?$/i.test(path);
+  if (convention === 'always') {
+    const next = path.endsWith('/') ? path : `${path}/`;
+    return query ? `${next}?${query}` : next;
+  }
+  if (isRoot) return query ? `${path.replace(/\/$/, '/')}?${query}` : path.replace(/\/?$/, '/');
+  const next = path.replace(/\/+$/, '');
+  return query ? `${next}?${query}` : next;
+}

package/src/core/verify.js

@@ -1,21 +1,17 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
 import { blockThirdPartyRequests } from './blocking.js';
-import { toServerUrl } from './routes.js';
+import { DEFAULTS } from './config.js';
+import { frameworkFromMarkers, isHydrationError } from './frameworks.js';
+import { compareContainers } from './hydration-diff.js';
+import { routeToFile } from './output.js';
+import { isPageEmpty } from './page-content.js';
+import { resolveRouteConfig } from './route-config.js';
+import { normaliseRoute, toServerUrl } from './routes.js';
 import { mapLimit } from './scheduler.js';
 import { settlePage, trackNetwork } from './settle.js';
 
-const HYDRATION_PATTERNS = [
-  /hydrat/i,
-  /did not match/i,
-  /server-rendered HTML/i,
-  // React 16 and 17 phrase their mismatch warnings differently and never use the
-  // minified codes, which only exist from React 18 on.
-  /did not expect server HTML/i,
-  /expected server HTML to contain/i,
-  /react\.dev\/errors\/(418|419|421|422|423|424|425)/i,
-  /Minified React error #(418|419|421|422|423|424|425)\b/i,
-];
-
-// Containers used by the common React setups, so the probe can find the prerendered
+// Containers used by the common setups, so the probe can find the prerendered
 // subtree without configuration.
 const CONTAINER_SELECTOR = '#root, #app, #__next';
 
@@ -45,19 +41,9 @@ function bootProbe() {
   });
 }
 
-/**
- * Classifies a console or page error as a React hydration failure. Exported so the
- * pattern set is testable without a browser. Patterns stay narrow on purpose: a
- * false positive fails a user's build, so unrelated 4xx React codes and generic
- * "does not match" messages are deliberately not treated as hydration errors.
- */
-export function isHydrationError(message) {
-  return HYDRATION_PATTERNS.some((pattern) => pattern.test(message));
-}
-
 /**
  * Classifies how a prerendered page booted. `hydrated` means the app adopted the
- * markup, `re-rendered` means it discarded it, `undetected` means no React root was
+ * markup, `re-rendered` means it discarded it, `undetected` means no framework root was
  * found on the container, and `unknown` means the prerendered child was never seen.
  */
 export function classifyBoot({ markupPreserved, frameworkMounted }) {
@@ -71,16 +57,22 @@ export function classifyBoot({ markupPreserved, frameworkMounted }) {
  * hydration errors plus how each route booted. This is the pass that catches markup
  * that only breaks once the app re-attaches, and apps that never attach to it at all.
  */
-export async function verifyRoutes({ browser, origin, routes, config }) {
+export async function verifyRoutes({ browser, origin, routes, config, outputDir = null }) {
   const { results, errors } = await mapLimit(routes, config.concurrency, (route) =>
-    verifyRoute({ browser, origin, route, config }),
+    verifyRoute({ browser, origin, route, config: resolveRouteConfig(config, route), outputDir }),
   );
   const verified = results.filter(Boolean);
   const modes = { hydrated: 0, 're-rendered': 0, undetected: 0, unknown: 0 };
-  for (const entry of verified) modes[entry.mode] += 1;
+  const frameworks = {};
+  for (const entry of verified) {
+    modes[entry.mode] += 1;
+    const name = entry.framework ?? 'unknown';
+    frameworks[name] = (frameworks[name] ?? 0) + 1;
+  }
   return {
     routes: verified,
     modes,
+    frameworks,
     ok: verified.every((entry) => entry.ok) && errors.length === 0,
     errors: errors.map(({ item, error }) => ({ route: item, message: error.message })),
   };
@@ -90,26 +82,23 @@ export async function verifyRoutes({ browser, origin, routes, config }) {
  * Verifies one route against the same request rules as the render pass, so what is
  * verified matches what was rendered.
  */
-async function verifyRoute({ browser, origin, route, config }) {
+async function verifyRoute({ browser, origin, route, config, outputDir }) {
   const context = await browser.newContext({
     viewport: config.viewport,
     storageState: config.storageState ?? undefined,
-    userAgent: config.userAgent ?? undefined,
+    userAgent:
+      config.userAgent === DEFAULTS.userAgent ? undefined : (config.userAgent ?? undefined),
     ignoreHTTPSErrors: config.ignoreHTTPSErrors,
+    reducedMotion: config.freezeAnimations ? 'reduce' : 'no-preference',
+    serviceWorkers: 'block',
   });
   const page = await context.newPage();
-  const hydrationErrors = [];
-  const consoleErrors = [];
-  const pageErrors = [];
-  // Hydration messages arrive through either channel, so both are classified the same way.
-  const collect = (message, target) => {
-    if (isHydrationError(message)) hydrationErrors.push(message);
-    else target.push(message);
-  };
+  const consoleMessages = [];
+  const pageMessages = [];
   page.on('console', (message) => {
-    if (message.type() === 'error') collect(message.text(), consoleErrors);
+    if (message.type() === 'error') consoleMessages.push(message.text());
   });
-  page.on('pageerror', (error) => collect(error.message, pageErrors));
+  page.on('pageerror', (error) => pageMessages.push(error.message));
 
   const tracker = trackNetwork(page);
   try {
@@ -117,32 +106,97 @@ async function verifyRoute({ browser, origin, route, config }) {
       await blockThirdPartyRequests(page, origin, config.allowedHosts);
     }
     await page.addInitScript(bootProbe);
-    await page.goto(toServerUrl(route, config.base, origin), {
+    const response = await page.goto(toServerUrl(route, config.base, origin), {
       waitUntil: 'domcontentloaded',
       timeout: config.timeout,
     });
+    const status = response?.status() ?? null;
     await settlePage(page, config, tracker);
+    const empty = await isPageEmpty(page);
+    const statusCode = await page.evaluate(() => {
+      const marker = document.querySelector('meta[name="prerender-status-code"]');
+      if (!marker) return null;
+      const value = Number.parseInt(marker.getAttribute('content') ?? '', 10);
+      return Number.isFinite(value) ? value : null;
+    });
+    const notFoundRoute = config.notFoundRoute ? normaliseRoute(config.notFoundRoute) : null;
+    const isNotFoundRoute = notFoundRoute !== null && normaliseRoute(route) === notFoundRoute;
+    const notFound = statusCode === 404 || isNotFoundRoute;
 
     const boot = await page.evaluate((selector) => {
       const probe = window.__snappyBoot ?? {};
       const container =
         document.querySelector(selector) ?? document.body?.firstElementChild ?? null;
-      const frameworkMounted = container
-        ? Object.keys(container).some(
-            (key) => key.startsWith('__reactContainer') || key === '_reactRootContainer',
-          )
-        : false;
+      const markers = container
+        ? {
+            react: Object.keys(container).some(
+              (key) => key.startsWith('__reactContainer') || key === '_reactRootContainer',
+            ),
+            vue: Boolean(
+              container.__vue_app__ ||
+                container.__vueParentComponent ||
+                container._vnode ||
+                container.__vue__,
+            ),
+            preact: Boolean(
+              container.__k ||
+                container._children ||
+                container.firstElementChild?.__k ||
+                container.firstElementChild?._children,
+            ),
+            svelte: Boolean(
+              container.__svelte_meta || container.querySelector?.('[data-svelte-h]'),
+            ),
+          }
+        : {};
       const child = probe.prerenderedChild ?? null;
-      return { frameworkMounted, markupPreserved: child ? child.isConnected : null };
+      return { markers, markupPreserved: child ? child.isConnected : null };
     }, CONTAINER_SELECTOR);
+    const detected = frameworkFromMarkers(boot.markers, config.framework);
+    // Classify only after the framework is known, so an explicit `framework` choice and
+    // the detected framework select the matching message patterns.
+    const hydrationErrors = [];
+    const consoleErrors = [];
+    const pageErrors = [];
+    for (const message of consoleMessages) {
+      if (isHydrationError(message, detected.name ?? 'auto')) hydrationErrors.push(message);
+      else consoleErrors.push(message);
+    }
+    for (const message of pageMessages) {
+      if (isHydrationError(message, detected.name ?? 'auto')) hydrationErrors.push(message);
+      else pageErrors.push(message);
+    }
+    const mode = classifyBoot({
+      markupPreserved: boot.markupPreserved,
+      frameworkMounted: detected.mounted,
+    });
+    let firstMismatch = null;
+    if (outputDir && (hydrationErrors.length > 0 || mode === 're-rendered')) {
+      const liveHtml = await page.evaluate((selector) => {
+        const container = document.querySelector(selector) ?? document.body;
+        return container?.outerHTML ?? null;
+      }, CONTAINER_SELECTOR);
+      const file = path.join(outputDir, routeToFile(route, config));
+      const expectedHtml = await fs.readFile(file, 'utf8').catch(() => null);
+      if (liveHtml && expectedHtml) firstMismatch = compareContainers(expectedHtml, liveHtml);
+    }
 
     return {
       route,
+      status,
+      statusCode,
+      notFound,
+      framework: detected.name,
+      empty,
       hydrationErrors,
       consoleErrors,
       pageErrors,
-      ok: hydrationErrors.length === 0,
-      mode: classifyBoot(boot),
+      ok:
+        hydrationErrors.length === 0 &&
+        (status === null || status < 400 || (isNotFoundRoute && status === 404)) &&
+        !empty,
+      mode,
+      firstMismatch,
       markupPreserved: boot.markupPreserved,
     };
   } finally {